From e9b303322c81acb4c1fc742fa67d2aa5b9f48910 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Thu, 26 Sep 2024 05:11:01 -0400 Subject: [PATCH 01/43] Fix packaging tests after addition of new wolfi-based image (#112831) * Add more missing wolfi references to fix tests * packaging tests require access to docker registry * Fix symlink for es distributions jdk cacerts in wolfi docker * Fix native support on wolfi images * Fix provided keystore packaging tests for wolfi * Add utils used for testing to wolfi image * Explicitly set default shell to bash in docker images * Fix docker config issues * Apply review feedback around docker login --------- Co-authored-by: Rene Groeschke --- .buildkite/hooks/pre-command | 12 ++++++++---- .buildkite/pipelines/periodic-packaging.yml | 3 ++- .ci/scripts/packaging-test.sh | 1 + .../gradle/internal/docker/DockerBuildTask.java | 11 +++++++++++ distribution/docker/src/docker/Dockerfile | 13 +++++++++++-- .../elasticsearch/packaging/test/DockerTests.java | 3 +++ .../packaging/test/KeystoreManagementTests.java | 5 ++++- .../packaging/test/PackagingTestCase.java | 6 ++++-- .../elasticsearch/packaging/util/Distribution.java | 7 +++++-- .../elasticsearch/packaging/util/docker/Docker.java | 4 ++-- .../packaging/util/docker/DockerRun.java | 1 + 11 files changed, 52 insertions(+), 14 deletions(-) diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index b6b730fc3de8b..0c0ede8c3a076 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -78,11 +78,15 @@ if [[ "${USE_SNYK_CREDENTIALS:-}" == "true" ]]; then fi if [[ "${USE_PROD_DOCKER_CREDENTIALS:-}" == "true" ]]; then - DOCKER_REGISTRY_USERNAME="$(vault read -field=username secret/ci/elastic-elasticsearch/migrated/prod_docker_registry_credentials)" - export DOCKER_REGISTRY_USERNAME + if which docker > /dev/null 2>&1; then + DOCKER_REGISTRY_USERNAME="$(vault read -field=username secret/ci/elastic-elasticsearch/migrated/prod_docker_registry_credentials)" + export DOCKER_REGISTRY_USERNAME - DOCKER_REGISTRY_PASSWORD="$(vault read -field=password secret/ci/elastic-elasticsearch/migrated/prod_docker_registry_credentials)" - export DOCKER_REGISTRY_PASSWORD + DOCKER_REGISTRY_PASSWORD="$(vault read -field=password secret/ci/elastic-elasticsearch/migrated/prod_docker_registry_credentials)" + export DOCKER_REGISTRY_PASSWORD + + docker login --username "$DOCKER_REGISTRY_USERNAME" --password "$DOCKER_REGISTRY_PASSWORD" docker.elastic.co + fi fi if [[ "$BUILDKITE_AGENT_META_DATA_PROVIDER" != *"k8s"* ]]; then diff --git a/.buildkite/pipelines/periodic-packaging.yml b/.buildkite/pipelines/periodic-packaging.yml index 8ef8f5954887e..76cc543a6898e 100644 --- a/.buildkite/pipelines/periodic-packaging.yml +++ b/.buildkite/pipelines/periodic-packaging.yml @@ -30,7 +30,8 @@ steps: image: family/elasticsearch-{{matrix.image}} diskSizeGb: 350 machineType: n1-standard-8 - env: {} + env: + USE_PROD_DOCKER_CREDENTIALS: "true" - group: packaging-tests-upgrade steps: - label: "{{matrix.image}} / 8.0.1 / packaging-tests-upgrade" diff --git a/.ci/scripts/packaging-test.sh b/.ci/scripts/packaging-test.sh index 6b9938dabffa8..bb7547933b213 100755 --- a/.ci/scripts/packaging-test.sh +++ b/.ci/scripts/packaging-test.sh @@ -77,5 +77,6 @@ sudo -E env \ --unset=ES_JAVA_HOME \ --unset=JAVA_HOME \ SYSTEM_JAVA_HOME=`readlink -f -n $BUILD_JAVA_HOME` \ + DOCKER_CONFIG="${HOME}/.docker" \ ./gradlew -g $HOME/.gradle --scan --parallel --build-cache -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/ --continue $@ diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/DockerBuildTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/DockerBuildTask.java index 8971f27838578..9b28401994ee2 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/DockerBuildTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/DockerBuildTask.java @@ -30,6 +30,7 @@ import org.gradle.api.tasks.PathSensitivity; import org.gradle.api.tasks.TaskAction; import org.gradle.process.ExecOperations; +import org.gradle.process.ExecSpec; import org.gradle.workers.WorkAction; import org.gradle.workers.WorkParameters; import org.gradle.workers.WorkerExecutor; @@ -166,6 +167,7 @@ private void pullBaseImage(String baseImage) { for (int attempt = 1; attempt <= maxAttempts; attempt++) { try { LoggedExec.exec(execOperations, spec -> { + maybeConfigureDockerConfig(spec); spec.executable("docker"); spec.args("pull"); spec.args(baseImage); @@ -181,6 +183,13 @@ private void pullBaseImage(String baseImage) { throw new GradleException("Failed to pull Docker base image [" + baseImage + "], all attempts failed"); } + private void maybeConfigureDockerConfig(ExecSpec spec) { + String dockerConfig = System.getenv("DOCKER_CONFIG"); + if (dockerConfig != null) { + spec.environment("DOCKER_CONFIG", dockerConfig); + } + } + @Override public void execute() { final Parameters parameters = getParameters(); @@ -193,6 +202,8 @@ public void execute() { final boolean isCrossPlatform = isCrossPlatform(); LoggedExec.exec(execOperations, spec -> { + maybeConfigureDockerConfig(spec); + spec.executable("docker"); if (isCrossPlatform) { diff --git a/distribution/docker/src/docker/Dockerfile b/distribution/docker/src/docker/Dockerfile index 47f79749cbefa..fd2516f2fdc9a 100644 --- a/distribution/docker/src/docker/Dockerfile +++ b/distribution/docker/src/docker/Dockerfile @@ -163,9 +163,16 @@ RUN <%= retry.loop(package_manager, " ${package_manager} update && \n" + " ${package_manager} upgrade && \n" + " ${package_manager} add --no-cache \n" + - " bash ca-certificates curl libsystemd netcat-openbsd p11-kit p11-kit-trust shadow tini unzip zip zstd && \n" + + " bash java-cacerts curl libstdc++ libsystemd netcat-openbsd p11-kit p11-kit-trust posix-libc-utils shadow tini unzip zip zstd && \n" + " rm -rf /var/cache/apk/* " ) %> + +# Set Bash as the default shell for future commands +SHELL ["/bin/bash", "-c"] + +# Optionally set Bash as the default shell in the container at runtime +CMD ["/bin/bash"] + <% } else if (docker_base == "default" || docker_base == "cloud") { %> # Change default shell to bash, then install required packages with retries. @@ -224,7 +231,7 @@ COPY --from=builder --chown=0:0 /opt /opt <% } %> ENV PATH /usr/share/elasticsearch/bin:\$PATH - +ENV SHELL /bin/bash COPY ${bin_dir}/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh # 1. Sync the user and group permissions of /etc/passwd @@ -249,6 +256,8 @@ RUN chmod g=u /etc/passwd && \\ # stays up-to-date with changes to Ubuntu's store) COPY bin/docker-openjdk /etc/ca-certificates/update.d/docker-openjdk RUN /etc/ca-certificates/update.d/docker-openjdk +<% } else if (docker_base == 'wolfi') { %> +RUN ln -sf /etc/ssl/certs/java/cacerts /usr/share/elasticsearch/jdk/lib/security/cacerts <% } else { %> RUN ln -sf /etc/pki/ca-trust/extracted/java/cacerts /usr/share/elasticsearch/jdk/lib/security/cacerts <% } %> diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/DockerTests.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/DockerTests.java index a9402c324f7fc..f588b78c78cc8 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/DockerTests.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/DockerTests.java @@ -386,6 +386,9 @@ public void test040JavaUsesTheOsProvidedKeystore() { if (distribution.packaging == Packaging.DOCKER_UBI || distribution.packaging == Packaging.DOCKER_IRON_BANK) { // In these images, the `cacerts` file ought to be a symlink here assertThat(path, equalTo("/etc/pki/ca-trust/extracted/java/cacerts")); + } else if (distribution.packaging == Packaging.DOCKER_WOLFI) { + // In these images, the `cacerts` file ought to be a symlink here + assertThat(path, equalTo("/etc/ssl/certs/java/cacerts")); } else { // Whereas on other images, it's a real file so the real path is the same assertThat(path, equalTo("/usr/share/elasticsearch/jdk/lib/security/cacerts")); diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/KeystoreManagementTests.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/KeystoreManagementTests.java index 5b86796aa80ca..a988a446f561f 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/KeystoreManagementTests.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/KeystoreManagementTests.java @@ -436,7 +436,10 @@ private void verifyKeystorePermissions() { switch (distribution.packaging) { case TAR, ZIP -> assertThat(keystore, file(File, ARCHIVE_OWNER, ARCHIVE_OWNER, p660)); case DEB, RPM -> assertThat(keystore, file(File, "root", "elasticsearch", p660)); - case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD, DOCKER_CLOUD_ESS -> assertThat(keystore, DockerFileMatcher.file(p660)); + case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD, DOCKER_CLOUD_ESS, DOCKER_WOLFI -> assertThat( + keystore, + DockerFileMatcher.file(p660) + ); default -> throw new IllegalStateException("Unknown Elasticsearch packaging type."); } } diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java index a1a9af3b6e307..644990105f60f 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java @@ -245,7 +245,7 @@ protected static void install() throws Exception { installation = Packages.installPackage(sh, distribution); Packages.verifyPackageInstallation(installation, distribution, sh); } - case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD, DOCKER_CLOUD_ESS -> { + case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD, DOCKER_CLOUD_ESS, DOCKER_WOLFI -> { installation = Docker.runContainer(distribution); Docker.verifyContainerInstallation(installation); } @@ -337,6 +337,7 @@ public Shell.Result runElasticsearchStartCommand(String password, boolean daemon case DOCKER_IRON_BANK: case DOCKER_CLOUD: case DOCKER_CLOUD_ESS: + case DOCKER_WOLFI: // nothing, "installing" docker image is running it return Shell.NO_OP; default: @@ -359,6 +360,7 @@ public void stopElasticsearch() throws Exception { case DOCKER_IRON_BANK: case DOCKER_CLOUD: case DOCKER_CLOUD_ESS: + case DOCKER_WOLFI: // nothing, "installing" docker image is running it break; default: @@ -371,7 +373,7 @@ public void awaitElasticsearchStartup(Shell.Result result) throws Exception { switch (distribution.packaging) { case TAR, ZIP -> Archives.assertElasticsearchStarted(installation); case DEB, RPM -> Packages.assertElasticsearchStarted(sh, installation); - case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD, DOCKER_CLOUD_ESS -> Docker.waitForElasticsearchToStart(); + case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD, DOCKER_CLOUD_ESS, DOCKER_WOLFI -> Docker.waitForElasticsearchToStart(); default -> throw new IllegalStateException("Unknown Elasticsearch packaging type."); } } diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/Distribution.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/Distribution.java index b3ea54425af8e..05cef4a0818ba 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/Distribution.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/Distribution.java @@ -37,6 +37,8 @@ public Distribution(Path path) { this.packaging = Packaging.DOCKER_CLOUD; } else if (filename.endsWith(".cloud-ess.tar")) { this.packaging = Packaging.DOCKER_CLOUD_ESS; + } else if (filename.endsWith(".wolfi.tar")) { + this.packaging = Packaging.DOCKER_WOLFI; } else { int lastDot = filename.lastIndexOf('.'); this.packaging = Packaging.valueOf(filename.substring(lastDot + 1).toUpperCase(Locale.ROOT)); @@ -61,7 +63,7 @@ public boolean isPackage() { */ public boolean isDocker() { return switch (packaging) { - case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD, DOCKER_CLOUD_ESS -> true; + case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD, DOCKER_CLOUD_ESS, DOCKER_WOLFI -> true; default -> false; }; } @@ -76,7 +78,8 @@ public enum Packaging { DOCKER_UBI(".ubi.tar", Platforms.isDocker()), DOCKER_IRON_BANK(".ironbank.tar", Platforms.isDocker()), DOCKER_CLOUD(".cloud.tar", Platforms.isDocker()), - DOCKER_CLOUD_ESS(".cloud-ess.tar", Platforms.isDocker()); + DOCKER_CLOUD_ESS(".cloud-ess.tar", Platforms.isDocker()), + DOCKER_WOLFI(".wolfi.tar", Platforms.isDocker()); /** The extension of this distribution's file */ public final String extension; diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/Docker.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/Docker.java index cb8a955a5972c..c38eaa58f0552 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/Docker.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/Docker.java @@ -486,9 +486,9 @@ public static void verifyContainerInstallation(Installation es) { // Ensure the `elasticsearch` user and group exist. // These lines will both throw an exception if the command fails dockerShell.run("id elasticsearch"); - dockerShell.run("getent group elasticsearch"); + dockerShell.run("grep -E '^elasticsearch:' /etc/group"); - final Shell.Result passwdResult = dockerShell.run("getent passwd elasticsearch"); + final Shell.Result passwdResult = dockerShell.run("grep -E '^elasticsearch:' /etc/passwd"); final String homeDir = passwdResult.stdout().trim().split(":")[5]; assertThat("elasticsearch user's home directory is incorrect", homeDir, equalTo("/usr/share/elasticsearch")); diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/DockerRun.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/DockerRun.java index 6c58bcba09879..2b3eb7ff7a617 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/DockerRun.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/DockerRun.java @@ -167,6 +167,7 @@ public static String getImageName(Distribution distribution) { case DOCKER_IRON_BANK -> "-ironbank"; case DOCKER_CLOUD -> "-cloud"; case DOCKER_CLOUD_ESS -> "-cloud-ess"; + case DOCKER_WOLFI -> "-wolfi"; default -> throw new IllegalStateException("Unexpected distribution packaging type: " + distribution.packaging); }; From 11c0bf8d6ee4bbd6e982a04211cc3994c5dd48e0 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Thu, 26 Sep 2024 11:19:46 +0200 Subject: [PATCH 02/43] Fix cancellation race condition in `onIndexAvailableForSearch` (#113386) This PR fixes following race conditions in `onIndexAvailableForSearch` introduced in https://github.com/elastic/elasticsearch/pull/112813: 1. If the method is called when the index is already available, cancellation is still scheduled and may execute before successful completion (manifested in test failures https://github.com/elastic/elasticsearch/issues/113336) 2. If the cancel task runs _before_ `addStateListener`, it may fail to remove the listener (noticed while fixing the first issue) These race conditions only manifest for small timeout windows, and are completely bypassed for 0 timeout windows based on other checks in prod code, so the practical impact is fortunately limited. Resolves: https://github.com/elastic/elasticsearch/issues/113336 --- muted-tests.yml | 3 - .../SecurityIndexManagerIntegTests.java | 110 +++++++++++++++++- .../support/SecurityIndexManager.java | 86 ++++++++++---- 3 files changed, 168 insertions(+), 31 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index 602d790246648..728f3d0bd6c72 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -251,9 +251,6 @@ tests: - class: org.elasticsearch.xpack.test.rest.XPackRestIT method: test {p0=transform/transforms_force_delete/Test force deleting a running transform} issue: https://github.com/elastic/elasticsearch/issues/113327 -- class: org.elasticsearch.xpack.security.support.SecurityIndexManagerIntegTests - method: testOnIndexAvailableForSearchIndexAlreadyAvailable - issue: https://github.com/elastic/elasticsearch/issues/113336 - class: org.elasticsearch.xpack.test.rest.XPackRestIT method: test {p0=analytics/top_metrics/sort by scaled float field} issue: https://github.com/elastic/elasticsearch/issues/113340 diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerIntegTests.java index 32337f0d66896..44cbf03f220a1 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerIntegTests.java @@ -26,18 +26,24 @@ import org.elasticsearch.xpack.core.security.action.user.PutUserResponse; import org.elasticsearch.xpack.security.authz.store.NativePrivilegeStore; import org.hamcrest.Matchers; +import org.junit.After; import org.junit.Before; import java.util.ArrayList; import java.util.List; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.xpack.security.support.SecuritySystemIndices.SECURITY_MAIN_ALIAS; import static org.hamcrest.Matchers.arrayContaining; +import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.notNullValue; @@ -45,6 +51,14 @@ public class SecurityIndexManagerIntegTests extends SecurityIntegTestCase { + private final int concurrentCallsToOnAvailable = 6; + private final ExecutorService executor = Executors.newFixedThreadPool(concurrentCallsToOnAvailable); + + @After + public void shutdownExecutor() { + executor.shutdown(); + } + public void testConcurrentOperationsTryingToCreateSecurityIndexAndAlias() throws Exception { final int processors = Runtime.getRuntime().availableProcessors(); final int numThreads = Math.min(50, scaledRandomIntBetween((processors + 1) / 2, 4 * processors)); // up to 50 threads @@ -110,6 +124,12 @@ public void testOnIndexAvailableForSearchIndexCompletesWithinTimeout() throws Ex // pick longer wait than in the assertBusy that waits for below to ensure index has had enough time to initialize securityIndexManager.onIndexAvailableForSearch((ActionListener) future, TimeValue.timeValueSeconds(40)); + // check listener added + assertThat( + securityIndexManager.getStateChangeListeners(), + hasItem(instanceOf(SecurityIndexManager.StateConsumerWithCancellable.class)) + ); + createSecurityIndexWithWaitForActiveShards(); assertBusy( @@ -121,6 +141,12 @@ public void testOnIndexAvailableForSearchIndexCompletesWithinTimeout() throws Ex // security index creation is complete and index is available for search; therefore whenIndexAvailableForSearch should report // success in time future.actionGet(); + + // check no remaining listeners + assertThat( + securityIndexManager.getStateChangeListeners(), + not(hasItem(instanceOf(SecurityIndexManager.StateConsumerWithCancellable.class))) + ); } @SuppressWarnings("unchecked") @@ -152,6 +178,69 @@ public void testOnIndexAvailableForSearchIndexAlreadyAvailable() throws Exceptio securityIndexManager.onIndexAvailableForSearch((ActionListener) future, TimeValue.timeValueSeconds(10)); future.actionGet(); } + + // check no remaining listeners + assertThat( + securityIndexManager.getStateChangeListeners(), + not(hasItem(instanceOf(SecurityIndexManager.StateConsumerWithCancellable.class))) + ); + } + + @SuppressWarnings("unchecked") + public void testOnIndexAvailableForSearchIndexUnderConcurrentLoad() throws Exception { + final SecurityIndexManager securityIndexManager = internalCluster().getInstances(NativePrivilegeStore.class) + .iterator() + .next() + .getSecurityIndexManager(); + // Long time out calls should all succeed + final List> futures = new ArrayList<>(); + for (int i = 0; i < concurrentCallsToOnAvailable / 2; i++) { + final Future future = executor.submit(() -> { + try { + final ActionFuture f = new PlainActionFuture<>(); + securityIndexManager.onIndexAvailableForSearch((ActionListener) f, TimeValue.timeValueSeconds(40)); + f.actionGet(); + } catch (Exception ex) { + fail(ex, "should not have encountered exception"); + } + return null; + }); + futures.add(future); + } + + // short time-out tasks should all time out + for (int i = 0; i < concurrentCallsToOnAvailable / 2; i++) { + final Future future = executor.submit(() -> { + expectThrows(ElasticsearchTimeoutException.class, () -> { + final ActionFuture f = new PlainActionFuture<>(); + securityIndexManager.onIndexAvailableForSearch((ActionListener) f, TimeValue.timeValueMillis(10)); + f.actionGet(); + }); + return null; + }); + futures.add(future); + } + + // Sleep a second for short-running calls to timeout + Thread.sleep(1000); + + createSecurityIndexWithWaitForActiveShards(); + // ensure security index manager state is fully in the expected precondition state for this test (ready for search) + assertBusy( + () -> assertThat(securityIndexManager.isAvailable(SecurityIndexManager.Availability.SEARCH_SHARDS), is(true)), + 30, + TimeUnit.SECONDS + ); + + for (var future : futures) { + future.get(10, TimeUnit.SECONDS); + } + + // check no remaining listeners + assertThat( + securityIndexManager.getStateChangeListeners(), + not(hasItem(instanceOf(SecurityIndexManager.StateConsumerWithCancellable.class))) + ); } @SuppressWarnings("unchecked") @@ -163,9 +252,24 @@ public void testOnIndexAvailableForSearchIndexWaitTimeOut() { .next() .getSecurityIndexManager(); - final ActionFuture future = new PlainActionFuture<>(); - securityIndexManager.onIndexAvailableForSearch((ActionListener) future, TimeValue.timeValueMillis(100)); - expectThrows(ElasticsearchTimeoutException.class, future::actionGet); + { + final ActionFuture future = new PlainActionFuture<>(); + securityIndexManager.onIndexAvailableForSearch((ActionListener) future, TimeValue.timeValueMillis(100)); + expectThrows(ElasticsearchTimeoutException.class, future::actionGet); + } + + // Also works with 0 timeout + { + final ActionFuture future = new PlainActionFuture<>(); + securityIndexManager.onIndexAvailableForSearch((ActionListener) future, TimeValue.timeValueMillis(0)); + expectThrows(ElasticsearchTimeoutException.class, future::actionGet); + } + + // check no remaining listeners + assertThat( + securityIndexManager.getStateChangeListeners(), + not(hasItem(instanceOf(SecurityIndexManager.StateConsumerWithCancellable.class))) + ); } public void testSecurityIndexSettingsCannotBeChanged() throws Exception { diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java index a6c8de003c159..6d9b0ef6aeebe 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java @@ -54,6 +54,7 @@ import java.util.Objects; import java.util.Set; import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.BiConsumer; import java.util.function.Consumer; import java.util.stream.Collectors; @@ -364,45 +365,80 @@ public void accept(State previousState, State nextState) { * Notifies {@code listener} once the security index is available, or calls {@code onFailure} on {@code timeout}. */ public void onIndexAvailableForSearch(ActionListener listener, TimeValue timeout) { - logger.info("Will wait for security index [{}] to become available for search", getConcreteIndexName()); + logger.info("Will wait for security index [{}] for [{}] to become available for search", getConcreteIndexName(), timeout); - final ActionListener notifyOnceListener = ActionListener.notifyOnce(listener); + if (state.indexAvailableForSearch) { + logger.debug("Security index [{}] is already available", getConcreteIndexName()); + listener.onResponse(null); + return; + } + final AtomicBoolean isDone = new AtomicBoolean(false); final var indexAvailableForSearchListener = new StateConsumerWithCancellable() { @Override public void accept(SecurityIndexManager.State previousState, SecurityIndexManager.State nextState) { if (nextState.indexAvailableForSearch) { - assert cancellable != null; - // cancel and removeStateListener are idempotent - cancellable.cancel(); - removeStateListener(this); - notifyOnceListener.onResponse(null); + if (isDone.compareAndSet(false, true)) { + cancel(); + removeStateListener(this); + listener.onResponse(null); + } } } }; + // add listener _before_ registering timeout -- this way we are guaranteed it gets removed (either by timeout below, or successful + // completion above) + addStateListener(indexAvailableForSearchListener); + // schedule failure handling on timeout -- keep reference to cancellable so a successful completion can cancel the timeout - indexAvailableForSearchListener.cancellable = client.threadPool().schedule(() -> { - removeStateListener(indexAvailableForSearchListener); - notifyOnceListener.onFailure( - new ElasticsearchTimeoutException( - "timed out waiting for security index [" + getConcreteIndexName() + "] to become available for search" - ) - ); - }, timeout, client.threadPool().generic()); + indexAvailableForSearchListener.setCancellable(client.threadPool().schedule(() -> { + if (isDone.compareAndSet(false, true)) { + removeStateListener(indexAvailableForSearchListener); + listener.onFailure( + new ElasticsearchTimeoutException( + "timed out waiting for security index [" + getConcreteIndexName() + "] to become available for search" + ) + ); + } + }, timeout, client.threadPool().generic())); + } - // in case the state has meanwhile changed to available, return immediately - if (state.indexAvailableForSearch) { - indexAvailableForSearchListener.cancellable.cancel(); - notifyOnceListener.onResponse(null); - } else { - addStateListener(indexAvailableForSearchListener); - } + // pkg-private for testing + List> getStateChangeListeners() { + return stateChangeListeners; } - private abstract static class StateConsumerWithCancellable + /** + * This class ensures that if cancel() is called _before_ setCancellable(), the passed-in cancellable is still correctly cancelled on + * a subsequent setCancellable() call. + */ + // pkg-private for testing + abstract static class StateConsumerWithCancellable implements - BiConsumer { - volatile Scheduler.ScheduledCancellable cancellable; + BiConsumer, + Scheduler.Cancellable { + private volatile Scheduler.ScheduledCancellable cancellable; + private volatile boolean cancelled = false; + + void setCancellable(Scheduler.ScheduledCancellable cancellable) { + this.cancellable = cancellable; + if (cancelled) { + cancel(); + } + } + + public boolean cancel() { + cancelled = true; + if (cancellable != null) { + // cancellable is idempotent, so it's fine to potentially call it multiple times + return cancellable.cancel(); + } + return isCancelled(); + } + + public boolean isCancelled() { + return cancelled; + } } private Tuple checkIndexAvailable(ClusterState state) { From 0d275c65dca4aeba90bd8ed632644a433c817408 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Thu, 26 Sep 2024 12:06:28 +0200 Subject: [PATCH 03/43] Log when clients are randomly reset in ITs (#113510) This is useful for debugging tests and should not be too noisy since it's a rare event. --- .../main/java/org/elasticsearch/test/InternalTestCluster.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java index ff66d59a21c5b..7a04384298933 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java +++ b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java @@ -1536,7 +1536,9 @@ private void randomlyResetClients() { // only reset the clients on nightly tests, it causes heavy load... if (RandomizedTest.isNightly() && rarely(random)) { final Collection nodesAndClients = nodes.values(); + logger.info("Resetting [{}] node clients on internal test cluster", nodesAndClients.size()); for (NodeAndClient nodeAndClient : nodesAndClients) { + logger.info("Resetting [{}] node client on internal test cluster", nodeAndClient.name); nodeAndClient.resetClient(); } } From 98db01b271680732a0862db1a07f6677b49672c3 Mon Sep 17 00:00:00 2001 From: Luigi Dell'Aquila Date: Thu, 26 Sep 2024 12:22:49 +0200 Subject: [PATCH 04/43] ES|QL: Fix warnings for date tests (#113586) Fixes: https://github.com/elastic/elasticsearch/issues/113540 Fixes: https://github.com/elastic/elasticsearch/issues/113539 More generic warning regex for Java 23 date patterns (already fixed in 8.x, no need to backport) --- muted-tests.yml | 6 ------ .../resources/rest-api-spec/test/esql/70_locale.yml | 4 ++-- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index 728f3d0bd6c72..cf821355b1d3c 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -290,12 +290,6 @@ tests: - class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT method: test {p0=search/180_locale_dependent_mapping/Test Index and Search locale dependent mappings / dates} issue: https://github.com/elastic/elasticsearch/issues/113537 -- class: org.elasticsearch.xpack.esql.qa.mixed.EsqlClientYamlIT - method: test {p0=esql/70_locale/Date format with default locale} - issue: https://github.com/elastic/elasticsearch/issues/113539 -- class: org.elasticsearch.xpack.esql.qa.mixed.EsqlClientYamlIT - method: test {p0=esql/70_locale/Date format with Italian locale} - issue: https://github.com/elastic/elasticsearch/issues/113540 - class: org.elasticsearch.xpack.inference.TextEmbeddingCrudIT method: testPutE5WithTrainedModelAndInference issue: https://github.com/elastic/elasticsearch/issues/113565 diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/70_locale.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/70_locale.yml index 05edf6cdfb5a8..5a9a2a21e21bc 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/70_locale.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/70_locale.yml @@ -29,7 +29,7 @@ setup: - do: allowed_warnings_regex: - "No limit defined, adding default limit of \\[.*\\]" - - "Date format \\[MMMM\\] contains textual field specifiers that could change in JDK 23" + - "Date format \\[MMMM\\] contains textual field specifiers that could change in JDK 23.*" esql.query: body: query: 'FROM events | eval fixed_format = date_format("MMMM", @timestamp), variable_format = date_format(format, @timestamp) | sort @timestamp | keep @timestamp, fixed_format, variable_format' @@ -51,7 +51,7 @@ setup: - do: allowed_warnings_regex: - "No limit defined, adding default limit of \\[.*\\]" - - "Date format \\[MMMM\\] contains textual field specifiers that could change in JDK 23" + - "Date format \\[MMMM\\] contains textual field specifiers that could change in JDK 23.*" esql.query: body: query: 'FROM events | eval fixed_format = date_format("MMMM", @timestamp), variable_format = date_format(format, @timestamp) | sort @timestamp | keep @timestamp, fixed_format, variable_format' From b344493298a0db19f9e95d8252fe90e79102dd5a Mon Sep 17 00:00:00 2001 From: Kostas Krikellas <131142368+kkrik-es@users.noreply.github.com> Date: Thu, 26 Sep 2024 14:57:31 +0300 Subject: [PATCH 05/43] Rest skipped tests after backporting (#113591) The skip test entries were added in https://github.com/elastic/elasticsearch/pull/113584, no longer needed after backporting it. --- modules/dot-prefix-validation/build.gradle | 10 ---------- rest-api-spec/build.gradle | 22 ---------------------- 2 files changed, 32 deletions(-) diff --git a/modules/dot-prefix-validation/build.gradle b/modules/dot-prefix-validation/build.gradle index b300cae20d717..6e232570b4a22 100644 --- a/modules/dot-prefix-validation/build.gradle +++ b/modules/dot-prefix-validation/build.gradle @@ -27,13 +27,3 @@ tasks.named('yamlRestTest') { tasks.named('yamlRestCompatTest') { usesDefaultDistribution() } - -tasks.named("yamlRestCompatTestTransform").configure( - { task -> - task.skipTest("tsdb/140_routing_path/multi-value routing path field", "Multi-value routing paths are allowed now. See #112645") - task.skipTest( - "dot_prefix/10_basic/Deprecated index template with a dot prefix index pattern", - "Tentantively disabled until #112092 gets backported to 8.x" - ) - } -) diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index 1d69c170d7553..a742e83255bbb 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -55,28 +55,6 @@ tasks.named("precommit").configure { } tasks.named("yamlRestCompatTestTransform").configure({task -> - task.skipTest("tsdb/140_routing_path/multi-value routing path field", "Multi-value routing paths are allowed now. See #112645") task.skipTest("indices.sort/10_basic/Index Sort", "warning does not exist for compatibility") task.skipTest("search/330_fetch_fields/Test search rewrite", "warning does not exist for compatibility") - task.skipTest("search/540_ignore_above_synthetic_source/ignore_above mapping level setting on arrays", "Temporary mute while backporting to 8.x") - task.skipTest("indices.create/20_synthetic_source/subobjects auto", "Tentantively disabled until #112092 gets backported to 8.x") - task.skipTest( - "index/92_metrics_auto_subobjects/Metrics object indexing with synthetic source", - "Tentantively disabled until #112092 gets backported to 8.x" - ) - task.skipTest( - "index/92_metrics_auto_subobjects/Root without subobjects with synthetic source", - "Tentantively disabled until #112092 gets backported to 8.x" - ) - task.skipTest( - "indices.put_index_template/15_composition/Composable index templates that include subobjects: auto at root", - "Tentantively disabled until #112092 gets backported to 8.x" - ) - task.skipTest( - "indices.put_index_template/15_composition/Composable index templates that include subobjects: auto on arbitrary field", - "Tentantively disabled until #112092 gets backported to 8.x" - ) - task.skipTest("index/92_metrics_auto_subobjects/Metrics object indexing", "Tentantively disabled until #112092 gets backported to 8.x") - task.skipTest("index/92_metrics_auto_subobjects/Root with metrics", "Tentantively disabled until #112092 gets backported to 8.x") - task.skipTest("search/330_fetch_fields/Test with subobjects: auto", "Tentantively disabled until #112092 gets backported to 8.x") }) From 5c6778cc8632950a50da4f34151bd0403c806dc6 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Thu, 26 Sep 2024 22:26:40 +1000 Subject: [PATCH 06/43] Mute org.elasticsearch.integration.KibanaUserRoleIntegTests testFieldMappings #113592 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index cf821355b1d3c..528f1e3be8d17 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -299,6 +299,9 @@ tests: - class: org.elasticsearch.xpack.ml.integration.MlJobIT method: testCantCreateJobWithSameID issue: https://github.com/elastic/elasticsearch/issues/113581 +- class: org.elasticsearch.integration.KibanaUserRoleIntegTests + method: testFieldMappings + issue: https://github.com/elastic/elasticsearch/issues/113592 # Examples: # From fc9954e031e3f94fca58765755fbddae93115656 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Thu, 26 Sep 2024 22:26:58 +1000 Subject: [PATCH 07/43] Mute org.elasticsearch.integration.KibanaUserRoleIntegTests testSearchAndMSearch #113593 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 528f1e3be8d17..5d7474af06d86 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -302,6 +302,9 @@ tests: - class: org.elasticsearch.integration.KibanaUserRoleIntegTests method: testFieldMappings issue: https://github.com/elastic/elasticsearch/issues/113592 +- class: org.elasticsearch.integration.KibanaUserRoleIntegTests + method: testSearchAndMSearch + issue: https://github.com/elastic/elasticsearch/issues/113593 # Examples: # From e2281a1158976bd5fac73c8e77e9a67c769dd3b2 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna <93581129+salvatore-campagna@users.noreply.github.com> Date: Thu, 26 Sep 2024 14:44:03 +0200 Subject: [PATCH 08/43] Introduce an `IndexSettingsProvider` to inject logsdb index mode (#113505) Here we introduce a new implementation of `IndexSettingProvider` whose goal is to "inject" the `index.mode` setting with value `logsdb` when a cluster setting `cluster.logsdb.enabled` is `true`. We also make sure that: * the existing `index.mode` is not set * the datastream name matches the `logs-*-*` pattern * `logs@settings` component template is used --- .../LogsIndexModeDisabledRestTestIT.java | 70 +++- .../LogsIndexModeEnabledRestTestIT.java | 87 ++++- .../logsdb/LogsIndexModeRestTestIT.java | 19 +- .../core/src/main/java/module-info.java | 1 + .../cluster/settings/ClusterSettings.java | 19 + .../main/resources/logs@settings-logsdb.json | 26 ++ .../src/main/resources/logs@settings.json | 1 - .../xpack/logsdb/LogsDBPlugin.java | 12 +- .../LogsdbIndexModeSettingsProvider.java | 89 +++++ .../LogsdbIndexModeSettingsProviderTests.java | 326 ++++++++++++++++++ .../stack/LegacyStackTemplateRegistry.java | 7 +- .../xpack/stack/StackPlugin.java | 2 +- .../xpack/stack/StackTemplateRegistry.java | 21 +- 13 files changed, 648 insertions(+), 32 deletions(-) create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/settings/ClusterSettings.java create mode 100644 x-pack/plugin/core/template-resources/src/main/resources/logs@settings-logsdb.json create mode 100644 x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java create mode 100644 x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeDisabledRestTestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeDisabledRestTestIT.java index c9818a34169de..123ca3b806153 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeDisabledRestTestIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeDisabledRestTestIT.java @@ -11,6 +11,7 @@ import org.elasticsearch.client.RestClient; import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.cluster.local.distribution.DistributionType; import org.hamcrest.Matchers; @@ -23,6 +24,22 @@ public class LogsIndexModeDisabledRestTestIT extends LogsIndexModeRestTestIT { + private static final String MAPPINGS = """ + { + "template": { + "mappings": { + "properties": { + "@timestamp": { + "type": "date" + }, + "message": { + "type": "text" + } + } + } + } + }"""; + @ClassRule() public static ElasticsearchCluster cluster = ElasticsearchCluster.local() .distribution(DistributionType.DEFAULT) @@ -50,8 +67,59 @@ public void setup() throws Exception { public void testLogsSettingsIndexModeDisabled() throws IOException { assertOK(createDataStream(client, "logs-custom-dev")); - final String indexMode = (String) getSetting(client, getDataStreamBackingIndex(client, "logs-custom-dev", 0), "index.mode"); + final String indexMode = (String) getSetting( + client, + getDataStreamBackingIndex(client, "logs-custom-dev", 0), + IndexSettings.MODE.getKey() + ); assertThat(indexMode, Matchers.not(equalTo(IndexMode.LOGSDB.getName()))); } + public void testTogglingLogsdb() throws IOException { + putComponentTemplate(client, "logs@settings", MAPPINGS); + assertOK(createDataStream(client, "logs-custom-dev")); + final String indexModeBefore = (String) getSetting( + client, + getDataStreamBackingIndex(client, "logs-custom-dev", 0), + IndexSettings.MODE.getKey() + ); + assertThat(indexModeBefore, Matchers.not(equalTo(IndexMode.LOGSDB.getName()))); + assertOK(putClusterSetting(client, "cluster.logsdb.enabled", "true")); + final String indexModeAfter = (String) getSetting( + client, + getDataStreamBackingIndex(client, "logs-custom-dev", 0), + IndexSettings.MODE.getKey() + ); + assertThat(indexModeAfter, Matchers.not(equalTo(IndexMode.LOGSDB.getName()))); + assertOK(rolloverDataStream(client, "logs-custom-dev")); + final String indexModeLater = (String) getSetting( + client, + getDataStreamBackingIndex(client, "logs-custom-dev", 1), + IndexSettings.MODE.getKey() + ); + assertThat(indexModeLater, equalTo(IndexMode.LOGSDB.getName())); + assertOK(putClusterSetting(client, "cluster.logsdb.enabled", "false")); + assertOK(rolloverDataStream(client, "logs-custom-dev")); + final String indexModeFinal = (String) getSetting( + client, + getDataStreamBackingIndex(client, "logs-custom-dev", 2), + IndexSettings.MODE.getKey() + ); + assertThat(indexModeFinal, Matchers.not(equalTo(IndexMode.LOGSDB.getName()))); + + } + + public void testEnablingLogsdb() throws IOException { + putComponentTemplate(client, "logs@settings", MAPPINGS); + assertOK(putClusterSetting(client, "cluster.logsdb.enabled", true)); + assertOK(createDataStream(client, "logs-custom-dev")); + final String indexMode = (String) getSetting( + client, + getDataStreamBackingIndex(client, "logs-custom-dev", 0), + IndexSettings.MODE.getKey() + ); + assertThat(indexMode, equalTo(IndexMode.LOGSDB.getName())); + assertOK(putClusterSetting(client, "cluster.logsdb.enabled", false)); + } + } diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeEnabledRestTestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeEnabledRestTestIT.java index d7bdf54007d69..a024a2c0f303c 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeEnabledRestTestIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeEnabledRestTestIT.java @@ -10,8 +10,10 @@ package org.elasticsearch.datastreams.logsdb; import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.RestClient; import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.cluster.local.distribution.DistributionType; import org.hamcrest.Matchers; @@ -179,7 +181,11 @@ public void setup() throws Exception { public void testCreateDataStream() throws IOException { assertOK(putComponentTemplate(client, "logs@custom", MAPPINGS)); assertOK(createDataStream(client, "logs-custom-dev")); - final String indexMode = (String) getSetting(client, getDataStreamBackingIndex(client, "logs-custom-dev", 0), "index.mode"); + final String indexMode = (String) getSetting( + client, + getDataStreamBackingIndex(client, "logs-custom-dev", 0), + IndexSettings.MODE.getKey() + ); assertThat(indexMode, equalTo(IndexMode.LOGSDB.getName())); } @@ -224,4 +230,83 @@ public void testRolloverDataStream() throws IOException { assertThat(firstBackingIndex, Matchers.not(equalTo(secondBackingIndex))); assertThat(getDataStreamBackingIndices(client, "logs-custom-dev").size(), equalTo(2)); } + + public void testLogsAtSettingWithStandardOverride() throws IOException { + assertOK(putComponentTemplate(client, "logs@custom", """ + { + "template": { + "settings": { + "index": { + "mode": "standard" + } + } + } + } + """)); + assertOK(createDataStream(client, "logs-custom-dev")); + final String indexMode = (String) getSetting( + client, + getDataStreamBackingIndex(client, "logs-custom-dev", 0), + IndexSettings.MODE.getKey() + ); + assertThat(indexMode, equalTo(IndexMode.STANDARD.getName())); + } + + public void testLogsAtSettingWithTimeSeriesOverride() throws IOException { + assertOK(putComponentTemplate(client, "logs@custom", """ + { + "template": { + "settings": { + "index": { + "routing_path": [ "hostname" ], + "mode": "time_series", + "sort.field": [], + "sort.order": [] + } + }, + "mappings": { + "properties": { + "hostname": { + "type": "keyword", + "time_series_dimension": true + } + } + } + } + } + """)); + assertOK(createDataStream(client, "logs-custom-dev")); + final String indexMode = (String) getSetting( + client, + getDataStreamBackingIndex(client, "logs-custom-dev", 0), + IndexSettings.MODE.getKey() + ); + assertThat(indexMode, equalTo(IndexMode.TIME_SERIES.getName())); + } + + public void testLogsAtSettingWithTimeSeriesOverrideFailure() { + // NOTE: apm@settings defines sorting on @timestamp and template composition results in index.mode "time_series" + // with a non-allowed index.sort.field '@timestamp'. This fails at template composition stage before the index is even created. + final ResponseException ex = assertThrows(ResponseException.class, () -> putComponentTemplate(client, "logs@custom", """ + { + "template": { + "settings": { + "index": { + "routing_path": [ "hostname" ], + "mode": "time_series" + } + }, + "mappings": { + "properties": { + "hostname": { + "type": "keyword", + "time_series_dimension": true + } + } + } + } + } + """)); + assertTrue(ex.getMessage().contains("[index.mode=time_series] is incompatible with [index.sort.field]")); + } } diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeRestTestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeRestTestIT.java index 7d65207794598..22ac2b6d7d239 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeRestTestIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeRestTestIT.java @@ -33,10 +33,16 @@ protected static void waitForLogs(RestClient client) throws Exception { }); } - protected static Response putComponentTemplate(final RestClient client, final String templateName, final String mappings) + protected static Response putComponentTemplate(final RestClient client, final String componentTemplate, final String contends) throws IOException { - final Request request = new Request("PUT", "/_component_template/" + templateName); - request.setJsonEntity(mappings); + final Request request = new Request("PUT", "/_component_template/" + componentTemplate); + request.setJsonEntity(contends); + return client.performRequest(request); + } + + protected static Response putTemplate(final RestClient client, final String template, final String contents) throws IOException { + final Request request = new Request("PUT", "/_index_template/" + template); + request.setJsonEntity(contents); return client.performRequest(request); } @@ -87,4 +93,11 @@ protected static Response bulkIndex(final RestClient client, final String dataSt bulkRequest.addParameter("refresh", "true"); return client.performRequest(bulkRequest); } + + protected static Response putClusterSetting(final RestClient client, final String settingName, final Object settingValue) + throws IOException { + final Request request = new Request("PUT", "/_cluster/settings"); + request.setJsonEntity("{ \"transient\": { \"" + settingName + "\": " + settingValue + " } }"); + return client.performRequest(request); + } } diff --git a/x-pack/plugin/core/src/main/java/module-info.java b/x-pack/plugin/core/src/main/java/module-info.java index 72436bb9d5171..47848310fe781 100644 --- a/x-pack/plugin/core/src/main/java/module-info.java +++ b/x-pack/plugin/core/src/main/java/module-info.java @@ -228,6 +228,7 @@ exports org.elasticsearch.xpack.core.watcher.trigger; exports org.elasticsearch.xpack.core.watcher.watch; exports org.elasticsearch.xpack.core.watcher; + exports org.elasticsearch.xpack.cluster.settings; provides org.elasticsearch.action.admin.cluster.node.info.ComponentVersionNumber with diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/settings/ClusterSettings.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/settings/ClusterSettings.java new file mode 100644 index 0000000000000..1127889783f16 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/settings/ClusterSettings.java @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.cluster.settings; + +import org.elasticsearch.common.settings.Setting; + +public class ClusterSettings { + public static final Setting CLUSTER_LOGSDB_ENABLED = Setting.boolSetting( + "cluster.logsdb.enabled", + false, + Setting.Property.Dynamic, + Setting.Property.NodeScope + ); +} diff --git a/x-pack/plugin/core/template-resources/src/main/resources/logs@settings-logsdb.json b/x-pack/plugin/core/template-resources/src/main/resources/logs@settings-logsdb.json new file mode 100644 index 0000000000000..eabdd6fb9fad2 --- /dev/null +++ b/x-pack/plugin/core/template-resources/src/main/resources/logs@settings-logsdb.json @@ -0,0 +1,26 @@ +{ + "template": { + "settings": { + "index": { + "lifecycle": { + "name": "logs" + }, + "mode": "logsdb", + "codec": "best_compression", + "mapping": { + "ignore_malformed": true, + "total_fields": { + "ignore_dynamic_beyond_limit": true + } + }, + "default_pipeline": "logs@default-pipeline" + } + } + }, + "_meta": { + "description": "default settings for the logs index template installed by x-pack", + "managed": true + }, + "version": ${xpack.stack.template.version}, + "deprecated": ${xpack.stack.template.deprecated} +} diff --git a/x-pack/plugin/core/template-resources/src/main/resources/logs@settings.json b/x-pack/plugin/core/template-resources/src/main/resources/logs@settings.json index e9a9f2611ad7b..ca2659b8d8dea 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/logs@settings.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/logs@settings.json @@ -5,7 +5,6 @@ "lifecycle": { "name": "logs" }, - "mode": "${xpack.stack.template.logsdb.index.mode}", "codec": "best_compression", "mapping": { "ignore_malformed": true, diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java index e38f953be96a3..833555a7884ea 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java @@ -17,6 +17,7 @@ import java.util.Collection; import java.util.List; +import static org.elasticsearch.xpack.cluster.settings.ClusterSettings.CLUSTER_LOGSDB_ENABLED; import static org.elasticsearch.xpack.logsdb.SyntheticSourceLicenseService.FALLBACK_SETTING; public class LogsDBPlugin extends Plugin { @@ -24,9 +25,12 @@ public class LogsDBPlugin extends Plugin { private final Settings settings; private final SyntheticSourceLicenseService licenseService; + private final LogsdbIndexModeSettingsProvider logsdbIndexModeSettingsProvider; + public LogsDBPlugin(Settings settings) { this.settings = settings; this.licenseService = new SyntheticSourceLicenseService(settings); + this.logsdbIndexModeSettingsProvider = new LogsdbIndexModeSettingsProvider(settings); } @Override @@ -34,6 +38,10 @@ public Collection createComponents(PluginServices services) { licenseService.setLicenseState(XPackPlugin.getSharedLicenseState()); var clusterSettings = services.clusterService().getClusterSettings(); clusterSettings.addSettingsUpdateConsumer(FALLBACK_SETTING, licenseService::setSyntheticSourceFallback); + clusterSettings.addSettingsUpdateConsumer( + CLUSTER_LOGSDB_ENABLED, + logsdbIndexModeSettingsProvider::updateClusterIndexModeLogsdbEnabled + ); // Nothing to share here: return super.createComponents(services); } @@ -43,11 +51,11 @@ public Collection getAdditionalIndexSettingProviders(Index if (DiscoveryNode.isStateless(settings)) { return List.of(); } - return List.of(new SyntheticSourceIndexSettingsProvider(licenseService)); + return List.of(new SyntheticSourceIndexSettingsProvider(licenseService), logsdbIndexModeSettingsProvider); } @Override public List> getSettings() { - return List.of(FALLBACK_SETTING); + return List.of(FALLBACK_SETTING, CLUSTER_LOGSDB_ENABLED); } } diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java new file mode 100644 index 0000000000000..3f6bb66dfa438 --- /dev/null +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.logsdb; + +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService; +import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.regex.Regex; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettingProvider; +import org.elasticsearch.index.IndexSettings; + +import java.time.Instant; +import java.util.List; +import java.util.Locale; + +import static org.elasticsearch.xpack.cluster.settings.ClusterSettings.CLUSTER_LOGSDB_ENABLED; + +final class LogsdbIndexModeSettingsProvider implements IndexSettingProvider { + private static final String LOGS_PATTERN = "logs-*-*"; + private volatile boolean isLogsdbEnabled; + + LogsdbIndexModeSettingsProvider(final Settings settings) { + this.isLogsdbEnabled = CLUSTER_LOGSDB_ENABLED.get(settings); + } + + void updateClusterIndexModeLogsdbEnabled(boolean isLogsdbEnabled) { + this.isLogsdbEnabled = isLogsdbEnabled; + } + + @Override + public Settings getAdditionalIndexSettings( + final String indexName, + final String dataStreamName, + boolean isTimeSeries, + final Metadata metadata, + final Instant resolvedAt, + final Settings settings, + final List combinedTemplateMappings + ) { + if (isLogsdbEnabled == false || dataStreamName == null) { + return Settings.EMPTY; + } + + final IndexMode indexMode = resolveIndexMode(settings.get(IndexSettings.MODE.getKey())); + if (indexMode != null) { + return Settings.EMPTY; + } + + if (usesLogsAtSettingsComponentTemplate(metadata, dataStreamName) && matchesLogsPattern(dataStreamName)) { + return Settings.builder().put("index.mode", IndexMode.LOGSDB.getName()).build(); + } + + return Settings.EMPTY; + } + + private static boolean matchesLogsPattern(final String name) { + return Regex.simpleMatch(LOGS_PATTERN, name); + } + + private IndexMode resolveIndexMode(final String mode) { + return mode != null ? Enum.valueOf(IndexMode.class, mode.toUpperCase(Locale.ROOT)) : null; + } + + private boolean usesLogsAtSettingsComponentTemplate(final Metadata metadata, final String name) { + final String template = MetadataIndexTemplateService.findV2Template(metadata, name, false); + if (template == null) { + return false; + } + final ComposableIndexTemplate composableIndexTemplate = metadata.templatesV2().get(template); + if (composableIndexTemplate == null) { + return false; + } + for (final String componentTemplate : composableIndexTemplate.composedOf()) { + if ("logs@settings".equals(componentTemplate)) { + return true; + } + } + return false; + } + +} diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java new file mode 100644 index 0000000000000..eeb5389644c02 --- /dev/null +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java @@ -0,0 +1,326 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.logsdb; + +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplateMetadata; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.Template; +import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.List; +import java.util.Map; + +public class LogsdbIndexModeSettingsProviderTests extends ESTestCase { + + public static final String DEFAULT_MAPPING = """ + { + "_doc": { + "properties": { + "@timestamp": { + "type": "date" + }, + "message": { + "type": "keyword" + }, + "host.name": { + "type": "keyword" + } + } + } + } + """; + + public void testLogsDbDisabled() throws IOException { + final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( + Settings.builder().put("cluster.logsdb.enabled", false).build() + ); + + final Settings additionalIndexSettings = provider.getAdditionalIndexSettings( + null, + "logs-apache-production", + false, + Metadata.EMPTY_METADATA, + Instant.now().truncatedTo(ChronoUnit.SECONDS), + Settings.EMPTY, + List.of(new CompressedXContent(DEFAULT_MAPPING)) + ); + + assertTrue(additionalIndexSettings.isEmpty()); + } + + public void testOnIndexCreation() throws IOException { + final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( + Settings.builder().put("cluster.logsdb.enabled", true).build() + ); + + final Settings additionalIndexSettings = provider.getAdditionalIndexSettings( + "logs-apache-production", + null, + false, + Metadata.EMPTY_METADATA, + Instant.now().truncatedTo(ChronoUnit.SECONDS), + Settings.EMPTY, + List.of(new CompressedXContent(DEFAULT_MAPPING)) + ); + + assertTrue(additionalIndexSettings.isEmpty()); + } + + public void testOnExplicitStandardIndex() throws IOException { + final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( + Settings.builder().put("cluster.logsdb.enabled", true).build() + ); + + final Settings additionalIndexSettings = provider.getAdditionalIndexSettings( + null, + "logs-apache-production", + false, + Metadata.EMPTY_METADATA, + Instant.now().truncatedTo(ChronoUnit.SECONDS), + Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.STANDARD.getName()).build(), + List.of(new CompressedXContent(DEFAULT_MAPPING)) + ); + + assertTrue(additionalIndexSettings.isEmpty()); + } + + public void testOnExplicitTimeSeriesIndex() throws IOException { + final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( + Settings.builder().put("cluster.logsdb.enabled", true).build() + ); + + final Settings additionalIndexSettings = provider.getAdditionalIndexSettings( + null, + "logs-apache-production", + false, + Metadata.EMPTY_METADATA, + Instant.now().truncatedTo(ChronoUnit.SECONDS), + Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.getName()).build(), + List.of(new CompressedXContent(DEFAULT_MAPPING)) + ); + + assertTrue(additionalIndexSettings.isEmpty()); + } + + public void testNonLogsDataStream() throws IOException { + final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( + Settings.builder().put("cluster.logsdb.enabled", true).build() + ); + + final Settings additionalIndexSettings = provider.getAdditionalIndexSettings( + null, + "logs", + false, + Metadata.EMPTY_METADATA, + Instant.now().truncatedTo(ChronoUnit.SECONDS), + Settings.EMPTY, + List.of(new CompressedXContent(DEFAULT_MAPPING)) + ); + + assertTrue(additionalIndexSettings.isEmpty()); + } + + public void testWithoutLogsComponentTemplate() throws IOException { + final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( + Settings.builder().put("cluster.logsdb.enabled", true).build() + ); + + final Settings additionalIndexSettings = provider.getAdditionalIndexSettings( + null, + "logs-apache-production", + false, + buildMetadata(List.of("*"), List.of()), + Instant.now().truncatedTo(ChronoUnit.SECONDS), + Settings.EMPTY, + List.of(new CompressedXContent(DEFAULT_MAPPING)) + ); + + assertTrue(additionalIndexSettings.isEmpty()); + } + + public void testWithLogsComponentTemplate() throws IOException { + final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( + Settings.builder().put("cluster.logsdb.enabled", true).build() + ); + + final Settings additionalIndexSettings = provider.getAdditionalIndexSettings( + null, + "logs-apache-production", + false, + buildMetadata(List.of("*"), List.of("logs@settings")), + Instant.now().truncatedTo(ChronoUnit.SECONDS), + Settings.EMPTY, + List.of(new CompressedXContent(DEFAULT_MAPPING)) + ); + + assertIndexMode(additionalIndexSettings, IndexMode.LOGSDB.getName()); + } + + public void testWithMultipleComponentTemplates() throws IOException { + final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( + Settings.builder().put("cluster.logsdb.enabled", true).build() + ); + + final Settings additionalIndexSettings = provider.getAdditionalIndexSettings( + null, + "logs-apache-production", + false, + buildMetadata(List.of("*"), List.of("logs@settings", "logs@custom")), + Instant.now().truncatedTo(ChronoUnit.SECONDS), + Settings.EMPTY, + List.of(new CompressedXContent(DEFAULT_MAPPING)) + ); + + assertIndexMode(additionalIndexSettings, IndexMode.LOGSDB.getName()); + } + + public void testWithCustomComponentTemplatesOnly() throws IOException { + final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( + Settings.builder().put("cluster.logsdb.enabled", true).build() + ); + + final Settings additionalIndexSettings = provider.getAdditionalIndexSettings( + null, + "logs-apache-production", + false, + buildMetadata(List.of("*"), List.of("logs@custom", "custom-component-template")), + Instant.now().truncatedTo(ChronoUnit.SECONDS), + Settings.EMPTY, + List.of(new CompressedXContent(DEFAULT_MAPPING)) + ); + + assertTrue(additionalIndexSettings.isEmpty()); + } + + public void testNonMatchingTemplateIndexPattern() throws IOException { + final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( + Settings.builder().put("cluster.logsdb.enabled", true).build() + ); + + final Settings additionalIndexSettings = provider.getAdditionalIndexSettings( + null, + "logs-apache-production", + false, + buildMetadata(List.of("standard-apache-production"), List.of("logs@settings")), + Instant.now().truncatedTo(ChronoUnit.SECONDS), + Settings.EMPTY, + List.of(new CompressedXContent(DEFAULT_MAPPING)) + ); + + assertTrue(additionalIndexSettings.isEmpty()); + } + + public void testCaseSensitivity() throws IOException { + final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( + Settings.builder().put("cluster.logsdb.enabled", true).build() + ); + + final Settings additionalIndexSettings = provider.getAdditionalIndexSettings( + null, + "LOGS-apache-production", + false, + Metadata.EMPTY_METADATA, + Instant.now().truncatedTo(ChronoUnit.SECONDS), + Settings.EMPTY, + List.of(new CompressedXContent(DEFAULT_MAPPING)) + ); + + assertTrue(additionalIndexSettings.isEmpty()); + } + + public void testMultipleHyphensInDataStreamName() throws IOException { + final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( + Settings.builder().put("cluster.logsdb.enabled", true).build() + ); + + final Settings additionalIndexSettings = provider.getAdditionalIndexSettings( + null, + "logs-apache-production-eu", + false, + Metadata.EMPTY_METADATA, + Instant.now().truncatedTo(ChronoUnit.SECONDS), + Settings.EMPTY, + List.of(new CompressedXContent(DEFAULT_MAPPING)) + ); + + assertTrue(additionalIndexSettings.isEmpty()); + } + + public void testBeforeAndAFterSettingUpdate() throws IOException { + final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( + Settings.builder().put("cluster.logsdb.enabled", false).build() + ); + + final Settings beforeSettings = provider.getAdditionalIndexSettings( + null, + "logs-apache-production", + false, + buildMetadata(List.of("*"), List.of("logs@settings")), + Instant.now().truncatedTo(ChronoUnit.SECONDS), + Settings.EMPTY, + List.of(new CompressedXContent(DEFAULT_MAPPING)) + ); + + assertTrue(beforeSettings.isEmpty()); + + provider.updateClusterIndexModeLogsdbEnabled(true); + + final Settings afterSettings = provider.getAdditionalIndexSettings( + null, + "logs-apache-production", + false, + buildMetadata(List.of("*"), List.of("logs@settings")), + Instant.now().truncatedTo(ChronoUnit.SECONDS), + Settings.EMPTY, + List.of(new CompressedXContent(DEFAULT_MAPPING)) + ); + + assertIndexMode(afterSettings, IndexMode.LOGSDB.getName()); + + provider.updateClusterIndexModeLogsdbEnabled(false); + + final Settings laterSettings = provider.getAdditionalIndexSettings( + null, + "logs-apache-production", + false, + buildMetadata(List.of("*"), List.of("logs@settings")), + Instant.now().truncatedTo(ChronoUnit.SECONDS), + Settings.EMPTY, + List.of(new CompressedXContent(DEFAULT_MAPPING)) + ); + + assertTrue(laterSettings.isEmpty()); + } + + private static Metadata buildMetadata(final List indexPatterns, final List componentTemplates) throws IOException { + final Template template = new Template(Settings.EMPTY, new CompressedXContent(DEFAULT_MAPPING), null); + final ComposableIndexTemplate composableTemplate = ComposableIndexTemplate.builder() + .indexPatterns(indexPatterns) + .template(template) + .componentTemplates(componentTemplates) + .priority(1_000L) + .version(1L) + .build(); + return Metadata.builder() + .putCustom(ComposableIndexTemplateMetadata.TYPE, new ComposableIndexTemplateMetadata(Map.of("composable", composableTemplate))) + .build(); + } + + private void assertIndexMode(final Settings settings, final String expectedIndexMode) { + assertEquals(expectedIndexMode, settings.get(IndexSettings.MODE.getKey())); + } + +} diff --git a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/LegacyStackTemplateRegistry.java b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/LegacyStackTemplateRegistry.java index 62d22c0c0a9cc..b2dc04c1178e4 100644 --- a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/LegacyStackTemplateRegistry.java +++ b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/LegacyStackTemplateRegistry.java @@ -51,12 +51,7 @@ public class LegacyStackTemplateRegistry extends IndexTemplateRegistry { private final FeatureService featureService; private volatile boolean stackTemplateEnabled; - private static final Map ADDITIONAL_TEMPLATE_VARIABLES = Map.of( - "xpack.stack.template.deprecated", - "true", - "xpack.stack.template.logsdb.index.mode", - "standard" - ); + private static final Map ADDITIONAL_TEMPLATE_VARIABLES = Map.of("xpack.stack.template.deprecated", "true"); // General mappings conventions for any data that ends up in a data stream public static final String DATA_STREAMS_MAPPINGS_COMPONENT_TEMPLATE_NAME = "data-streams-mappings"; diff --git a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackPlugin.java b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackPlugin.java index cc127883652af..71d01798323d3 100644 --- a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackPlugin.java +++ b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackPlugin.java @@ -23,7 +23,7 @@ public StackPlugin(Settings settings) { @Override public List> getSettings() { - return List.of(StackTemplateRegistry.STACK_TEMPLATES_ENABLED, StackTemplateRegistry.CLUSTER_LOGSDB_ENABLED); + return List.of(StackTemplateRegistry.STACK_TEMPLATES_ENABLED); } @Override diff --git a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java index 592842f61eee8..b45f17e434388 100644 --- a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java +++ b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java @@ -18,7 +18,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.features.FeatureService; import org.elasticsearch.features.NodeFeature; -import org.elasticsearch.index.IndexMode; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.XContentParserConfiguration; @@ -36,6 +35,8 @@ import java.util.List; import java.util.Map; +import static org.elasticsearch.xpack.cluster.settings.ClusterSettings.CLUSTER_LOGSDB_ENABLED; + public class StackTemplateRegistry extends IndexTemplateRegistry { private static final Logger logger = LogManager.getLogger(StackTemplateRegistry.class); @@ -58,15 +59,6 @@ public class StackTemplateRegistry extends IndexTemplateRegistry { Setting.Property.Dynamic ); - /** - * if index.mode "logsdb" is applied by default in logs@settings for 'logs-*-*' - */ - public static final Setting CLUSTER_LOGSDB_ENABLED = Setting.boolSetting( - "cluster.logsdb.enabled", - false, - Setting.Property.NodeScope - ); - private final ClusterService clusterService; private final FeatureService featureService; private final Map componentTemplateConfigs; @@ -167,15 +159,10 @@ private Map loadComponentTemplateConfigs(boolean logs ), new IndexTemplateConfig( LOGS_SETTINGS_COMPONENT_TEMPLATE_NAME, - "/logs@settings.json", + logsDbEnabled ? "/logs@settings-logsdb.json" : "/logs@settings.json", REGISTRY_VERSION, TEMPLATE_VERSION_VARIABLE, - Map.of( - "xpack.stack.template.deprecated", - "false", - "xpack.stack.template.logsdb.index.mode", - logsDbEnabled ? IndexMode.LOGSDB.getName() : IndexMode.STANDARD.getName() - ) + Map.of("xpack.stack.template.deprecated", "false") ), new IndexTemplateConfig( METRICS_MAPPINGS_COMPONENT_TEMPLATE_NAME, From 1b67dabadb35b4c79d67ff30ccdfce1cd5929ce8 Mon Sep 17 00:00:00 2001 From: Benjamin Trent Date: Thu, 26 Sep 2024 08:51:02 -0400 Subject: [PATCH 09/43] Fix collapse interaction with stored fields (#112761) Collapse dynamically will add values to the DocumentField values array. There are a few scenarios where this is immutable and most of these are OK. However, we get in trouble when we create an immutable set for StoredValues which collapse later tries to update. The other option for this fix was to make an array copy for `values` in every `DocumentField` ctor, this seemed very expensive and could get out of hand. So, I decided to fix this one bug instead. closes https://github.com/elastic/elasticsearch/issues/112646 --- docs/changelog/112761.yaml | 6 ++++ .../search/CollapseSearchResultsIT.java | 30 +++++++++++++++++++ .../fetch/subphase/StoredFieldsPhase.java | 4 ++- 3 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 docs/changelog/112761.yaml diff --git a/docs/changelog/112761.yaml b/docs/changelog/112761.yaml new file mode 100644 index 0000000000000..fe63f38f365a4 --- /dev/null +++ b/docs/changelog/112761.yaml @@ -0,0 +1,6 @@ +pr: 112761 +summary: Fix collapse interaction with stored fields +area: Search +type: bug +issues: + - 112646 diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/CollapseSearchResultsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/CollapseSearchResultsIT.java index 48dda7fd30068..89474a0181597 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/CollapseSearchResultsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/CollapseSearchResultsIT.java @@ -14,6 +14,7 @@ import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.search.collapse.CollapseBuilder; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.xcontent.XContentType; import java.util.Map; import java.util.Set; @@ -85,4 +86,33 @@ public void testCollapseWithFields() { } ); } + + public void testCollapseWithStoredFields() { + final String indexName = "test_collapse"; + createIndex(indexName); + final String collapseField = "collapse_field"; + assertAcked(indicesAdmin().preparePutMapping(indexName).setSource(""" + { + "dynamic": "strict", + "properties": { + "collapse_field": { "type": "keyword", "store": true }, + "ts": { "type": "date", "store": true } + } + } + """, XContentType.JSON)); + index(indexName, "id_1_0", Map.of(collapseField, "value1", "ts", 0)); + index(indexName, "id_1_1", Map.of(collapseField, "value1", "ts", 1)); + index(indexName, "id_2_0", Map.of(collapseField, "value2", "ts", 2)); + refresh(indexName); + + assertNoFailuresAndResponse( + prepareSearch(indexName).setQuery(new MatchAllQueryBuilder()) + .setFetchSource(false) + .storedFields("*") + .setCollapse(new CollapseBuilder(collapseField)), + searchResponse -> { + assertEquals(collapseField, searchResponse.getHits().getCollapseField()); + } + ); + } } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/StoredFieldsPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/StoredFieldsPhase.java index b3211f0b1e31c..17b57645d7d5f 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/StoredFieldsPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/StoredFieldsPhase.java @@ -27,6 +27,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; /** * Process stored fields loaded from a HitContext into DocumentFields @@ -42,7 +43,8 @@ List process(Map> loadedFields) { if (inputs == null) { return List.of(); } - return inputs.stream().map(ft::valueForDisplay).toList(); + // This is eventually provided to DocumentField, which needs this collection to be mutable + return inputs.stream().map(ft::valueForDisplay).collect(Collectors.toList()); } boolean hasValue(Map> loadedFields) { From 701ed61fd31aed60e9e1ae7ba5ab4d24f266f928 Mon Sep 17 00:00:00 2001 From: Dan Rubinstein Date: Thu, 26 Sep 2024 09:35:54 -0400 Subject: [PATCH 10/43] Adding inference endpoint creation validation for MistralService, GoogleAiStudioService, and HuggingFaceService (#113492) * Adding inference endpoint creation validation for MistralService, GoogleAiStudioService, and HuggingFaceService * Moving invalid model type exception to shared ServiceUtils function * Fixing naming inconsistency * Updating HuggingFaceIT ELSER tests for inference endpoint validation --- .../qa/mixed/HuggingFaceServiceMixedIT.java | 1 + .../HuggingFaceServiceUpgradeIT.java | 2 + .../inference/services/ServiceUtils.java | 7 +++ .../googleaistudio/GoogleAiStudioService.java | 40 ++++++++--------- .../huggingface/HuggingFaceService.java | 42 +++++++++--------- .../services/mistral/MistralService.java | 43 +++++++++--------- .../services/openai/OpenAiService.java | 7 +-- .../SimpleServiceIntegrationValidator.java | 20 +++++++-- .../GoogleAiStudioServiceTests.java | 39 ++++++++++++++++ .../huggingface/HuggingFaceServiceTests.java | 39 ++++++++++++++++ .../services/mistral/MistralServiceTests.java | 44 +++++++++++++++++++ .../services/openai/OpenAiServiceTests.java | 5 ++- ...impleServiceIntegrationValidatorTests.java | 1 - 13 files changed, 213 insertions(+), 77 deletions(-) diff --git a/x-pack/plugin/inference/qa/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/inference/qa/mixed/HuggingFaceServiceMixedIT.java b/x-pack/plugin/inference/qa/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/inference/qa/mixed/HuggingFaceServiceMixedIT.java index a2793f9060d8a..59d3faf6489a6 100644 --- a/x-pack/plugin/inference/qa/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/inference/qa/mixed/HuggingFaceServiceMixedIT.java +++ b/x-pack/plugin/inference/qa/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/inference/qa/mixed/HuggingFaceServiceMixedIT.java @@ -84,6 +84,7 @@ public void testElser() throws IOException { final String inferenceId = "mixed-cluster-elser"; final String upgradedClusterId = "upgraded-cluster-elser"; + elserServer.enqueue(new MockResponse().setResponseCode(200).setBody(elserResponse())); put(inferenceId, elserConfig(getUrl(elserServer)), TaskType.SPARSE_EMBEDDING); var configs = (List>) get(TaskType.SPARSE_EMBEDDING, inferenceId).get("endpoints"); diff --git a/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/HuggingFaceServiceUpgradeIT.java b/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/HuggingFaceServiceUpgradeIT.java index 36ee472cc0a13..9c9a377bbb001 100644 --- a/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/HuggingFaceServiceUpgradeIT.java +++ b/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/HuggingFaceServiceUpgradeIT.java @@ -117,6 +117,7 @@ public void testElser() throws IOException { var testTaskType = TaskType.SPARSE_EMBEDDING; if (isOldCluster()) { + elserServer.enqueue(new MockResponse().setResponseCode(200).setBody(elserResponse())); put(oldClusterId, elserConfig(getUrl(elserServer)), testTaskType); var configs = (List>) get(testTaskType, oldClusterId).get(old_cluster_endpoint_identifier); assertThat(configs, hasSize(1)); @@ -136,6 +137,7 @@ public void testElser() throws IOException { assertElser(oldClusterId); // New endpoint + elserServer.enqueue(new MockResponse().setResponseCode(200).setBody(elserResponse())); put(upgradedClusterId, elserConfig(getUrl(elserServer)), testTaskType); configs = (List>) get(upgradedClusterId).get("endpoints"); assertThat(configs, hasSize(1)); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ServiceUtils.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ServiceUtils.java index 6c4904f8918a7..6eb0331913009 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ServiceUtils.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ServiceUtils.java @@ -202,6 +202,13 @@ public static ElasticsearchStatusException unknownSettingsError(Map invalidModelType) { + throw new ElasticsearchStatusException( + Strings.format("Can't update embedding details for model with unexpected type %s", invalidModelType), + RestStatus.BAD_REQUEST + ); + } + public static String missingSettingErrorMsg(String settingName, String scope) { return Strings.format("[%s] does not contain the required setting [%s]", scope, settingName); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioService.java index 08eb67ca744a4..422fc5b0ed720 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioService.java @@ -35,6 +35,7 @@ import org.elasticsearch.xpack.inference.services.googleaistudio.completion.GoogleAiStudioCompletionModel; import org.elasticsearch.xpack.inference.services.googleaistudio.embeddings.GoogleAiStudioEmbeddingsModel; import org.elasticsearch.xpack.inference.services.googleaistudio.embeddings.GoogleAiStudioEmbeddingsServiceSettings; +import org.elasticsearch.xpack.inference.services.validation.ModelValidatorBuilder; import java.util.List; import java.util.Map; @@ -187,30 +188,29 @@ public TransportVersion getMinimalSupportedVersion() { @Override public void checkModelConfig(Model model, ActionListener listener) { - if (model instanceof GoogleAiStudioEmbeddingsModel embeddingsModel) { - ServiceUtils.getEmbeddingSize( - model, - this, - listener.delegateFailureAndWrap((l, size) -> l.onResponse(updateModelWithEmbeddingDetails(embeddingsModel, size))) - ); - } else { - listener.onResponse(model); - } + // TODO: Remove this function once all services have been updated to use the new model validators + ModelValidatorBuilder.buildModelValidator(model.getTaskType()).validate(this, model, listener); } - private GoogleAiStudioEmbeddingsModel updateModelWithEmbeddingDetails(GoogleAiStudioEmbeddingsModel model, int embeddingSize) { - var similarityFromModel = model.getServiceSettings().similarity(); - var similarityToUse = similarityFromModel == null ? SimilarityMeasure.DOT_PRODUCT : similarityFromModel; + @Override + public Model updateModelWithEmbeddingDetails(Model model, int embeddingSize) { + if (model instanceof GoogleAiStudioEmbeddingsModel embeddingsModel) { + var serviceSettings = embeddingsModel.getServiceSettings(); + var similarityFromModel = serviceSettings.similarity(); + var similarityToUse = similarityFromModel == null ? SimilarityMeasure.DOT_PRODUCT : similarityFromModel; - GoogleAiStudioEmbeddingsServiceSettings serviceSettings = new GoogleAiStudioEmbeddingsServiceSettings( - model.getServiceSettings().modelId(), - model.getServiceSettings().maxInputTokens(), - embeddingSize, - similarityToUse, - model.getServiceSettings().rateLimitSettings() - ); + var updatedServiceSettings = new GoogleAiStudioEmbeddingsServiceSettings( + serviceSettings.modelId(), + serviceSettings.maxInputTokens(), + embeddingSize, + similarityToUse, + serviceSettings.rateLimitSettings() + ); - return new GoogleAiStudioEmbeddingsModel(model, serviceSettings); + return new GoogleAiStudioEmbeddingsModel(embeddingsModel, updatedServiceSettings); + } else { + throw ServiceUtils.invalidModelTypeForUpdateModelWithEmbeddingDetails(model.getClass()); + } } @Override diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceService.java index bdfa87e77b708..6b142edca80aa 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceService.java @@ -29,6 +29,7 @@ import org.elasticsearch.xpack.inference.services.ServiceUtils; import org.elasticsearch.xpack.inference.services.huggingface.elser.HuggingFaceElserModel; import org.elasticsearch.xpack.inference.services.huggingface.embeddings.HuggingFaceEmbeddingsModel; +import org.elasticsearch.xpack.inference.services.validation.ModelValidatorBuilder; import java.util.List; import java.util.Map; @@ -67,34 +68,31 @@ protected HuggingFaceModel createModel( @Override public void checkModelConfig(Model model, ActionListener listener) { + // TODO: Remove this function once all services have been updated to use the new model validators + ModelValidatorBuilder.buildModelValidator(model.getTaskType()).validate(this, model, listener); + } + + @Override + public Model updateModelWithEmbeddingDetails(Model model, int embeddingSize) { if (model instanceof HuggingFaceEmbeddingsModel embeddingsModel) { - ServiceUtils.getEmbeddingSize( - model, - this, - listener.delegateFailureAndWrap((l, size) -> l.onResponse(updateModelWithEmbeddingDetails(embeddingsModel, size))) + var serviceSettings = embeddingsModel.getServiceSettings(); + var similarityFromModel = serviceSettings.similarity(); + var similarityToUse = similarityFromModel == null ? SimilarityMeasure.COSINE : similarityFromModel; + + var updatedServiceSettings = new HuggingFaceServiceSettings( + serviceSettings.uri(), + similarityToUse, + embeddingSize, + embeddingsModel.getTokenLimit(), + serviceSettings.rateLimitSettings() ); + + return new HuggingFaceEmbeddingsModel(embeddingsModel, updatedServiceSettings); } else { - listener.onResponse(model); + throw ServiceUtils.invalidModelTypeForUpdateModelWithEmbeddingDetails(model.getClass()); } } - private static HuggingFaceEmbeddingsModel updateModelWithEmbeddingDetails(HuggingFaceEmbeddingsModel model, int embeddingSize) { - // default to cosine similarity - var similarity = model.getServiceSettings().similarity() == null - ? SimilarityMeasure.COSINE - : model.getServiceSettings().similarity(); - - var serviceSettings = new HuggingFaceServiceSettings( - model.getServiceSettings().uri(), - similarity, - embeddingSize, - model.getTokenLimit(), - model.getServiceSettings().rateLimitSettings() - ); - - return new HuggingFaceEmbeddingsModel(model, serviceSettings); - } - @Override protected void doChunkedInfer( Model model, diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/MistralService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/MistralService.java index 1acc13f50778b..221951f7a621e 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/MistralService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/MistralService.java @@ -33,6 +33,7 @@ import org.elasticsearch.xpack.inference.services.ServiceUtils; import org.elasticsearch.xpack.inference.services.mistral.embeddings.MistralEmbeddingsModel; import org.elasticsearch.xpack.inference.services.mistral.embeddings.MistralEmbeddingsServiceSettings; +import org.elasticsearch.xpack.inference.services.validation.ModelValidatorBuilder; import java.util.List; import java.util.Map; @@ -214,32 +215,28 @@ private MistralEmbeddingsModel createModelFromPersistent( @Override public void checkModelConfig(Model model, ActionListener listener) { - if (model instanceof MistralEmbeddingsModel embeddingsModel) { - ServiceUtils.getEmbeddingSize( - model, - this, - listener.delegateFailureAndWrap((l, size) -> l.onResponse(updateEmbeddingModelConfig(embeddingsModel, size))) - ); - } else { - listener.onResponse(model); - } + // TODO: Remove this function once all services have been updated to use the new model validators + ModelValidatorBuilder.buildModelValidator(model.getTaskType()).validate(this, model, listener); } - private MistralEmbeddingsModel updateEmbeddingModelConfig(MistralEmbeddingsModel embeddingsModel, int embeddingsSize) { - var embeddingServiceSettings = embeddingsModel.getServiceSettings(); - - var similarityFromModel = embeddingsModel.getServiceSettings().similarity(); - var similarityToUse = similarityFromModel == null ? SimilarityMeasure.DOT_PRODUCT : similarityFromModel; + @Override + public Model updateModelWithEmbeddingDetails(Model model, int embeddingSize) { + if (model instanceof MistralEmbeddingsModel embeddingsModel) { + var serviceSettings = embeddingsModel.getServiceSettings(); - MistralEmbeddingsServiceSettings serviceSettings = new MistralEmbeddingsServiceSettings( - embeddingServiceSettings.modelId(), - embeddingsSize, - embeddingServiceSettings.maxInputTokens(), - similarityToUse, - embeddingServiceSettings.rateLimitSettings() - ); + var similarityFromModel = embeddingsModel.getServiceSettings().similarity(); + var similarityToUse = similarityFromModel == null ? SimilarityMeasure.DOT_PRODUCT : similarityFromModel; - return new MistralEmbeddingsModel(embeddingsModel, serviceSettings); + MistralEmbeddingsServiceSettings updatedServiceSettings = new MistralEmbeddingsServiceSettings( + serviceSettings.modelId(), + embeddingSize, + serviceSettings.maxInputTokens(), + similarityToUse, + serviceSettings.rateLimitSettings() + ); + return new MistralEmbeddingsModel(embeddingsModel, updatedServiceSettings); + } else { + throw ServiceUtils.invalidModelTypeForUpdateModelWithEmbeddingDetails(model.getClass()); + } } - } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiService.java index 7cea1ec7df46c..f9565a915124f 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiService.java @@ -12,7 +12,6 @@ import org.elasticsearch.TransportVersions; import org.elasticsearch.action.ActionListener; import org.elasticsearch.core.Nullable; -import org.elasticsearch.core.Strings; import org.elasticsearch.core.TimeValue; import org.elasticsearch.inference.ChunkedInferenceServiceResults; import org.elasticsearch.inference.ChunkingOptions; @@ -35,6 +34,7 @@ import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; import org.elasticsearch.xpack.inference.services.SenderService; import org.elasticsearch.xpack.inference.services.ServiceComponents; +import org.elasticsearch.xpack.inference.services.ServiceUtils; import org.elasticsearch.xpack.inference.services.openai.completion.OpenAiChatCompletionModel; import org.elasticsearch.xpack.inference.services.openai.embeddings.OpenAiEmbeddingsModel; import org.elasticsearch.xpack.inference.services.openai.embeddings.OpenAiEmbeddingsServiceSettings; @@ -307,10 +307,7 @@ public Model updateModelWithEmbeddingDetails(Model model, int embeddingSize) { return new OpenAiEmbeddingsModel(embeddingsModel, updatedServiceSettings); } else { - throw new ElasticsearchStatusException( - Strings.format("Can't update embedding details for model with unexpected type %s", model.getClass()), - RestStatus.BAD_REQUEST - ); + throw ServiceUtils.invalidModelTypeForUpdateModelWithEmbeddingDetails(model.getClass()); } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/validation/SimpleServiceIntegrationValidator.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/validation/SimpleServiceIntegrationValidator.java index 9fc5748746085..6233a7e0b6b29 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/validation/SimpleServiceIntegrationValidator.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/validation/SimpleServiceIntegrationValidator.java @@ -1,3 +1,4 @@ + /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one * or more contributor license agreements. Licensed under the Elastic License @@ -33,14 +34,25 @@ public void validate(InferenceService service, Model model, ActionListener { + ActionListener.wrap(r -> { if (r != null) { - delegate.onResponse(r); + listener.onResponse(r); } else { - delegate.onFailure( - new ElasticsearchStatusException("Could not make a validation call to the selected service", RestStatus.BAD_REQUEST) + listener.onFailure( + new ElasticsearchStatusException( + "Could not complete inference endpoint creation as validation call to service returned null response.", + RestStatus.BAD_REQUEST + ) ); } + }, e -> { + listener.onFailure( + new ElasticsearchStatusException( + "Could not complete inference endpoint creation as validation call to service threw an exception.", + RestStatus.BAD_REQUEST, + e + ) + ); }) ); } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioServiceTests.java index a8882bb244512..89d6a010bbc07 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioServiceTests.java @@ -914,6 +914,45 @@ public void testCheckModelConfig_DoesNotUpdateSimilarity_WhenItIsSpecifiedAsCosi } } + public void testUpdateModelWithEmbeddingDetails_InvalidModelProvided() throws IOException { + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + try (var service = new GoogleAiStudioService(senderFactory, createWithEmptySettings(threadPool))) { + var model = GoogleAiStudioCompletionModelTests.createModel(randomAlphaOfLength(10), randomAlphaOfLength(10)); + assertThrows( + ElasticsearchStatusException.class, + () -> { service.updateModelWithEmbeddingDetails(model, randomNonNegativeInt()); } + ); + } + } + + public void testUpdateModelWithEmbeddingDetails_NullSimilarityInOriginalModel() throws IOException { + testUpdateModelWithEmbeddingDetails_Successful(null); + } + + public void testUpdateModelWithEmbeddingDetails_NonNullSimilarityInOriginalModel() throws IOException { + testUpdateModelWithEmbeddingDetails_Successful(randomFrom(SimilarityMeasure.values())); + } + + private void testUpdateModelWithEmbeddingDetails_Successful(SimilarityMeasure similarityMeasure) throws IOException { + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + try (var service = new GoogleAiStudioService(senderFactory, createWithEmptySettings(threadPool))) { + var embeddingSize = randomNonNegativeInt(); + var model = GoogleAiStudioEmbeddingsModelTests.createModel( + randomAlphaOfLength(10), + randomAlphaOfLength(10), + randomAlphaOfLength(10), + randomNonNegativeInt(), + similarityMeasure + ); + + Model updatedModel = service.updateModelWithEmbeddingDetails(model, embeddingSize); + + SimilarityMeasure expectedSimilarityMeasure = similarityMeasure == null ? SimilarityMeasure.DOT_PRODUCT : similarityMeasure; + assertEquals(expectedSimilarityMeasure, updatedModel.getServiceSettings().similarity()); + assertEquals(embeddingSize, updatedModel.getServiceSettings().dimensions().intValue()); + } + } + public static Map buildExpectationCompletions(List completions) { return Map.of( ChatCompletionResults.COMPLETION, diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceServiceTests.java index f68aedd69f365..5ea9f82e5b60c 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceServiceTests.java @@ -595,6 +595,45 @@ public void testCheckModelConfig_DefaultsSimilarityToCosine() throws IOException } } + public void testUpdateModelWithEmbeddingDetails_InvalidModelProvided() throws IOException { + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + try (var service = new HuggingFaceService(senderFactory, createWithEmptySettings(threadPool))) { + var model = HuggingFaceElserModelTests.createModel(randomAlphaOfLength(10), randomAlphaOfLength(10)); + assertThrows( + ElasticsearchStatusException.class, + () -> { service.updateModelWithEmbeddingDetails(model, randomNonNegativeInt()); } + ); + } + } + + public void testUpdateModelWithEmbeddingDetails_NullSimilarityInOriginalModel() throws IOException { + testUpdateModelWithEmbeddingDetails_Successful(null); + } + + public void testUpdateModelWithEmbeddingDetails_NonNullSimilarityInOriginalModel() throws IOException { + testUpdateModelWithEmbeddingDetails_Successful(randomFrom(SimilarityMeasure.values())); + } + + private void testUpdateModelWithEmbeddingDetails_Successful(SimilarityMeasure similarityMeasure) throws IOException { + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + try (var service = new HuggingFaceService(senderFactory, createWithEmptySettings(threadPool))) { + var embeddingSize = randomNonNegativeInt(); + var model = HuggingFaceEmbeddingsModelTests.createModel( + randomAlphaOfLength(10), + randomAlphaOfLength(10), + randomNonNegativeInt(), + randomNonNegativeInt(), + similarityMeasure + ); + + Model updatedModel = service.updateModelWithEmbeddingDetails(model, embeddingSize); + + SimilarityMeasure expectedSimilarityMeasure = similarityMeasure == null ? SimilarityMeasure.COSINE : similarityMeasure; + assertEquals(expectedSimilarityMeasure, updatedModel.getServiceSettings().similarity()); + assertEquals(embeddingSize, updatedModel.getServiceSettings().dimensions().intValue()); + } + } + public void testChunkedInfer_CallsInfer_TextEmbedding_ConvertsFloatResponse() throws IOException { var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/mistral/MistralServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/mistral/MistralServiceTests.java index c833f00c4c433..9d0fd954c44f9 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/mistral/MistralServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/mistral/MistralServiceTests.java @@ -30,6 +30,7 @@ import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.core.inference.action.InferenceAction; import org.elasticsearch.xpack.core.inference.results.InferenceChunkedTextEmbeddingFloatResults; +import org.elasticsearch.xpack.inference.ModelConfigurationsTests; import org.elasticsearch.xpack.inference.external.http.HttpClientManager; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSenderTests; @@ -38,6 +39,7 @@ import org.elasticsearch.xpack.inference.services.mistral.embeddings.MistralEmbeddingModelTests; import org.elasticsearch.xpack.inference.services.mistral.embeddings.MistralEmbeddingsModel; import org.elasticsearch.xpack.inference.services.mistral.embeddings.MistralEmbeddingsServiceSettings; +import org.elasticsearch.xpack.inference.services.settings.RateLimitSettingsTests; import org.hamcrest.CoreMatchers; import org.hamcrest.MatcherAssert; import org.hamcrest.Matchers; @@ -388,6 +390,48 @@ public void testCheckModelConfig_ForEmbeddingsModel_Works() throws IOException { } } + public void testUpdateModelWithEmbeddingDetails_InvalidModelProvided() throws IOException { + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + try (var service = new MistralService(senderFactory, createWithEmptySettings(threadPool))) { + var model = new Model(ModelConfigurationsTests.createRandomInstance()); + + assertThrows( + ElasticsearchStatusException.class, + () -> { service.updateModelWithEmbeddingDetails(model, randomNonNegativeInt()); } + ); + } + } + + public void testUpdateModelWithEmbeddingDetails_NullSimilarityInOriginalModel() throws IOException { + testUpdateModelWithEmbeddingDetails_Successful(null); + } + + public void testUpdateModelWithEmbeddingDetails_NonNullSimilarityInOriginalModel() throws IOException { + testUpdateModelWithEmbeddingDetails_Successful(randomFrom(SimilarityMeasure.values())); + } + + private void testUpdateModelWithEmbeddingDetails_Successful(SimilarityMeasure similarityMeasure) throws IOException { + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + try (var service = new MistralService(senderFactory, createWithEmptySettings(threadPool))) { + var embeddingSize = randomNonNegativeInt(); + var model = MistralEmbeddingModelTests.createModel( + randomAlphaOfLength(10), + randomAlphaOfLength(10), + randomAlphaOfLength(10), + randomNonNegativeInt(), + randomNonNegativeInt(), + similarityMeasure, + RateLimitSettingsTests.createRandom() + ); + + Model updatedModel = service.updateModelWithEmbeddingDetails(model, embeddingSize); + + SimilarityMeasure expectedSimilarityMeasure = similarityMeasure == null ? SimilarityMeasure.DOT_PRODUCT : similarityMeasure; + assertEquals(expectedSimilarityMeasure, updatedModel.getServiceSettings().similarity()); + assertEquals(embeddingSize, updatedModel.getServiceSettings().dimensions().intValue()); + } + } + public void testInfer_ThrowsErrorWhenModelIsNotMistralEmbeddingsModel() throws IOException { var sender = mock(Sender.class); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/OpenAiServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/OpenAiServiceTests.java index e4a304f818328..a5e8c1d7eb26e 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/OpenAiServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/OpenAiServiceTests.java @@ -1433,7 +1433,7 @@ private void testUpdateModelWithEmbeddingDetails_Successful(SimilarityMeasure si randomAlphaOfLength(10), randomAlphaOfLength(10), randomAlphaOfLength(10), - null, + similarityMeasure, randomNonNegativeInt(), randomNonNegativeInt(), randomBoolean() @@ -1441,7 +1441,8 @@ private void testUpdateModelWithEmbeddingDetails_Successful(SimilarityMeasure si Model updatedModel = service.updateModelWithEmbeddingDetails(model, embeddingSize); - assertEquals(SimilarityMeasure.DOT_PRODUCT, updatedModel.getServiceSettings().similarity()); + SimilarityMeasure expectedSimilarityMeasure = similarityMeasure == null ? SimilarityMeasure.DOT_PRODUCT : similarityMeasure; + assertEquals(expectedSimilarityMeasure, updatedModel.getServiceSettings().similarity()); assertEquals(embeddingSize, updatedModel.getServiceSettings().dimensions().intValue()); } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/validation/SimpleServiceIntegrationValidatorTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/validation/SimpleServiceIntegrationValidatorTests.java index 23000ce431e7b..ef295e4070cc3 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/validation/SimpleServiceIntegrationValidatorTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/validation/SimpleServiceIntegrationValidatorTests.java @@ -122,7 +122,6 @@ private void verifyCallToService(boolean withQuery) { eq(InferenceAction.Request.DEFAULT_TIMEOUT), any() ); - verify(mockActionListener).delegateFailureAndWrap(any()); verifyNoMoreInteractions(mockInferenceService, mockModel, mockActionListener, mockInferenceServiceResults); } } From 1faa35176020e7d500a7ca823705c0d87c9335c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Cea=20Fontenla?= Date: Thu, 26 Sep 2024 16:03:29 +0200 Subject: [PATCH 11/43] Add CircuitBreaker to TDigest, Step 2: Add CB to array wrappers (#113105) Part of https://github.com/elastic/elasticsearch/issues/99815 ## Steps 1. Migrate TDigest classes to use a custom Array implementation. Temporarily use a simple array wrapper (https://github.com/elastic/elasticsearch/pull/112810) 2. Implement CircuitBreaking in the `MemoryTrackingTDigestArrays` class. Add `Releasable` and ensure it's always closed within TDigest (This PR) 3. Pass the CircuitBreaker as a parameter to TDigestState from wherever it's being used 4. Account remaining TDigest classes size ("SHALLOW_SIZE") Every step should be safely mergeable to main: - The first and second steps should have no impact. - The third and fourth ones will start increasing the CB count partially. ## Remarks To simplify testing the CircuitBreaker, added a helper method + `@After` to ESTestCase. Right now CBs are usually tested through MockBigArrays. E.g: https://github.com/elastic/elasticsearch/blob/f7a0196b454b17f7928728a26084000238c4efaa/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java#L1263-L1265 So I guess there was no need for this yet. But I may have missed something somewhere. Also, I'm separating this PR from the "step 3" as integrating this (CB) in the current usages may require some refactor of external code, which may be somewhat more _dangerous_ --- .../benchmark/tdigest/SortBench.java | 9 +- .../benchmark/tdigest/TDigestBench.java | 11 +- libs/tdigest/build.gradle | 2 + libs/tdigest/src/main/java/module-info.java | 2 + .../elasticsearch/tdigest/AVLGroupTree.java | 8 +- .../elasticsearch/tdigest/AVLTreeDigest.java | 41 +- .../elasticsearch/tdigest/HybridDigest.java | 7 + .../org/elasticsearch/tdigest/IntAVLTree.java | 8 +- .../elasticsearch/tdigest/MergingDigest.java | 13 +- .../elasticsearch/tdigest/SortingDigest.java | 6 + .../org/elasticsearch/tdigest/TDigest.java | 3 +- .../tdigest/arrays/TDigestByteArray.java | 4 +- .../tdigest/arrays/TDigestDoubleArray.java | 4 +- .../tdigest/arrays/TDigestIntArray.java | 4 +- .../tdigest/arrays/TDigestLongArray.java | 4 +- .../tdigest/arrays/WrapperTDigestArrays.java | 258 ----------- .../tdigest/AVLGroupTreeTests.java | 15 +- .../tdigest/AVLTreeDigestTests.java | 4 +- .../tdigest/AlternativeMergeTests.java | 9 +- .../elasticsearch/tdigest/BigCountTests.java | 4 +- .../BigCountTestsMergingDigestTests.java | 4 +- .../tdigest/BigCountTestsTreeDigestTests.java | 4 +- .../tdigest/ComparisonTests.java | 13 +- .../tdigest/IntAVLTreeTests.java | 13 +- .../elasticsearch/tdigest/MedianTests.java | 13 +- .../org/elasticsearch/tdigest/SortTests.java | 14 +- .../tdigest/TDigestTestCase.java | 109 +++++ .../elasticsearch/tdigest/TDigestTests.java | 10 +- .../metrics/EmptyTDigestState.java | 4 +- .../metrics/MemoryTrackingTDigestArrays.java | 401 ++++++++++++++++++ .../aggregations/metrics/TDigestState.java | 18 +- .../MemoryTrackingTDigestArraysTests.java | 360 ++++++++++++++++ .../metrics/TDigestStateTests.java | 235 +++++----- .../org/elasticsearch/test/ESTestCase.java | 16 + 34 files changed, 1147 insertions(+), 483 deletions(-) delete mode 100644 libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/WrapperTDigestArrays.java create mode 100644 libs/tdigest/src/test/java/org/elasticsearch/tdigest/TDigestTestCase.java create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/metrics/MemoryTrackingTDigestArrays.java create mode 100644 server/src/test/java/org/elasticsearch/search/aggregations/metrics/MemoryTrackingTDigestArraysTests.java diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/tdigest/SortBench.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/tdigest/SortBench.java index 423db48337586..4bec6a183fe94 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/tdigest/SortBench.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/tdigest/SortBench.java @@ -21,10 +21,12 @@ package org.elasticsearch.benchmark.tdigest; +import org.elasticsearch.common.breaker.NoopCircuitBreaker; +import org.elasticsearch.search.aggregations.metrics.MemoryTrackingTDigestArrays; import org.elasticsearch.tdigest.Sort; +import org.elasticsearch.tdigest.arrays.TDigestArrays; import org.elasticsearch.tdigest.arrays.TDigestDoubleArray; import org.elasticsearch.tdigest.arrays.TDigestIntArray; -import org.elasticsearch.tdigest.arrays.WrapperTDigestArrays; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; import org.openjdk.jmh.annotations.Fork; @@ -51,7 +53,8 @@ @State(Scope.Thread) public class SortBench { private final int size = 100000; - private final TDigestDoubleArray values = WrapperTDigestArrays.INSTANCE.newDoubleArray(size); + private final TDigestArrays arrays = new MemoryTrackingTDigestArrays(new NoopCircuitBreaker("default-wrapper-tdigest-arrays")); + private final TDigestDoubleArray values = arrays.newDoubleArray(size); @Param({ "0", "1", "-1" }) public int sortDirection; @@ -72,7 +75,7 @@ public void setup() { @Benchmark public void stableSort() { - TDigestIntArray order = WrapperTDigestArrays.INSTANCE.newIntArray(size); + TDigestIntArray order = arrays.newIntArray(size); for (int i = 0; i < size; i++) { order.set(i, i); } diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/tdigest/TDigestBench.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/tdigest/TDigestBench.java index 58bb5b07d22cd..36ffc34c482d7 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/tdigest/TDigestBench.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/tdigest/TDigestBench.java @@ -21,9 +21,11 @@ package org.elasticsearch.benchmark.tdigest; +import org.elasticsearch.common.breaker.NoopCircuitBreaker; +import org.elasticsearch.search.aggregations.metrics.MemoryTrackingTDigestArrays; import org.elasticsearch.tdigest.MergingDigest; import org.elasticsearch.tdigest.TDigest; -import org.elasticsearch.tdigest.arrays.WrapperTDigestArrays; +import org.elasticsearch.tdigest.arrays.TDigestArrays; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; import org.openjdk.jmh.annotations.Fork; @@ -56,24 +58,25 @@ @Threads(1) @State(Scope.Thread) public class TDigestBench { + private static final TDigestArrays arrays = new MemoryTrackingTDigestArrays(new NoopCircuitBreaker("default-wrapper-tdigest-arrays")); public enum TDigestFactory { MERGE { @Override TDigest create(double compression) { - return new MergingDigest(WrapperTDigestArrays.INSTANCE, compression, (int) (10 * compression)); + return new MergingDigest(arrays, compression, (int) (10 * compression)); } }, AVL_TREE { @Override TDigest create(double compression) { - return TDigest.createAvlTreeDigest(WrapperTDigestArrays.INSTANCE, compression); + return TDigest.createAvlTreeDigest(arrays, compression); } }, HYBRID { @Override TDigest create(double compression) { - return TDigest.createHybridDigest(WrapperTDigestArrays.INSTANCE, compression); + return TDigest.createHybridDigest(arrays, compression); } }; diff --git a/libs/tdigest/build.gradle b/libs/tdigest/build.gradle index 771df2e83d85d..df60862b27386 100644 --- a/libs/tdigest/build.gradle +++ b/libs/tdigest/build.gradle @@ -22,6 +22,8 @@ apply plugin: 'elasticsearch.build' apply plugin: 'elasticsearch.publish' dependencies { + api project(':libs:elasticsearch-core') + testImplementation(project(":test:framework")) { exclude group: 'org.elasticsearch', module: 'elasticsearch-tdigest' } diff --git a/libs/tdigest/src/main/java/module-info.java b/libs/tdigest/src/main/java/module-info.java index 8edaff3f31d8c..cc7ff1810905f 100644 --- a/libs/tdigest/src/main/java/module-info.java +++ b/libs/tdigest/src/main/java/module-info.java @@ -18,6 +18,8 @@ */ module org.elasticsearch.tdigest { + requires org.elasticsearch.base; + exports org.elasticsearch.tdigest; exports org.elasticsearch.tdigest.arrays; } diff --git a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/AVLGroupTree.java b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/AVLGroupTree.java index 8528db2128729..a1a65e1e71cde 100644 --- a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/AVLGroupTree.java +++ b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/AVLGroupTree.java @@ -21,6 +21,8 @@ package org.elasticsearch.tdigest; +import org.elasticsearch.core.Releasable; +import org.elasticsearch.core.Releasables; import org.elasticsearch.tdigest.arrays.TDigestArrays; import org.elasticsearch.tdigest.arrays.TDigestDoubleArray; import org.elasticsearch.tdigest.arrays.TDigestLongArray; @@ -31,7 +33,7 @@ /** * A tree of t-digest centroids. */ -final class AVLGroupTree extends AbstractCollection { +final class AVLGroupTree extends AbstractCollection implements Releasable { /* For insertions into the tree */ private double centroid; private long count; @@ -267,4 +269,8 @@ private void checkAggregates(int node) { } } + @Override + public void close() { + Releasables.close(centroids, counts, aggregatedCounts, tree); + } } diff --git a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/AVLTreeDigest.java b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/AVLTreeDigest.java index c28f86b9b8edc..f6b027edb1e9c 100644 --- a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/AVLTreeDigest.java +++ b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/AVLTreeDigest.java @@ -21,6 +21,7 @@ package org.elasticsearch.tdigest; +import org.elasticsearch.core.Releasables; import org.elasticsearch.tdigest.arrays.TDigestArrays; import java.util.Collection; @@ -153,26 +154,27 @@ public void compress() { } needsCompression = false; - AVLGroupTree centroids = summary; - this.summary = new AVLGroupTree(arrays); + try (AVLGroupTree centroids = summary) { + this.summary = new AVLGroupTree(arrays); - final int[] nodes = new int[centroids.size()]; - nodes[0] = centroids.first(); - for (int i = 1; i < nodes.length; ++i) { - nodes[i] = centroids.next(nodes[i - 1]); - assert nodes[i] != IntAVLTree.NIL; - } - assert centroids.next(nodes[nodes.length - 1]) == IntAVLTree.NIL; + final int[] nodes = new int[centroids.size()]; + nodes[0] = centroids.first(); + for (int i = 1; i < nodes.length; ++i) { + nodes[i] = centroids.next(nodes[i - 1]); + assert nodes[i] != IntAVLTree.NIL; + } + assert centroids.next(nodes[nodes.length - 1]) == IntAVLTree.NIL; - for (int i = centroids.size() - 1; i > 0; --i) { - final int other = gen.nextInt(i + 1); - final int tmp = nodes[other]; - nodes[other] = nodes[i]; - nodes[i] = tmp; - } + for (int i = centroids.size() - 1; i > 0; --i) { + final int other = gen.nextInt(i + 1); + final int tmp = nodes[other]; + nodes[other] = nodes[i]; + nodes[i] = tmp; + } - for (int node : nodes) { - add(centroids.mean(node), centroids.count(node)); + for (int node : nodes) { + add(centroids.mean(node), centroids.count(node)); + } } } @@ -356,4 +358,9 @@ public int byteSize() { compress(); return 64 + summary.size() * 13; } + + @Override + public void close() { + Releasables.close(summary); + } } diff --git a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/HybridDigest.java b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/HybridDigest.java index c28a99fbd6d44..8d03ce4e303a6 100644 --- a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/HybridDigest.java +++ b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/HybridDigest.java @@ -19,6 +19,7 @@ package org.elasticsearch.tdigest; +import org.elasticsearch.core.Releasables; import org.elasticsearch.tdigest.arrays.TDigestArrays; import java.util.Collection; @@ -110,6 +111,7 @@ public void reserve(long size) { } mergingDigest.reserve(size); // Release the allocated SortingDigest. + sortingDigest.close(); sortingDigest = null; } else { sortingDigest.reserve(size); @@ -196,4 +198,9 @@ public int byteSize() { } return sortingDigest.byteSize(); } + + @Override + public void close() { + Releasables.close(sortingDigest, mergingDigest); + } } diff --git a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/IntAVLTree.java b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/IntAVLTree.java index cda8aecdb2ccc..b4a82257693d8 100644 --- a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/IntAVLTree.java +++ b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/IntAVLTree.java @@ -21,6 +21,8 @@ package org.elasticsearch.tdigest; +import org.elasticsearch.core.Releasable; +import org.elasticsearch.core.Releasables; import org.elasticsearch.tdigest.arrays.TDigestArrays; import org.elasticsearch.tdigest.arrays.TDigestByteArray; import org.elasticsearch.tdigest.arrays.TDigestIntArray; @@ -33,7 +35,7 @@ * want to add data to the nodes, typically by using arrays and node * identifiers as indices. */ -abstract class IntAVLTree { +abstract class IntAVLTree implements Releasable { /** * We use 0 instead of -1 so that left(NIL) works without * condition. @@ -586,4 +588,8 @@ int size() { } + @Override + public void close() { + Releasables.close(parent, left, right, depth); + } } diff --git a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/MergingDigest.java b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/MergingDigest.java index 1649af041ee19..f2ccfc33aa2a9 100644 --- a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/MergingDigest.java +++ b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/MergingDigest.java @@ -21,6 +21,7 @@ package org.elasticsearch.tdigest; +import org.elasticsearch.core.Releasables; import org.elasticsearch.tdigest.arrays.TDigestArrays; import org.elasticsearch.tdigest.arrays.TDigestDoubleArray; import org.elasticsearch.tdigest.arrays.TDigestIntArray; @@ -66,8 +67,6 @@ * what the AVLTreeDigest uses and no dynamic allocation is required at all. */ public class MergingDigest extends AbstractTDigest { - private final TDigestArrays arrays; - private int mergeCount = 0; private final double publicCompression; @@ -138,8 +137,6 @@ public MergingDigest(TDigestArrays arrays, double compression, int bufferSize) { * @param size Size of main buffer */ public MergingDigest(TDigestArrays arrays, double compression, int bufferSize, int size) { - this.arrays = arrays; - // ensure compression >= 10 // default size = 2 * ceil(compression) // default bufferSize = 5 * size @@ -274,9 +271,6 @@ private void merge( incomingWeight.set(incomingCount, weight, 0, lastUsedCell); incomingCount += lastUsedCell; - if (incomingOrder == null) { - incomingOrder = arrays.newIntArray(incomingCount); - } Sort.stableSort(incomingOrder, incomingMean, incomingCount); totalWeight += unmergedWeight; @@ -581,4 +575,9 @@ public String toString() { + "-" + (useTwoLevelCompression ? "twoLevel" : "oneLevel"); } + + @Override + public void close() { + Releasables.close(weight, mean, tempWeight, tempMean, order); + } } diff --git a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/SortingDigest.java b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/SortingDigest.java index 94b5c667e0672..f063ca9a511c6 100644 --- a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/SortingDigest.java +++ b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/SortingDigest.java @@ -19,6 +19,7 @@ package org.elasticsearch.tdigest; +import org.elasticsearch.core.Releasables; import org.elasticsearch.tdigest.arrays.TDigestArrays; import org.elasticsearch.tdigest.arrays.TDigestDoubleArray; @@ -137,4 +138,9 @@ public void reserve(long size) { public int byteSize() { return values.size() * 8; } + + @Override + public void close() { + Releasables.close(values); + } } diff --git a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/TDigest.java b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/TDigest.java index 4e79f9e68cd02..e578a688738cb 100644 --- a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/TDigest.java +++ b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/TDigest.java @@ -21,6 +21,7 @@ package org.elasticsearch.tdigest; +import org.elasticsearch.core.Releasable; import org.elasticsearch.tdigest.arrays.TDigestArrays; import java.util.Collection; @@ -37,7 +38,7 @@ * - test coverage roughly at 90% * - easy to adapt for use with map-reduce */ -public abstract class TDigest { +public abstract class TDigest implements Releasable { protected ScaleFunction scale = ScaleFunction.K_2; double min = Double.POSITIVE_INFINITY; double max = Double.NEGATIVE_INFINITY; diff --git a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/TDigestByteArray.java b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/TDigestByteArray.java index 481dde9784008..ae8e84800b433 100644 --- a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/TDigestByteArray.java +++ b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/TDigestByteArray.java @@ -21,10 +21,12 @@ package org.elasticsearch.tdigest.arrays; +import org.elasticsearch.core.Releasable; + /** * Minimal interface for ByteArray-like classes used within TDigest. */ -public interface TDigestByteArray { +public interface TDigestByteArray extends Releasable { int size(); byte get(int index); diff --git a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/TDigestDoubleArray.java b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/TDigestDoubleArray.java index 92530db5e7dc4..1699dbd9beaf1 100644 --- a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/TDigestDoubleArray.java +++ b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/TDigestDoubleArray.java @@ -21,10 +21,12 @@ package org.elasticsearch.tdigest.arrays; +import org.elasticsearch.core.Releasable; + /** * Minimal interface for DoubleArray-like classes used within TDigest. */ -public interface TDigestDoubleArray { +public interface TDigestDoubleArray extends Releasable { int size(); double get(int index); diff --git a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/TDigestIntArray.java b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/TDigestIntArray.java index c944a4f8faf07..44e366aacd173 100644 --- a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/TDigestIntArray.java +++ b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/TDigestIntArray.java @@ -21,10 +21,12 @@ package org.elasticsearch.tdigest.arrays; +import org.elasticsearch.core.Releasable; + /** * Minimal interface for IntArray-like classes used within TDigest. */ -public interface TDigestIntArray { +public interface TDigestIntArray extends Releasable { int size(); int get(int index); diff --git a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/TDigestLongArray.java b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/TDigestLongArray.java index 7e75dd512e86d..5deea6b28b1ed 100644 --- a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/TDigestLongArray.java +++ b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/TDigestLongArray.java @@ -21,10 +21,12 @@ package org.elasticsearch.tdigest.arrays; +import org.elasticsearch.core.Releasable; + /** * Minimal interface for LongArray-like classes used within TDigest. */ -public interface TDigestLongArray { +public interface TDigestLongArray extends Releasable { int size(); long get(int index); diff --git a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/WrapperTDigestArrays.java b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/WrapperTDigestArrays.java deleted file mode 100644 index ce2dd4f8d8e1d..0000000000000 --- a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/WrapperTDigestArrays.java +++ /dev/null @@ -1,258 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - * This project is based on a modification of https://github.com/tdunning/t-digest which is licensed under the Apache 2.0 License. - */ - -package org.elasticsearch.tdigest.arrays; - -import java.util.Arrays; - -/** - * Temporal TDigestArrays with raw arrays. - * - *

- * Delete after the right implementation for BigArrays is made. - *

- */ -public class WrapperTDigestArrays implements TDigestArrays { - - public static final WrapperTDigestArrays INSTANCE = new WrapperTDigestArrays(); - - private WrapperTDigestArrays() {} - - @Override - public WrapperTDigestDoubleArray newDoubleArray(int initialCapacity) { - return new WrapperTDigestDoubleArray(initialCapacity); - } - - @Override - public WrapperTDigestIntArray newIntArray(int initialSize) { - return new WrapperTDigestIntArray(initialSize); - } - - @Override - public TDigestLongArray newLongArray(int initialSize) { - return new WrapperTDigestLongArray(initialSize); - } - - @Override - public TDigestByteArray newByteArray(int initialSize) { - return new WrapperTDigestByteArray(initialSize); - } - - public WrapperTDigestDoubleArray newDoubleArray(double[] array) { - return new WrapperTDigestDoubleArray(array); - } - - public WrapperTDigestIntArray newIntArray(int[] array) { - return new WrapperTDigestIntArray(array); - } - - public static class WrapperTDigestDoubleArray implements TDigestDoubleArray { - private double[] array; - private int size; - - public WrapperTDigestDoubleArray(int initialSize) { - this(new double[initialSize]); - } - - public WrapperTDigestDoubleArray(double[] array) { - this.array = array; - this.size = array.length; - } - - @Override - public int size() { - return size; - } - - @Override - public double get(int index) { - assert index >= 0 && index < size; - return array[index]; - } - - @Override - public void set(int index, double value) { - assert index >= 0 && index < size; - array[index] = value; - } - - @Override - public void add(double value) { - ensureCapacity(size + 1); - array[size++] = value; - } - - @Override - public void sort() { - Arrays.sort(array, 0, size); - } - - @Override - public void ensureCapacity(int requiredCapacity) { - if (requiredCapacity > array.length) { - int newSize = array.length + (array.length >> 1); - if (newSize < requiredCapacity) { - newSize = requiredCapacity; - } - double[] newArray = new double[newSize]; - System.arraycopy(array, 0, newArray, 0, size); - array = newArray; - } - } - - @Override - public void resize(int newSize) { - if (newSize > array.length) { - array = Arrays.copyOf(array, newSize); - } - if (newSize > size) { - Arrays.fill(array, size, newSize, 0); - } - size = newSize; - } - } - - public static class WrapperTDigestIntArray implements TDigestIntArray { - private int[] array; - private int size; - - public WrapperTDigestIntArray(int initialSize) { - this(new int[initialSize]); - } - - public WrapperTDigestIntArray(int[] array) { - this.array = array; - this.size = array.length; - } - - @Override - public int size() { - return size; - } - - @Override - public int get(int index) { - assert index >= 0 && index < size; - return array[index]; - } - - @Override - public void set(int index, int value) { - assert index >= 0 && index < size; - array[index] = value; - } - - @Override - public void resize(int newSize) { - if (newSize > array.length) { - array = Arrays.copyOf(array, newSize); - } - if (newSize > size) { - Arrays.fill(array, size, newSize, 0); - } - size = newSize; - } - } - - public static class WrapperTDigestLongArray implements TDigestLongArray { - private long[] array; - private int size; - - public WrapperTDigestLongArray(int initialSize) { - this(new long[initialSize]); - } - - public WrapperTDigestLongArray(long[] array) { - this.array = array; - this.size = array.length; - } - - @Override - public int size() { - return size; - } - - @Override - public long get(int index) { - assert index >= 0 && index < size; - return array[index]; - } - - @Override - public void set(int index, long value) { - assert index >= 0 && index < size; - array[index] = value; - } - - @Override - public void resize(int newSize) { - if (newSize > array.length) { - array = Arrays.copyOf(array, newSize); - } - if (newSize > size) { - Arrays.fill(array, size, newSize, 0); - } - size = newSize; - } - } - - public static class WrapperTDigestByteArray implements TDigestByteArray { - private byte[] array; - private int size; - - public WrapperTDigestByteArray(int initialSize) { - this(new byte[initialSize]); - } - - public WrapperTDigestByteArray(byte[] array) { - this.array = array; - this.size = array.length; - } - - @Override - public int size() { - return size; - } - - @Override - public byte get(int index) { - assert index >= 0 && index < size; - return array[index]; - } - - @Override - public void set(int index, byte value) { - assert index >= 0 && index < size; - array[index] = value; - } - - @Override - public void resize(int newSize) { - if (newSize > array.length) { - array = Arrays.copyOf(array, newSize); - } - if (newSize > size) { - Arrays.fill(array, size, newSize, (byte) 0); - } - size = newSize; - } - } -} diff --git a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/AVLGroupTreeTests.java b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/AVLGroupTreeTests.java index 71be849f401f4..7ac55afd87808 100644 --- a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/AVLGroupTreeTests.java +++ b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/AVLGroupTreeTests.java @@ -21,13 +21,10 @@ package org.elasticsearch.tdigest; -import org.elasticsearch.tdigest.arrays.WrapperTDigestArrays; -import org.elasticsearch.test.ESTestCase; - -public class AVLGroupTreeTests extends ESTestCase { +public class AVLGroupTreeTests extends TDigestTestCase { public void testSimpleAdds() { - AVLGroupTree x = new AVLGroupTree(WrapperTDigestArrays.INSTANCE); + AVLGroupTree x = new AVLGroupTree(arrays()); assertEquals(IntAVLTree.NIL, x.floor(34)); assertEquals(IntAVLTree.NIL, x.first()); assertEquals(IntAVLTree.NIL, x.last()); @@ -46,7 +43,7 @@ public void testSimpleAdds() { } public void testBalancing() { - AVLGroupTree x = new AVLGroupTree(WrapperTDigestArrays.INSTANCE); + AVLGroupTree x = new AVLGroupTree(arrays()); for (int i = 0; i < 101; i++) { x.add(new Centroid(i)); } @@ -60,7 +57,7 @@ public void testBalancing() { public void testFloor() { // mostly tested in other tests - AVLGroupTree x = new AVLGroupTree(WrapperTDigestArrays.INSTANCE); + AVLGroupTree x = new AVLGroupTree(arrays()); for (int i = 0; i < 101; i++) { x.add(new Centroid(i / 2)); } @@ -73,7 +70,7 @@ public void testFloor() { } public void testHeadSum() { - AVLGroupTree x = new AVLGroupTree(WrapperTDigestArrays.INSTANCE); + AVLGroupTree x = new AVLGroupTree(arrays()); for (int i = 0; i < 1000; ++i) { x.add(randomDouble(), randomIntBetween(1, 10)); } @@ -88,7 +85,7 @@ public void testHeadSum() { } public void testFloorSum() { - AVLGroupTree x = new AVLGroupTree(WrapperTDigestArrays.INSTANCE); + AVLGroupTree x = new AVLGroupTree(arrays()); int total = 0; for (int i = 0; i < 1000; ++i) { int count = randomIntBetween(1, 10); diff --git a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/AVLTreeDigestTests.java b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/AVLTreeDigestTests.java index 3cd89de4746f1..f6dde4e168291 100644 --- a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/AVLTreeDigestTests.java +++ b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/AVLTreeDigestTests.java @@ -21,13 +21,11 @@ package org.elasticsearch.tdigest; -import org.elasticsearch.tdigest.arrays.WrapperTDigestArrays; - public class AVLTreeDigestTests extends TDigestTests { protected DigestFactory factory(final double compression) { return () -> { - AVLTreeDigest digest = new AVLTreeDigest(WrapperTDigestArrays.INSTANCE, compression); + AVLTreeDigest digest = new AVLTreeDigest(arrays(), compression); digest.setRandomSeed(randomLong()); return digest; }; diff --git a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/AlternativeMergeTests.java b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/AlternativeMergeTests.java index 4b95e9c0ee695..0d095ec37fa45 100644 --- a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/AlternativeMergeTests.java +++ b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/AlternativeMergeTests.java @@ -21,15 +21,12 @@ package org.elasticsearch.tdigest; -import org.elasticsearch.tdigest.arrays.WrapperTDigestArrays; -import org.elasticsearch.test.ESTestCase; - import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Random; -public class AlternativeMergeTests extends ESTestCase { +public class AlternativeMergeTests extends TDigestTestCase { /** * Computes size using the alternative scaling limit for both an idealized merge and for * a MergingDigest. @@ -37,8 +34,8 @@ public class AlternativeMergeTests extends ESTestCase { public void testMerges() { for (int n : new int[] { 100, 1000, 10000, 100000 }) { for (double compression : new double[] { 50, 100, 200, 400 }) { - MergingDigest mergingDigest = new MergingDigest(WrapperTDigestArrays.INSTANCE, compression); - AVLTreeDigest treeDigest = new AVLTreeDigest(WrapperTDigestArrays.INSTANCE, compression); + MergingDigest mergingDigest = new MergingDigest(arrays(), compression); + AVLTreeDigest treeDigest = new AVLTreeDigest(arrays(), compression); List data = new ArrayList<>(); Random gen = random(); for (int i = 0; i < n; i++) { diff --git a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/BigCountTests.java b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/BigCountTests.java index 68b07f1096eea..7520d76172ef9 100644 --- a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/BigCountTests.java +++ b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/BigCountTests.java @@ -21,9 +21,7 @@ package org.elasticsearch.tdigest; -import org.elasticsearch.test.ESTestCase; - -public abstract class BigCountTests extends ESTestCase { +public abstract class BigCountTests extends TDigestTestCase { public void testBigMerge() { TDigest digest = createDigest(); diff --git a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/BigCountTestsMergingDigestTests.java b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/BigCountTestsMergingDigestTests.java index 25cd1af05a0ba..ab28628200cce 100644 --- a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/BigCountTestsMergingDigestTests.java +++ b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/BigCountTestsMergingDigestTests.java @@ -21,11 +21,9 @@ package org.elasticsearch.tdigest; -import org.elasticsearch.tdigest.arrays.WrapperTDigestArrays; - public class BigCountTestsMergingDigestTests extends BigCountTests { @Override public TDigest createDigest() { - return new MergingDigest(WrapperTDigestArrays.INSTANCE, 100); + return new MergingDigest(arrays(), 100); } } diff --git a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/BigCountTestsTreeDigestTests.java b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/BigCountTestsTreeDigestTests.java index a2cdf49d8f8ad..a9af82164c2ba 100644 --- a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/BigCountTestsTreeDigestTests.java +++ b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/BigCountTestsTreeDigestTests.java @@ -21,11 +21,9 @@ package org.elasticsearch.tdigest; -import org.elasticsearch.tdigest.arrays.WrapperTDigestArrays; - public class BigCountTestsTreeDigestTests extends BigCountTests { @Override public TDigest createDigest() { - return new AVLTreeDigest(WrapperTDigestArrays.INSTANCE, 100); + return new AVLTreeDigest(arrays(), 100); } } diff --git a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/ComparisonTests.java b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/ComparisonTests.java index f5df0c2f86ea1..82620459891ec 100644 --- a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/ComparisonTests.java +++ b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/ComparisonTests.java @@ -21,13 +21,10 @@ package org.elasticsearch.tdigest; -import org.elasticsearch.tdigest.arrays.WrapperTDigestArrays; -import org.elasticsearch.test.ESTestCase; - import java.util.Arrays; import java.util.function.Supplier; -public class ComparisonTests extends ESTestCase { +public class ComparisonTests extends TDigestTestCase { private static final int SAMPLE_COUNT = 1_000_000; @@ -40,10 +37,10 @@ public class ComparisonTests extends ESTestCase { private void loadData(Supplier sampleGenerator) { final int COMPRESSION = 100; - avlTreeDigest = TDigest.createAvlTreeDigest(WrapperTDigestArrays.INSTANCE, COMPRESSION); - mergingDigest = TDigest.createMergingDigest(WrapperTDigestArrays.INSTANCE, COMPRESSION); - sortingDigest = TDigest.createSortingDigest(WrapperTDigestArrays.INSTANCE); - hybridDigest = TDigest.createHybridDigest(WrapperTDigestArrays.INSTANCE, COMPRESSION); + avlTreeDigest = TDigest.createAvlTreeDigest(arrays(), COMPRESSION); + mergingDigest = TDigest.createMergingDigest(arrays(), COMPRESSION); + sortingDigest = TDigest.createSortingDigest(arrays()); + hybridDigest = TDigest.createHybridDigest(arrays(), COMPRESSION); samples = new double[SAMPLE_COUNT]; for (int i = 0; i < SAMPLE_COUNT; i++) { diff --git a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/IntAVLTreeTests.java b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/IntAVLTreeTests.java index 58c91ae6e03e6..5178701e96c2c 100644 --- a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/IntAVLTreeTests.java +++ b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/IntAVLTreeTests.java @@ -21,8 +21,7 @@ package org.elasticsearch.tdigest; -import org.elasticsearch.tdigest.arrays.WrapperTDigestArrays; -import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.tdigest.arrays.TDigestArrays; import java.util.Arrays; import java.util.Iterator; @@ -30,7 +29,7 @@ import java.util.Random; import java.util.TreeMap; -public class IntAVLTreeTests extends ESTestCase { +public class IntAVLTreeTests extends TDigestTestCase { static class IntegerBag extends IntAVLTree { @@ -38,8 +37,8 @@ static class IntegerBag extends IntAVLTree { int[] values; int[] counts; - IntegerBag() { - super(WrapperTDigestArrays.INSTANCE); + IntegerBag(TDigestArrays arrays) { + super(arrays); values = new int[capacity()]; counts = new int[capacity()]; } @@ -89,7 +88,7 @@ protected void merge(int node) { public void testDualAdd() { Random r = random(); TreeMap map = new TreeMap<>(); - IntegerBag bag = new IntegerBag(); + IntegerBag bag = new IntegerBag(arrays()); for (int i = 0; i < 100000; ++i) { final int v = r.nextInt(100000); if (map.containsKey(v)) { @@ -112,7 +111,7 @@ public void testDualAdd() { public void testDualAddRemove() { Random r = random(); TreeMap map = new TreeMap<>(); - IntegerBag bag = new IntegerBag(); + IntegerBag bag = new IntegerBag(arrays()); for (int i = 0; i < 100000; ++i) { final int v = r.nextInt(1000); if (r.nextBoolean()) { diff --git a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/MedianTests.java b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/MedianTests.java index dd455b307344e..c8acec935c040 100644 --- a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/MedianTests.java +++ b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/MedianTests.java @@ -21,14 +21,11 @@ package org.elasticsearch.tdigest; -import org.elasticsearch.tdigest.arrays.WrapperTDigestArrays; -import org.elasticsearch.test.ESTestCase; - -public class MedianTests extends ESTestCase { +public class MedianTests extends TDigestTestCase { public void testAVL() { double[] data = new double[] { 7, 15, 36, 39, 40, 41 }; - TDigest digest = new AVLTreeDigest(WrapperTDigestArrays.INSTANCE, 100); + TDigest digest = new AVLTreeDigest(arrays(), 100); for (double value : data) { digest.add(value); } @@ -39,7 +36,7 @@ public void testAVL() { public void testMergingDigest() { double[] data = new double[] { 7, 15, 36, 39, 40, 41 }; - TDigest digest = new MergingDigest(WrapperTDigestArrays.INSTANCE, 100); + TDigest digest = new MergingDigest(arrays(), 100); for (double value : data) { digest.add(value); } @@ -50,7 +47,7 @@ public void testMergingDigest() { public void testSortingDigest() { double[] data = new double[] { 7, 15, 36, 39, 40, 41 }; - TDigest digest = new SortingDigest(WrapperTDigestArrays.INSTANCE); + TDigest digest = new SortingDigest(arrays()); for (double value : data) { digest.add(value); } @@ -61,7 +58,7 @@ public void testSortingDigest() { public void testHybridDigest() { double[] data = new double[] { 7, 15, 36, 39, 40, 41 }; - TDigest digest = new HybridDigest(WrapperTDigestArrays.INSTANCE, 100); + TDigest digest = new HybridDigest(arrays(), 100); for (double value : data) { digest.add(value); } diff --git a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/SortTests.java b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/SortTests.java index 7327dfb5aac3c..425e4d1497eda 100644 --- a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/SortTests.java +++ b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/SortTests.java @@ -22,22 +22,20 @@ package org.elasticsearch.tdigest; import org.elasticsearch.tdigest.arrays.TDigestIntArray; -import org.elasticsearch.tdigest.arrays.WrapperTDigestArrays; -import org.elasticsearch.test.ESTestCase; import java.util.HashMap; import java.util.Map; import java.util.Random; -public class SortTests extends ESTestCase { +public class SortTests extends TDigestTestCase { public void testReverse() { - TDigestIntArray x = WrapperTDigestArrays.INSTANCE.newIntArray(0); + TDigestIntArray x = arrays().newIntArray(0); // don't crash with no input Sort.reverse(x, 0, x.size()); // reverse stuff! - x = WrapperTDigestArrays.INSTANCE.newIntArray(new int[] { 1, 2, 3, 4, 5 }); + x = arrays().newIntArray(new int[] { 1, 2, 3, 4, 5 }); Sort.reverse(x, 0, x.size()); for (int i = 0; i < 5; i++) { assertEquals(5 - i, x.get(i)); @@ -59,7 +57,7 @@ public void testReverse() { assertEquals(4, x.get(3)); assertEquals(1, x.get(4)); - x = WrapperTDigestArrays.INSTANCE.newIntArray(new int[] { 1, 2, 3, 4, 5, 6 }); + x = arrays().newIntArray(new int[] { 1, 2, 3, 4, 5, 6 }); Sort.reverse(x, 0, x.size()); for (int i = 0; i < 6; i++) { assertEquals(6 - i, x.get(i)); @@ -229,8 +227,8 @@ private void checkOrder(int[] order, double[] values) { } private void sort(int[] order, double[] values, int n) { - var wrappedOrder = WrapperTDigestArrays.INSTANCE.newIntArray(order); - var wrappedValues = WrapperTDigestArrays.INSTANCE.newDoubleArray(values); + var wrappedOrder = arrays().newIntArray(order); + var wrappedValues = arrays().newDoubleArray(values); Sort.stableSort(wrappedOrder, wrappedValues, n); } diff --git a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/TDigestTestCase.java b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/TDigestTestCase.java new file mode 100644 index 0000000000000..76db01d5dd0bf --- /dev/null +++ b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/TDigestTestCase.java @@ -0,0 +1,109 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * This project is based on a modification of https://github.com/tdunning/t-digest which is licensed under the Apache 2.0 License. + */ + +package org.elasticsearch.tdigest; + +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.core.Releasable; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.search.aggregations.metrics.MemoryTrackingTDigestArrays; +import org.elasticsearch.tdigest.arrays.TDigestArrays; +import org.elasticsearch.tdigest.arrays.TDigestByteArray; +import org.elasticsearch.tdigest.arrays.TDigestDoubleArray; +import org.elasticsearch.tdigest.arrays.TDigestIntArray; +import org.elasticsearch.tdigest.arrays.TDigestLongArray; +import org.elasticsearch.test.ESTestCase; +import org.junit.After; + +import java.util.Collection; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Base class for TDigest tests that require {@link TDigestArrays} instances. + *

+ * This class provides arrays that will be automatically closed after the test. + * It will also test that all memory have been freed, as the arrays use a counting CircuitBreaker. + *

+ */ +public abstract class TDigestTestCase extends ESTestCase { + private final Collection trackedArrays = ConcurrentHashMap.newKeySet(); + + /** + * Create a new TDigestArrays instance with a limited breaker. This method may be called multiple times. + * + *

+ * The arrays created by this method will be automatically released after the test. + *

+ */ + protected DelegatingTDigestArrays arrays() { + return new DelegatingTDigestArrays(); + } + + /** + * Release all arrays before {@link ESTestCase} checks for unreleased bytes. + */ + @After + public void releaseArrays() { + Releasables.close(trackedArrays); + trackedArrays.clear(); + } + + private T register(T releasable) { + trackedArrays.add(releasable); + return releasable; + } + + protected final class DelegatingTDigestArrays implements TDigestArrays { + private final MemoryTrackingTDigestArrays delegate; + + DelegatingTDigestArrays() { + this.delegate = new MemoryTrackingTDigestArrays(newLimitedBreaker(ByteSizeValue.ofMb(100))); + } + + public TDigestDoubleArray newDoubleArray(double[] data) { + return register(delegate.newDoubleArray(data)); + } + + @Override + public TDigestDoubleArray newDoubleArray(int size) { + return register(delegate.newDoubleArray(size)); + } + + public TDigestIntArray newIntArray(int[] data) { + return register(delegate.newIntArray(data)); + } + + @Override + public TDigestIntArray newIntArray(int size) { + return register(delegate.newIntArray(size)); + } + + @Override + public TDigestLongArray newLongArray(int size) { + return register(delegate.newLongArray(size)); + } + + @Override + public TDigestByteArray newByteArray(int initialSize) { + return register(delegate.newByteArray(initialSize)); + } + } +} diff --git a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/TDigestTests.java b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/TDigestTests.java index 43f1e36afb314..89a0c037dc864 100644 --- a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/TDigestTests.java +++ b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/TDigestTests.java @@ -21,10 +21,6 @@ package org.elasticsearch.tdigest; -import org.elasticsearch.tdigest.arrays.TDigestArrays; -import org.elasticsearch.tdigest.arrays.WrapperTDigestArrays; -import org.elasticsearch.test.ESTestCase; - import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -34,7 +30,7 @@ /** * Base test case for TDigests, just extend this class and implement the abstract methods. */ -public abstract class TDigestTests extends ESTestCase { +public abstract class TDigestTests extends TDigestTestCase { public interface DigestFactory { TDigest create(); @@ -544,8 +540,4 @@ public void testMonotonicity() { lastQuantile = q; } } - - protected static TDigestArrays arrays() { - return WrapperTDigestArrays.INSTANCE; - } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/EmptyTDigestState.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/EmptyTDigestState.java index 6ae9c655df3e8..56ac38a70cf07 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/EmptyTDigestState.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/EmptyTDigestState.java @@ -9,12 +9,10 @@ package org.elasticsearch.search.aggregations.metrics; -import org.elasticsearch.tdigest.arrays.WrapperTDigestArrays; - public final class EmptyTDigestState extends TDigestState { public EmptyTDigestState() { // Use the sorting implementation to minimize memory allocation. - super(WrapperTDigestArrays.INSTANCE, Type.SORTING, 1.0D); + super(MemoryTrackingTDigestArrays.INSTANCE, Type.SORTING, 1.0D); } @Override diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MemoryTrackingTDigestArrays.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MemoryTrackingTDigestArrays.java new file mode 100644 index 0000000000000..e99bfbfe534c8 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MemoryTrackingTDigestArrays.java @@ -0,0 +1,401 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.search.aggregations.metrics; + +import org.apache.lucene.util.Accountable; +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.RamUsageEstimator; +import org.elasticsearch.common.breaker.CircuitBreaker; +import org.elasticsearch.common.breaker.NoopCircuitBreaker; +import org.elasticsearch.core.Releasable; +import org.elasticsearch.tdigest.arrays.TDigestArrays; +import org.elasticsearch.tdigest.arrays.TDigestByteArray; +import org.elasticsearch.tdigest.arrays.TDigestDoubleArray; +import org.elasticsearch.tdigest.arrays.TDigestIntArray; +import org.elasticsearch.tdigest.arrays.TDigestLongArray; + +import java.util.Arrays; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * TDigestArrays with raw arrays and circuit breaking. + */ +public class MemoryTrackingTDigestArrays implements TDigestArrays { + + /** + * Default no-op CB instance of the wrapper. + * + * @deprecated This instance shouldn't be used, and will be removed after all usages are replaced. + */ + @Deprecated + public static final MemoryTrackingTDigestArrays INSTANCE = new MemoryTrackingTDigestArrays( + new NoopCircuitBreaker("default-wrapper-tdigest-arrays") + ); + + private final CircuitBreaker breaker; + + public MemoryTrackingTDigestArrays(CircuitBreaker breaker) { + this.breaker = breaker; + } + + @Override + public MemoryTrackingTDigestDoubleArray newDoubleArray(int initialSize) { + breaker.addEstimateBytesAndMaybeBreak( + MemoryTrackingTDigestDoubleArray.estimatedRamBytesUsed(initialSize), + "tdigest-new-double-array" + ); + return new MemoryTrackingTDigestDoubleArray(breaker, initialSize); + } + + @Override + public MemoryTrackingTDigestIntArray newIntArray(int initialSize) { + breaker.addEstimateBytesAndMaybeBreak(MemoryTrackingTDigestIntArray.estimatedRamBytesUsed(initialSize), "tdigest-new-int-array"); + return new MemoryTrackingTDigestIntArray(breaker, initialSize); + } + + @Override + public TDigestLongArray newLongArray(int initialSize) { + breaker.addEstimateBytesAndMaybeBreak(MemoryTrackingTDigestLongArray.estimatedRamBytesUsed(initialSize), "tdigest-new-long-array"); + return new MemoryTrackingTDigestLongArray(breaker, initialSize); + } + + @Override + public TDigestByteArray newByteArray(int initialSize) { + breaker.addEstimateBytesAndMaybeBreak(MemoryTrackingTDigestByteArray.estimatedRamBytesUsed(initialSize), "tdigest-new-byte-array"); + return new MemoryTrackingTDigestByteArray(breaker, initialSize); + } + + public MemoryTrackingTDigestDoubleArray newDoubleArray(double[] array) { + breaker.addEstimateBytesAndMaybeBreak( + MemoryTrackingTDigestDoubleArray.estimatedRamBytesUsed(array.length), + "tdigest-new-double-array" + ); + return new MemoryTrackingTDigestDoubleArray(breaker, array); + } + + public MemoryTrackingTDigestIntArray newIntArray(int[] array) { + breaker.addEstimateBytesAndMaybeBreak(MemoryTrackingTDigestIntArray.estimatedRamBytesUsed(array.length), "tdigest-new-int-array"); + return new MemoryTrackingTDigestIntArray(breaker, array); + } + + private static long estimatedArraySize(long arrayLength, long bytesPerElement) { + return RamUsageEstimator.alignObjectSize(RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + arrayLength * bytesPerElement); + } + + private abstract static class AbstractMemoryTrackingArray implements Releasable, Accountable { + protected final CircuitBreaker breaker; + private final AtomicBoolean closed = new AtomicBoolean(false); + + AbstractMemoryTrackingArray(CircuitBreaker breaker) { + this.breaker = breaker; + } + + @Override + public final void close() { + if (closed.compareAndSet(false, true)) { + breaker.addWithoutBreaking(-ramBytesUsed()); + } + } + } + + public static class MemoryTrackingTDigestDoubleArray extends AbstractMemoryTrackingArray implements TDigestDoubleArray { + static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(MemoryTrackingTDigestDoubleArray.class); + + private double[] array; + private int size; + + public MemoryTrackingTDigestDoubleArray(CircuitBreaker breaker, int initialSize) { + this(breaker, new double[initialSize]); + } + + public MemoryTrackingTDigestDoubleArray(CircuitBreaker breaker, double[] array) { + super(breaker); + this.array = array; + this.size = array.length; + } + + public static long estimatedRamBytesUsed(int size) { + return SHALLOW_SIZE + estimatedArraySize(size, Double.BYTES); + } + + @Override + public long ramBytesUsed() { + return estimatedRamBytesUsed(array.length); + } + + @Override + public int size() { + return size; + } + + @Override + public double get(int index) { + assert index >= 0 && index < size; + return array[index]; + } + + @Override + public void set(int index, double value) { + assert index >= 0 && index < size; + array[index] = value; + } + + @Override + public void add(double value) { + ensureCapacity(size + 1); + array[size++] = value; + } + + @Override + public void sort() { + Arrays.sort(array, 0, size); + } + + @Override + public void resize(int newSize) { + ensureCapacity(newSize); + + if (newSize > size) { + Arrays.fill(array, size, newSize, 0); + } + + size = newSize; + } + + @Override + public void ensureCapacity(int requiredCapacity) { + if (requiredCapacity > array.length) { + double[] oldArray = array; + // Used for used bytes assertion + long oldRamBytesUsed = ramBytesUsed(); + long oldArraySize = RamUsageEstimator.sizeOf(oldArray); + + int newSize = ArrayUtil.oversize(requiredCapacity, Double.BYTES); + long newArraySize = estimatedArraySize(newSize, Double.BYTES); + breaker.addEstimateBytesAndMaybeBreak(newArraySize, "tdigest-new-capacity-double-array"); + array = Arrays.copyOf(array, newSize); + breaker.addWithoutBreaking(-RamUsageEstimator.sizeOf(oldArray)); + + assert ramBytesUsed() - oldRamBytesUsed == newArraySize - oldArraySize + : "ramBytesUsed() should be aligned with manual array calculations"; + } + } + } + + public static class MemoryTrackingTDigestIntArray extends AbstractMemoryTrackingArray implements TDigestIntArray { + static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(MemoryTrackingTDigestIntArray.class); + + private int[] array; + private int size; + + public MemoryTrackingTDigestIntArray(CircuitBreaker breaker, int initialSize) { + this(breaker, new int[initialSize]); + } + + public MemoryTrackingTDigestIntArray(CircuitBreaker breaker, int[] array) { + super(breaker); + this.array = array; + this.size = array.length; + } + + public static long estimatedRamBytesUsed(int size) { + return SHALLOW_SIZE + estimatedArraySize(size, Integer.BYTES); + } + + @Override + public long ramBytesUsed() { + return estimatedRamBytesUsed(array.length); + } + + @Override + public int size() { + return size; + } + + @Override + public int get(int index) { + assert index >= 0 && index < size; + return array[index]; + } + + @Override + public void set(int index, int value) { + assert index >= 0 && index < size; + array[index] = value; + } + + @Override + public void resize(int newSize) { + ensureCapacity(newSize); + if (newSize > size) { + Arrays.fill(array, size, newSize, 0); + } + size = newSize; + } + + private void ensureCapacity(int requiredCapacity) { + if (requiredCapacity > array.length) { + int[] oldArray = array; + // Used for used bytes assertion + long oldRamBytesUsed = ramBytesUsed(); + long oldArraySize = RamUsageEstimator.sizeOf(oldArray); + + int newSize = ArrayUtil.oversize(requiredCapacity, Integer.BYTES); + long newArraySize = estimatedArraySize(newSize, Integer.BYTES); + breaker.addEstimateBytesAndMaybeBreak(newArraySize, "tdigest-new-capacity-int-array"); + array = Arrays.copyOf(array, newSize); + breaker.addWithoutBreaking(-RamUsageEstimator.sizeOf(oldArray)); + + assert ramBytesUsed() - oldRamBytesUsed == newArraySize - oldArraySize + : "ramBytesUsed() should be aligned with manual array calculations"; + } + } + } + + public static class MemoryTrackingTDigestLongArray extends AbstractMemoryTrackingArray implements TDigestLongArray { + static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(MemoryTrackingTDigestLongArray.class); + + private long[] array; + private int size; + + public MemoryTrackingTDigestLongArray(CircuitBreaker breaker, int initialSize) { + this(breaker, new long[initialSize]); + } + + public MemoryTrackingTDigestLongArray(CircuitBreaker breaker, long[] array) { + super(breaker); + this.array = array; + this.size = array.length; + } + + public static long estimatedRamBytesUsed(int size) { + return SHALLOW_SIZE + estimatedArraySize(size, Long.BYTES); + } + + @Override + public long ramBytesUsed() { + return estimatedRamBytesUsed(array.length); + } + + @Override + public int size() { + return size; + } + + @Override + public long get(int index) { + assert index >= 0 && index < size; + return array[index]; + } + + @Override + public void set(int index, long value) { + assert index >= 0 && index < size; + array[index] = value; + } + + @Override + public void resize(int newSize) { + ensureCapacity(newSize); + if (newSize > size) { + Arrays.fill(array, size, newSize, 0); + } + size = newSize; + } + + private void ensureCapacity(int requiredCapacity) { + if (requiredCapacity > array.length) { + long[] oldArray = array; + // Used for used bytes assertion + long oldRamBytesUsed = ramBytesUsed(); + long oldArraySize = RamUsageEstimator.sizeOf(oldArray); + + int newSize = ArrayUtil.oversize(requiredCapacity, Long.BYTES); + long newArraySize = estimatedArraySize(newSize, Long.BYTES); + breaker.addEstimateBytesAndMaybeBreak(newArraySize, "tdigest-new-capacity-long-array"); + array = Arrays.copyOf(array, newSize); + breaker.addWithoutBreaking(-RamUsageEstimator.sizeOf(oldArray)); + + assert ramBytesUsed() - oldRamBytesUsed == newArraySize - oldArraySize + : "ramBytesUsed() should be aligned with manual array calculations"; + } + } + } + + public static class MemoryTrackingTDigestByteArray extends AbstractMemoryTrackingArray implements TDigestByteArray { + static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(MemoryTrackingTDigestByteArray.class); + + private byte[] array; + private int size; + + public MemoryTrackingTDigestByteArray(CircuitBreaker breaker, int initialSize) { + this(breaker, new byte[initialSize]); + } + + public MemoryTrackingTDigestByteArray(CircuitBreaker breaker, byte[] array) { + super(breaker); + this.array = array; + this.size = array.length; + } + + public static long estimatedRamBytesUsed(int size) { + return SHALLOW_SIZE + estimatedArraySize(size, Byte.BYTES); + } + + @Override + public long ramBytesUsed() { + return estimatedRamBytesUsed(array.length); + } + + @Override + public int size() { + return size; + } + + @Override + public byte get(int index) { + assert index >= 0 && index < size; + return array[index]; + } + + @Override + public void set(int index, byte value) { + assert index >= 0 && index < size; + array[index] = value; + } + + @Override + public void resize(int newSize) { + ensureCapacity(newSize); + if (newSize > size) { + Arrays.fill(array, size, newSize, (byte) 0); + } + size = newSize; + } + + private void ensureCapacity(int requiredCapacity) { + if (requiredCapacity > array.length) { + byte[] oldArray = array; + // Used for used bytes assertion + long oldRamBytesUsed = ramBytesUsed(); + long oldArraySize = RamUsageEstimator.sizeOf(oldArray); + + int newSize = ArrayUtil.oversize(requiredCapacity, Byte.BYTES); + long newArraySize = estimatedArraySize(newSize, Byte.BYTES); + breaker.addEstimateBytesAndMaybeBreak(newArraySize, "tdigest-new-capacity-byte-array"); + array = Arrays.copyOf(array, newSize); + breaker.addWithoutBreaking(-RamUsageEstimator.sizeOf(oldArray)); + + assert ramBytesUsed() - oldRamBytesUsed == newArraySize - oldArraySize + : "ramBytesUsed() should be aligned with manual array calculations"; + } + } + } +} diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TDigestState.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TDigestState.java index 48bdb59e430a5..78ef81684a256 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TDigestState.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TDigestState.java @@ -11,10 +11,11 @@ import org.elasticsearch.TransportVersions; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.Releasable; +import org.elasticsearch.core.Releasables; import org.elasticsearch.tdigest.Centroid; import org.elasticsearch.tdigest.TDigest; import org.elasticsearch.tdigest.arrays.TDigestArrays; -import org.elasticsearch.tdigest.arrays.WrapperTDigestArrays; import java.io.IOException; import java.util.Collection; @@ -25,7 +26,7 @@ * through factory method params, providing one optimized for performance (e.g. MergingDigest or HybridDigest) by default, or optionally one * that produces highly accurate results regardless of input size but its construction over the sample population takes 2x-10x longer. */ -public class TDigestState { +public class TDigestState implements Releasable { private final double compression; @@ -54,7 +55,7 @@ static Type valueForHighAccuracy() { */ @Deprecated public static TDigestState create(double compression) { - return create(WrapperTDigestArrays.INSTANCE, compression); + return create(MemoryTrackingTDigestArrays.INSTANCE, compression); } /** @@ -81,7 +82,7 @@ public static TDigestState createOptimizedForAccuracy(TDigestArrays arrays, doub */ @Deprecated public static TDigestState create(double compression, TDigestExecutionHint executionHint) { - return create(WrapperTDigestArrays.INSTANCE, compression, executionHint); + return create(MemoryTrackingTDigestArrays.INSTANCE, compression, executionHint); } /** @@ -106,7 +107,7 @@ public static TDigestState create(TDigestArrays arrays, double compression, TDig * @return a TDigestState object */ public static TDigestState createUsingParamsFrom(TDigestState state) { - return new TDigestState(WrapperTDigestArrays.INSTANCE, state.type, state.compression); + return new TDigestState(MemoryTrackingTDigestArrays.INSTANCE, state.type, state.compression); } protected TDigestState(TDigestArrays arrays, Type type, double compression) { @@ -143,7 +144,7 @@ public static void write(TDigestState state, StreamOutput out) throws IOExceptio */ @Deprecated public static TDigestState read(StreamInput in) throws IOException { - return read(WrapperTDigestArrays.INSTANCE, in); + return read(MemoryTrackingTDigestArrays.INSTANCE, in); } public static TDigestState read(TDigestArrays arrays, StreamInput in) throws IOException { @@ -267,4 +268,9 @@ public final double getMin() { public final double getMax() { return tdigest.getMax(); } + + @Override + public void close() { + Releasables.close(tdigest); + } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MemoryTrackingTDigestArraysTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MemoryTrackingTDigestArraysTests.java new file mode 100644 index 0000000000000..e57f39becef7c --- /dev/null +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MemoryTrackingTDigestArraysTests.java @@ -0,0 +1,360 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.search.aggregations.metrics; + +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.tdigest.arrays.TDigestArrays; +import org.elasticsearch.tdigest.arrays.TDigestByteArray; +import org.elasticsearch.tdigest.arrays.TDigestDoubleArray; +import org.elasticsearch.tdigest.arrays.TDigestIntArray; +import org.elasticsearch.tdigest.arrays.TDigestLongArray; +import org.elasticsearch.test.ESTestCase; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; + +public class MemoryTrackingTDigestArraysTests extends ESTestCase { + // Int arrays + + public void testIntEmpty() { + try (TDigestIntArray array = intArray(0)) { + assertThat(array.size(), equalTo(0)); + } + } + + public void testIntGetAndSet() { + int initialSize = randomIntBetween(10, 1000); + try (TDigestIntArray array = intArray(initialSize)) { + assertThat(array.size(), equalTo(initialSize)); + + int value = randomInt(); + for (int i = 9; i < initialSize; i++) { + array.set(i, value); + } + + for (int i = 0; i < initialSize; i++) { + if (i < 9) { + assertThat(array.get(i), equalTo(0)); + } else { + assertThat(array.get(i), equalTo(value)); + } + } + } + } + + public void testIntResize() { + int initialSize = randomIntBetween(10, 1000); + try (TDigestIntArray array = intArray(initialSize)) { + assertThat(array.size(), equalTo(initialSize)); + + // Fill with a non-zero value + int value = randomBoolean() ? randomIntBetween(Integer.MIN_VALUE, -1) : randomIntBetween(1, Integer.MAX_VALUE); + for (int i = 0; i < initialSize; i++) { + array.set(i, value); + } + + // Resize to a size-1 + array.resize(initialSize - 1); + assertThat(array.size(), equalTo(initialSize - 1)); + + for (int i = 0; i < initialSize - 1; i++) { + assertThat(array.get(i), equalTo(value)); + } + + // Resize to the original size + 1 + array.resize(initialSize + 1); + assertThat(array.size(), equalTo(initialSize + 1)); + + // Ensure all new elements are 0 + for (int i = 0; i < initialSize - 1; i++) { + if (i < initialSize) { + assertThat(array.get(i), equalTo(value)); + } else { + assertThat(array.get(i), equalTo(0)); + } + } + } + } + + public void testIntBulkSet() { + int initialSize = randomIntBetween(10, 1000); + int sourceArraySize = randomIntBetween(0, initialSize); + + try (TDigestIntArray array = intArray(initialSize); TDigestIntArray source = intArray(sourceArraySize)) { + assertThat(array.size(), equalTo(initialSize)); + assertThat(source.size(), equalTo(sourceArraySize)); + + int value = randomInt(); + for (int i = 0; i < sourceArraySize; i++) { + source.set(i, value); + } + + int initialOffset = randomIntBetween(0, initialSize - sourceArraySize); + int sourceOffset = randomIntBetween(0, sourceArraySize - 1); + int elementsToCopy = randomIntBetween(1, sourceArraySize - sourceOffset); + + array.set(initialOffset, source, sourceOffset, elementsToCopy); + + for (int i = 0; i < initialSize; i++) { + if (i < initialOffset || i >= initialOffset + elementsToCopy) { + assertThat(array.get(i), equalTo(0)); + } else { + assertThat(array.get(i), equalTo(value)); + } + } + } + } + + // Long arrays + + public void testLongEmpty() { + try (TDigestIntArray array = intArray(0)) { + assertThat(array.size(), equalTo(0)); + } + } + + public void testLongGetAndSet() { + int initialSize = randomIntBetween(10, 1000); + try (TDigestLongArray array = longArray(initialSize)) { + assertThat(array.size(), equalTo(initialSize)); + + long value = randomLong(); + for (int i = 9; i < initialSize; i++) { + array.set(i, value); + } + + for (int i = 0; i < initialSize; i++) { + if (i < 9) { + assertThat(array.get(i), equalTo(0L)); + } else { + assertThat(array.get(i), equalTo(value)); + } + } + } + } + + public void testLongResize() { + int initialSize = randomIntBetween(10, 1000); + try (TDigestLongArray array = longArray(initialSize)) { + assertThat(array.size(), equalTo(initialSize)); + + // Fill with a non-zero value + long value = randomBoolean() ? randomLongBetween(Long.MIN_VALUE, -1) : randomLongBetween(1, Long.MAX_VALUE); + for (int i = 0; i < initialSize; i++) { + array.set(i, value); + } + + // Resize to a size-1 + array.resize(initialSize - 1); + assertThat(array.size(), equalTo(initialSize - 1)); + + for (int i = 0; i < initialSize - 1; i++) { + assertThat(array.get(i), equalTo(value)); + } + + // Resize to the original size + 1 + array.resize(initialSize + 1); + assertThat(array.size(), equalTo(initialSize + 1)); + + // Ensure all new elements are 0 + for (int i = 0; i < initialSize - 1; i++) { + if (i < initialSize) { + assertThat(array.get(i), equalTo(value)); + } else { + assertThat(array.get(i), equalTo(0L)); + } + } + } + } + + // Byte arrays + + public void testByteEmpty() { + try (TDigestByteArray array = byteArray(0)) { + assertThat(array.size(), equalTo(0)); + } + } + + public void testByteGetAndSet() { + int initialSize = randomIntBetween(10, 1000); + try (TDigestByteArray array = byteArray(initialSize)) { + assertThat(array.size(), equalTo(initialSize)); + + byte value = randomByte(); + for (int i = 9; i < initialSize; i++) { + array.set(i, value); + } + + for (int i = 0; i < initialSize; i++) { + if (i < 9) { + assertThat(array.get(i), equalTo((byte) 0)); + } else { + assertThat(array.get(i), equalTo(value)); + } + } + } + } + + public void testByteResize() { + int initialSize = randomIntBetween(10, 1000); + try (TDigestByteArray array = byteArray(initialSize)) { + assertThat(array.size(), equalTo(initialSize)); + + // Fill with a non-zero value + byte value = randomBoolean() ? randomByteBetween(Byte.MIN_VALUE, (byte) -1) : randomByteBetween((byte) 1, Byte.MAX_VALUE); + for (int i = 0; i < initialSize; i++) { + array.set(i, value); + } + + // Resize to a size-1 + array.resize(initialSize - 1); + assertThat(array.size(), equalTo(initialSize - 1)); + + for (int i = 0; i < initialSize - 1; i++) { + assertThat(array.get(i), equalTo(value)); + } + + // Resize to the original size + 1 + array.resize(initialSize + 1); + assertThat(array.size(), equalTo(initialSize + 1)); + + // Ensure all new elements are 0 + for (int i = 0; i < initialSize - 1; i++) { + if (i < initialSize) { + assertThat(array.get(i), equalTo(value)); + } else { + assertThat(array.get(i), equalTo((byte) 0)); + } + } + } + } + + // Double arrays + + public void testDoubleEmpty() { + try (TDigestDoubleArray array = doubleArray(0)) { + assertThat(array.size(), equalTo(0)); + } + } + + public void testDoubleGetAndSet() { + int initialSize = randomIntBetween(10, 1000); + try (TDigestDoubleArray array = doubleArray(initialSize)) { + assertThat(array.size(), equalTo(initialSize)); + + double value = randomDoubleBetween(-Double.MAX_VALUE, Double.MAX_VALUE, true); + for (int i = 9; i < initialSize; i++) { + array.set(i, value); + } + + for (int i = 0; i < initialSize; i++) { + if (i < 9) { + assertThat(array.get(i), equalTo(0.0)); + } else { + assertThat(array.get(i), equalTo(value)); + } + } + } + } + + public void testDoubleAdd() { + int initialSize = randomIntBetween(10, 1000); + try (TDigestDoubleArray array = doubleArray(initialSize)) { + assertThat(array.size(), equalTo(initialSize)); + + int newValueCount = randomIntBetween(1, 100); + if (randomBoolean()) { + array.ensureCapacity(initialSize + newValueCount); + } + double value = randomDoubleBetween(-Double.MAX_VALUE, Double.MAX_VALUE, true); + for (int i = 0; i < newValueCount; i++) { + array.add(value); + } + + for (int i = 0; i < newValueCount; i++) { + if (i < initialSize) { + assertThat(array.get(i), equalTo(0.0)); + } else { + assertThat(array.get(i), equalTo(value)); + } + } + } + } + + public void testDoubleBulkSet() { + int initialSize = randomIntBetween(10, 1000); + int sourceArraySize = randomIntBetween(0, initialSize); + + try (TDigestDoubleArray array = doubleArray(initialSize); TDigestDoubleArray source = doubleArray(sourceArraySize)) { + assertThat(array.size(), equalTo(initialSize)); + assertThat(source.size(), equalTo(sourceArraySize)); + + double value = randomDoubleBetween(-Double.MAX_VALUE, Double.MAX_VALUE, true); + for (int i = 0; i < sourceArraySize; i++) { + source.set(i, value); + } + + int initialOffset = randomIntBetween(0, initialSize - sourceArraySize); + int sourceOffset = randomIntBetween(0, sourceArraySize - 1); + int elementsToCopy = randomIntBetween(1, sourceArraySize - sourceOffset); + + array.set(initialOffset, source, sourceOffset, elementsToCopy); + + for (int i = 0; i < initialSize; i++) { + if (i < initialOffset || i >= initialOffset + elementsToCopy) { + assertThat(array.get(i), equalTo(0.0)); + } else { + assertThat(array.get(i), equalTo(value)); + } + } + } + } + + public void testDoubleSort() { + try (TDigestDoubleArray array = doubleArray(0)) { + int elementsToAdd = randomIntBetween(0, 100); + array.ensureCapacity(elementsToAdd); + for (int i = 0; i < elementsToAdd; i++) { + array.add(randomDoubleBetween(-Double.MAX_VALUE, Double.MAX_VALUE, true)); + } + + array.sort(); + + double previous = -Double.MAX_VALUE; + for (int i = 0; i < array.size(); i++) { + double current = array.get(i); + assertThat(current, greaterThanOrEqualTo(previous)); + previous = current; + } + } + } + + // Helpers + + private TDigestIntArray intArray(int initialSize) { + return arrays().newIntArray(initialSize); + } + + private TDigestLongArray longArray(int initialSize) { + return arrays().newLongArray(initialSize); + } + + private TDigestByteArray byteArray(int initialSize) { + return arrays().newByteArray(initialSize); + } + + private TDigestDoubleArray doubleArray(int initialSize) { + return arrays().newDoubleArray(initialSize); + } + + private TDigestArrays arrays() { + return new MemoryTrackingTDigestArrays(newLimitedBreaker(ByteSizeValue.ofMb(100))); + } +} diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestStateTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestStateTests.java index e7799a133b5af..56d3d674b28ca 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestStateTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestStateTests.java @@ -16,8 +16,9 @@ import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.core.Releasables; import org.elasticsearch.tdigest.arrays.TDigestArrays; -import org.elasticsearch.tdigest.arrays.WrapperTDigestArrays; import org.elasticsearch.test.ESTestCase; import org.hamcrest.Matchers; @@ -33,140 +34,150 @@ public class TDigestStateTests extends ESTestCase { public void testMoreThan4BValues() { // Regression test for #19528 // See https://github.com/tdunning/t-digest/pull/70/files#diff-4487072cee29b939694825647928f742R439 - TDigestState digest = TDigestState.create(arrays(), 100); - for (int i = 0; i < 1000; ++i) { - digest.add(randomDouble()); - } - final int count = 1 << 29; - for (int i = 0; i < 10; ++i) { - digest.add(randomDouble(), count); - } - assertEquals(1000 + 10L * (1 << 29), digest.size()); - assertTrue(digest.size() > 2L * Integer.MAX_VALUE); - final double[] quantiles = new double[] { 0, 0.1, 0.5, 0.9, 1, randomDouble() }; - Arrays.sort(quantiles); - double prev = Double.NEGATIVE_INFINITY; - for (double q : quantiles) { - final double v = digest.quantile(q); - logger.trace("q=" + q + ", v=" + v); - assertThat(v, Matchers.either(Matchers.closeTo(prev, 0.0000001D)).or(Matchers.greaterThan(prev))); - assertTrue("Unexpectedly low value: " + v, v >= 0.0); - assertTrue("Unexpectedly high value: " + v, v <= 1.0); - prev = v; + try (TDigestState digest = TDigestState.create(arrays(), 100)) { + for (int i = 0; i < 1000; ++i) { + digest.add(randomDouble()); + } + final int count = 1 << 29; + for (int i = 0; i < 10; ++i) { + digest.add(randomDouble(), count); + } + assertEquals(1000 + 10L * (1 << 29), digest.size()); + assertTrue(digest.size() > 2L * Integer.MAX_VALUE); + final double[] quantiles = new double[] { 0, 0.1, 0.5, 0.9, 1, randomDouble() }; + Arrays.sort(quantiles); + double prev = Double.NEGATIVE_INFINITY; + for (double q : quantiles) { + final double v = digest.quantile(q); + logger.trace("q=" + q + ", v=" + v); + assertThat(v, Matchers.either(Matchers.closeTo(prev, 0.0000001D)).or(Matchers.greaterThan(prev))); + assertTrue("Unexpectedly low value: " + v, v >= 0.0); + assertTrue("Unexpectedly high value: " + v, v <= 1.0); + prev = v; + } } } public void testEqualsHashCode() { - final TDigestState empty1 = new EmptyTDigestState(); - final TDigestState empty2 = new EmptyTDigestState(); - final TDigestState a = TDigestState.create(arrays(), 200); - final TDigestState b = TDigestState.create(arrays(), 100); - final TDigestState c = TDigestState.create(arrays(), 100); + try ( + TDigestState empty1 = new EmptyTDigestState(); + TDigestState empty2 = new EmptyTDigestState(); + TDigestState a = TDigestState.create(arrays(), 200); + TDigestState b = TDigestState.create(arrays(), 100); + TDigestState c = TDigestState.create(arrays(), 100); + ) { - assertEquals(empty1, empty2); - assertEquals(empty1.hashCode(), empty2.hashCode()); + assertEquals(empty1, empty2); + assertEquals(empty1.hashCode(), empty2.hashCode()); - assertNotEquals(a, b); - assertNotEquals(a.hashCode(), b.hashCode()); + assertNotEquals(a, b); + assertNotEquals(a.hashCode(), b.hashCode()); - assertNotEquals(a, c); - assertNotEquals(a.hashCode(), c.hashCode()); + assertNotEquals(a, c); + assertNotEquals(a.hashCode(), c.hashCode()); - assertEquals(b, c); - assertEquals(b.hashCode(), c.hashCode()); + assertEquals(b, c); + assertEquals(b.hashCode(), c.hashCode()); - for (int i = 0; i < 100; i++) { - double value = randomDouble(); - a.add(value); - b.add(value); - c.add(value); - } + for (int i = 0; i < 100; i++) { + double value = randomDouble(); + a.add(value); + b.add(value); + c.add(value); + } - assertNotEquals(a, b); - assertNotEquals(a.hashCode(), b.hashCode()); + assertNotEquals(a, b); + assertNotEquals(a.hashCode(), b.hashCode()); - assertNotEquals(a, c); - assertNotEquals(a.hashCode(), c.hashCode()); + assertNotEquals(a, c); + assertNotEquals(a.hashCode(), c.hashCode()); - assertEquals(b, c); - assertEquals(b.hashCode(), c.hashCode()); + assertEquals(b, c); + assertEquals(b.hashCode(), c.hashCode()); - b.add(randomDouble()); - c.add(randomDouble()); + b.add(randomDouble()); + c.add(randomDouble()); - assertNotEquals(b, c); - assertNotEquals(b.hashCode(), c.hashCode()); + assertNotEquals(b, c); + assertNotEquals(b.hashCode(), c.hashCode()); + } } public void testHash() { final HashMap map = new HashMap<>(); final Set set = new HashSet<>(); - final TDigestState empty1 = new EmptyTDigestState(); - final TDigestState empty2 = new EmptyTDigestState(); - final TDigestState a = TDigestState.create(arrays(), 200); - final TDigestState b = TDigestState.create(arrays(), 100); - final TDigestState c = TDigestState.create(arrays(), 100); - - a.add(randomDouble()); - b.add(randomDouble()); - c.add(randomDouble()); - expectThrows(UnsupportedOperationException.class, () -> empty1.add(randomDouble())); - expectThrows(UnsupportedOperationException.class, () -> empty2.add(a)); - - map.put("empty1", empty1); - map.put("empty2", empty2); - map.put("a", a); - map.put("b", b); - map.put("c", c); - set.add(empty1); - set.add(empty2); - set.add(a); - set.add(b); - set.add(c); - - assertEquals(5, map.size()); - assertEquals(4, set.size()); - - assertEquals(empty1, map.get("empty1")); - assertEquals(empty2, map.get("empty2")); - assertEquals(a, map.get("a")); - assertEquals(b, map.get("b")); - assertEquals(c, map.get("c")); - - assertTrue(set.stream().anyMatch(digest -> digest.equals(a))); - assertTrue(set.stream().anyMatch(digest -> digest.equals(b))); - assertTrue(set.stream().anyMatch(digest -> digest.equals(c))); - assertTrue(set.stream().anyMatch(digest -> digest.equals(empty1))); - assertTrue(set.stream().anyMatch(digest -> digest.equals(empty2))); + try ( + TDigestState empty1 = new EmptyTDigestState(); + TDigestState empty2 = new EmptyTDigestState(); + TDigestState a = TDigestState.create(arrays(), 200); + TDigestState b = TDigestState.create(arrays(), 100); + TDigestState c = TDigestState.create(arrays(), 100); + ) { + + a.add(randomDouble()); + b.add(randomDouble()); + c.add(randomDouble()); + expectThrows(UnsupportedOperationException.class, () -> empty1.add(randomDouble())); + expectThrows(UnsupportedOperationException.class, () -> empty2.add(a)); + + map.put("empty1", empty1); + map.put("empty2", empty2); + map.put("a", a); + map.put("b", b); + map.put("c", c); + set.add(empty1); + set.add(empty2); + set.add(a); + set.add(b); + set.add(c); + + assertEquals(5, map.size()); + assertEquals(4, set.size()); + + assertEquals(empty1, map.get("empty1")); + assertEquals(empty2, map.get("empty2")); + assertEquals(a, map.get("a")); + assertEquals(b, map.get("b")); + assertEquals(c, map.get("c")); + + assertTrue(set.stream().anyMatch(digest -> digest.equals(a))); + assertTrue(set.stream().anyMatch(digest -> digest.equals(b))); + assertTrue(set.stream().anyMatch(digest -> digest.equals(c))); + assertTrue(set.stream().anyMatch(digest -> digest.equals(empty1))); + assertTrue(set.stream().anyMatch(digest -> digest.equals(empty2))); + } } public void testFactoryMethods() { - TDigestState fast = TDigestState.create(arrays(), 100); - TDigestState anotherFast = TDigestState.create(arrays(), 100); - TDigestState accurate = TDigestState.createOptimizedForAccuracy(arrays(), 100); - TDigestState anotherAccurate = TDigestState.createUsingParamsFrom(accurate); - - for (int i = 0; i < 100; i++) { - fast.add(i); - anotherFast.add(i); - accurate.add(i); - anotherAccurate.add(i); - } + try ( + TDigestState fast = TDigestState.create(arrays(), 100); + TDigestState anotherFast = TDigestState.create(arrays(), 100); + TDigestState accurate = TDigestState.createOptimizedForAccuracy(arrays(), 100); + TDigestState anotherAccurate = TDigestState.createUsingParamsFrom(accurate); + ) { - for (double p : new double[] { 0.1, 1, 10, 25, 50, 75, 90, 99, 99.9 }) { - double q = p / 100; - assertEquals(fast.quantile(q), accurate.quantile(q), 0.5); - assertEquals(fast.quantile(q), anotherFast.quantile(q), 1e-5); - assertEquals(accurate.quantile(q), anotherAccurate.quantile(q), 1e-5); + for (int i = 0; i < 100; i++) { + fast.add(i); + anotherFast.add(i); + accurate.add(i); + anotherAccurate.add(i); + } + + for (double p : new double[] { 0.1, 1, 10, 25, 50, 75, 90, 99, 99.9 }) { + double q = p / 100; + assertEquals(fast.quantile(q), accurate.quantile(q), 0.5); + assertEquals(fast.quantile(q), anotherFast.quantile(q), 1e-5); + assertEquals(accurate.quantile(q), anotherAccurate.quantile(q), 1e-5); + } + + assertEquals(fast, anotherFast); + assertEquals(accurate, anotherAccurate); + assertNotEquals(fast, accurate); + assertNotEquals(anotherFast, anotherAccurate); } - - assertEquals(fast, anotherFast); - assertEquals(accurate, anotherAccurate); - assertNotEquals(fast, accurate); - assertNotEquals(anotherFast, anotherAccurate); } - private static TDigestState writeToAndReadFrom(TDigestState state, TransportVersion version) throws IOException { + private TDigestState writeToAndReadFrom(TDigestState state, TransportVersion version) throws IOException { BytesRef serializedAggs = serialize(state, version); try ( StreamInput in = new NamedWriteableAwareStreamInput( @@ -203,9 +214,11 @@ public void testSerialization() throws IOException { TDigestState serializedBackwardsCompatible = writeToAndReadFrom(state, TransportVersions.V_8_8_1); assertNotEquals(serializedBackwardsCompatible, state); assertEquals(serializedBackwardsCompatible, backwardsCompatible); + + Releasables.close(state, backwardsCompatible, serialized, serializedBackwardsCompatible); } - private static TDigestArrays arrays() { - return WrapperTDigestArrays.INSTANCE; + private TDigestArrays arrays() { + return new MemoryTrackingTDigestArrays(newLimitedBreaker(ByteSizeValue.ofMb(100))); } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java index 00cfedb257187..e6fc32a8ebe1b 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java @@ -57,6 +57,7 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.CheckedSupplier; import org.elasticsearch.common.UUIDs; +import org.elasticsearch.common.breaker.CircuitBreaker; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.bytes.CompositeBytesReference; @@ -577,6 +578,21 @@ public final void before() { } } + private final List breakers = Collections.synchronizedList(new ArrayList<>()); + + protected final CircuitBreaker newLimitedBreaker(ByteSizeValue max) { + CircuitBreaker breaker = new MockBigArrays.LimitedBreaker("", max); + breakers.add(breaker); + return breaker; + } + + @After + public final void allBreakersMemoryReleased() { + for (CircuitBreaker breaker : breakers) { + assertThat(breaker.getUsed(), equalTo(0L)); + } + } + /** * Whether or not we check after each test whether it has left warnings behind. That happens if any deprecated feature or syntax * was used by the test and the test didn't assert on it using {@link #assertWarnings(String...)}. From 5e019998ef0e57a95ba73d114123e46ad52af9bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Istv=C3=A1n=20Zolt=C3=A1n=20Szab=C3=B3?= Date: Thu, 26 Sep 2024 16:09:28 +0200 Subject: [PATCH 12/43] [DOCS] Improves semantic text documentation. (#113606) --- .../inference/delete-inference.asciidoc | 7 ++-- .../mapping/types/semantic-text.asciidoc | 39 ++++++++++++++++--- 2 files changed, 37 insertions(+), 9 deletions(-) diff --git a/docs/reference/inference/delete-inference.asciidoc b/docs/reference/inference/delete-inference.asciidoc index 4df72ba672092..bee39bf9b9851 100644 --- a/docs/reference/inference/delete-inference.asciidoc +++ b/docs/reference/inference/delete-inference.asciidoc @@ -49,13 +49,12 @@ The type of {infer} task that the model performs. `dry_run`:: (Optional, Boolean) -When `true`, checks the {infer} processors that reference the endpoint and -returns them in a list, but does not delete the endpoint. Defaults to `false`. +When `true`, checks the `semantic_text` fields and {infer} processors that reference the endpoint and returns them in a list, but does not delete the endpoint. +Defaults to `false`. `force`:: (Optional, Boolean) -Deletes the endpoint regardless if it's used in an {infer} pipeline or in a -`semantic_text` field. +Deletes the endpoint regardless if it's used in a `semantic_text` field or in an {infer} pipeline. [discrete] diff --git a/docs/reference/mapping/types/semantic-text.asciidoc b/docs/reference/mapping/types/semantic-text.asciidoc index a006f288dc66d..d0fdf0145aa58 100644 --- a/docs/reference/mapping/types/semantic-text.asciidoc +++ b/docs/reference/mapping/types/semantic-text.asciidoc @@ -14,9 +14,8 @@ The `semantic_text` field type specifies an inference endpoint identifier that w You can create the inference endpoint by using the <>. This field type and the <> type make it simpler to perform semantic search on your data. -Using `semantic_text`, you won't need to specify how to generate embeddings for -your data, or how to index it. The inference endpoint automatically determines -the embedding generation, indexing, and query to use. +Using `semantic_text`, you won't need to specify how to generate embeddings for your data, or how to index it. +The {infer} endpoint automatically determines the embedding generation, indexing, and query to use. [source,console] ------------------------------------------------------------ @@ -32,7 +31,29 @@ PUT my-index-000001 } } ------------------------------------------------------------ -// TEST[skip:TBD] +// TEST[skip:Requires inference endpoint] + + +The recommended way to use semantic_text is by having dedicated {infer} endpoints for ingestion and search. +This ensures that search speed remains unaffected by ingestion workloads, and vice versa. +After creating dedicated {infer} endpoints for both, you can reference them using the `inference_id` and `search_inference_id` parameters when setting up the index mapping for an index that uses the `semantic_text` field. + +[source,console] +------------------------------------------------------------ +PUT my-index-000002 +{ + "mappings": { + "properties": { + "inference_field": { + "type": "semantic_text", + "inference_id": "my-elser-endpoint-for-ingest", + "search_inference_id": "my-elser-endpoint-for-search" + } + } + } +} +------------------------------------------------------------ +// TEST[skip:Requires inference endpoint] [discrete] @@ -41,9 +62,15 @@ PUT my-index-000001 `inference_id`:: (Required, string) -Inference endpoint that will be used to generate the embeddings for the field. +{infer-cap} endpoint that will be used to generate the embeddings for the field. Use the <> to create the endpoint. +If `search_inference_id` is specified, the {infer} endpoint defined by `inference_id` will only be used at index time. +`search_inference_id`:: +(Optional, string) +{infer-cap} endpoint that will be used to generate embeddings at query time. +Use the <> to create the endpoint. +If not specified, the {infer} endpoint defined by `inference_id` will be used at both index and query time. [discrete] [[infer-endpoint-validation]] @@ -55,6 +82,7 @@ When the first document is indexed, the `inference_id` will be used to generate WARNING: Removing an {infer} endpoint will cause ingestion of documents and semantic queries to fail on indices that define `semantic_text` fields with that {infer} endpoint as their `inference_id`. Trying to <> that is used on a `semantic_text` field will result in an error. + [discrete] [[auto-text-chunking]] ==== Automatic text chunking @@ -183,6 +211,7 @@ PUT test-index/_bulk Notice that both the `semantic_text` field and the source field are updated in the bulk request. + [discrete] [[limitations]] ==== Limitations From 6e73c1423b6c940fab9867346b1af708c9096be7 Mon Sep 17 00:00:00 2001 From: kosabogi <105062005+kosabogi@users.noreply.github.com> Date: Thu, 26 Sep 2024 16:12:28 +0200 Subject: [PATCH 13/43] Adds text_similarity task type to inference processor documentation (#113517) --- .../ingest/processors/inference.asciidoc | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/docs/reference/ingest/processors/inference.asciidoc b/docs/reference/ingest/processors/inference.asciidoc index c942959d34e53..fa4f246cdd7c8 100644 --- a/docs/reference/ingest/processors/inference.asciidoc +++ b/docs/reference/ingest/processors/inference.asciidoc @@ -455,6 +455,29 @@ include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenizatio ======= ===== +[discrete] +[[inference-processor-text-similarity-opt]] +==== Text similarity configuration options + +`text_similarity`::: +(Object, optional) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-text-similarity] ++ +.Properties of text_similarity inference +[%collapsible%open] +===== +`span_score_combination_function`:::: +(Optional, string) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-text-similarity-span-score-func] + +`tokenization`:::: +(Optional, object) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization] ++ +Refer to <> to review the properties of the +`tokenization` object. +===== + [discrete] [[inference-processor-zero-shot-opt]] From f0339ed30c054664ed4180dbab98f8efabb713d4 Mon Sep 17 00:00:00 2001 From: Jan Kuipers <148754765+jan-elastic@users.noreply.github.com> Date: Thu, 26 Sep 2024 16:18:18 +0200 Subject: [PATCH 14/43] Adaptive allocations: scale to zero allocations (#113455) --- .../test/cluster/FeatureFlag.java | 3 +- .../UpdateTrainedModelDeploymentAction.java | 2 +- .../xpack/ml/MachineLearning.java | 14 ++- .../xpack/ml/MlInitializationService.java | 7 +- .../TransportExternalInferModelAction.java | 7 +- .../TransportInternalInferModelAction.java | 23 +++-- .../AdaptiveAllocationsScaler.java | 23 ++++- .../AdaptiveAllocationsScalerService.java | 95 +++++++++++-------- .../ScaleToZeroFeatureFlag.java | 20 ++++ .../ml/MlInitializationServiceTests.java | 14 +-- .../AdaptiveAllocationsScalerTests.java | 43 +++++++++ 11 files changed, 180 insertions(+), 71 deletions(-) create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/ScaleToZeroFeatureFlag.java diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java index cb98f9de31ff5..7df791bf11559 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java @@ -18,7 +18,8 @@ public enum FeatureFlag { TIME_SERIES_MODE("es.index_mode_feature_flag_registered=true", Version.fromString("8.0.0"), null), FAILURE_STORE_ENABLED("es.failure_store_feature_flag_enabled=true", Version.fromString("8.12.0"), null), - CHUNKING_SETTINGS_ENABLED("es.inference_chunking_settings_feature_flag_enabled=true", Version.fromString("8.16.0"), null); + CHUNKING_SETTINGS_ENABLED("es.inference_chunking_settings_feature_flag_enabled=true", Version.fromString("8.16.0"), null), + INFERENCE_SCALE_TO_ZERO("es.inference_scale_to_zero_feature_flag_enabled=true", Version.fromString("8.16.0"), null); public final String systemProperty; public final Version from; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/UpdateTrainedModelDeploymentAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/UpdateTrainedModelDeploymentAction.java index 7fca223b2ee7e..cb578fdb157de 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/UpdateTrainedModelDeploymentAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/UpdateTrainedModelDeploymentAction.java @@ -161,7 +161,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws public ActionRequestValidationException validate() { ActionRequestValidationException validationException = new ActionRequestValidationException(); if (numberOfAllocations != null) { - if (numberOfAllocations < 1) { + if (numberOfAllocations < 0 || (isInternal == false && numberOfAllocations == 0)) { validationException.addValidationError("[" + NUMBER_OF_ALLOCATIONS + "] must be a positive integer"); } if (isInternal == false diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java index 1bc867a849090..f8a590a23a2c1 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java @@ -327,6 +327,7 @@ import org.elasticsearch.xpack.ml.dataframe.process.results.AnalyticsResult; import org.elasticsearch.xpack.ml.dataframe.process.results.MemoryUsageEstimationResult; import org.elasticsearch.xpack.ml.inference.TrainedModelStatsService; +import org.elasticsearch.xpack.ml.inference.adaptiveallocations.AdaptiveAllocationsScalerService; import org.elasticsearch.xpack.ml.inference.assignment.TrainedModelAssignmentClusterService; import org.elasticsearch.xpack.ml.inference.assignment.TrainedModelAssignmentService; import org.elasticsearch.xpack.ml.inference.deployment.DeploymentManager; @@ -1285,13 +1286,21 @@ public Collection createComponents(PluginServices services) { new MlAutoscalingDeciderService(memoryTracker, settings, nodeAvailabilityZoneMapper, clusterService) ); - MlInitializationService mlInitializationService = new MlInitializationService( - settings, + AdaptiveAllocationsScalerService adaptiveAllocationsScalerService = new AdaptiveAllocationsScalerService( threadPool, clusterService, client, inferenceAuditor, telemetryProvider.getMeterRegistry(), + machineLearningExtension.get().isNlpEnabled() + ); + + MlInitializationService mlInitializationService = new MlInitializationService( + settings, + threadPool, + clusterService, + client, + adaptiveAllocationsScalerService, mlAssignmentNotifier, machineLearningExtension.get().isAnomalyDetectionEnabled(), machineLearningExtension.get().isDataFrameAnalyticsEnabled(), @@ -1317,6 +1326,7 @@ public Collection createComponents(PluginServices services) { jobManagerHolder, autodetectProcessManager, mlInitializationService, + adaptiveAllocationsScalerService, jobDataCountsPersister, datafeedRunner, datafeedManager, diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlInitializationService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlInitializationService.java index 98dfb13d9e3e4..45a71a80de077 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlInitializationService.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlInitializationService.java @@ -30,11 +30,9 @@ import org.elasticsearch.common.component.LifecycleListener; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.gateway.GatewayService; -import org.elasticsearch.telemetry.metric.MeterRegistry; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.core.ml.annotations.AnnotationIndex; import org.elasticsearch.xpack.ml.inference.adaptiveallocations.AdaptiveAllocationsScalerService; -import org.elasticsearch.xpack.ml.notifications.InferenceAuditor; import java.util.Collections; import java.util.Map; @@ -67,8 +65,7 @@ public final class MlInitializationService implements ClusterStateListener { ThreadPool threadPool, ClusterService clusterService, Client client, - InferenceAuditor inferenceAuditor, - MeterRegistry meterRegistry, + AdaptiveAllocationsScalerService adaptiveAllocationsScalerService, MlAssignmentNotifier mlAssignmentNotifier, boolean isAnomalyDetectionEnabled, boolean isDataFrameAnalyticsEnabled, @@ -88,7 +85,7 @@ public final class MlInitializationService implements ClusterStateListener { isDataFrameAnalyticsEnabled, isNlpEnabled ), - new AdaptiveAllocationsScalerService(threadPool, clusterService, client, inferenceAuditor, meterRegistry, isNlpEnabled), + adaptiveAllocationsScalerService, clusterService ); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportExternalInferModelAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportExternalInferModelAction.java index 545dcfbefecec..5603e9c4dca8d 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportExternalInferModelAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportExternalInferModelAction.java @@ -13,6 +13,7 @@ import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.ml.action.InferModelAction; +import org.elasticsearch.xpack.ml.inference.adaptiveallocations.AdaptiveAllocationsScalerService; import org.elasticsearch.xpack.ml.inference.loadingservice.ModelLoadingService; import org.elasticsearch.xpack.ml.inference.persistence.TrainedModelProvider; @@ -25,7 +26,8 @@ public TransportExternalInferModelAction( Client client, ClusterService clusterService, XPackLicenseState licenseState, - TrainedModelProvider trainedModelProvider + TrainedModelProvider trainedModelProvider, + AdaptiveAllocationsScalerService adaptiveAllocationsScalerService ) { super( InferModelAction.EXTERNAL_NAME, @@ -35,7 +37,8 @@ public TransportExternalInferModelAction( client, clusterService, licenseState, - trainedModelProvider + trainedModelProvider, + adaptiveAllocationsScalerService ); } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportInternalInferModelAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportInternalInferModelAction.java index 0c4064348b3f6..b69f8c7d62eb2 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportInternalInferModelAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportInternalInferModelAction.java @@ -42,6 +42,7 @@ import org.elasticsearch.xpack.core.ml.inference.results.ErrorInferenceResults; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; import org.elasticsearch.xpack.ml.MachineLearning; +import org.elasticsearch.xpack.ml.inference.adaptiveallocations.AdaptiveAllocationsScalerService; import org.elasticsearch.xpack.ml.inference.loadingservice.LocalModel; import org.elasticsearch.xpack.ml.inference.loadingservice.ModelLoadingService; import org.elasticsearch.xpack.ml.inference.persistence.TrainedModelProvider; @@ -66,6 +67,7 @@ public class TransportInternalInferModelAction extends HandledTransportAction format("[%s] model deployment not allocated to any node", assignment.getDeploymentId())); - listener.onFailure( - ExceptionsHelper.conflictStatusException("Trained model deployment [" + request.getId() + "] is not allocated to any nodes") - ); + String message = "Trained model deployment [" + request.getId() + "] is not allocated to any nodes"; + boolean starting = adaptiveAllocationsScalerService.maybeStartAllocation(assignment); + if (starting) { + message += "; starting deployment of one allocation"; + } + logger.debug(message); + listener.onFailure(ExceptionsHelper.conflictStatusException(message)); return; } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScaler.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScaler.java index 044556d1b30ac..05e7202b8efe9 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScaler.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScaler.java @@ -10,6 +10,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.common.Strings; +import org.elasticsearch.core.TimeValue; /** * Processes measured requests counts and inference times and decides whether @@ -21,6 +22,12 @@ public class AdaptiveAllocationsScaler { static final double SCALE_UP_THRESHOLD = 0.9; private static final double SCALE_DOWN_THRESHOLD = 0.85; + /** + * The time interval without any requests that has to pass, before scaling down + * to zero allocations (in case min_allocations = 0). + */ + private static final long SCALE_TO_ZERO_AFTER_NO_REQUESTS_TIME_SECONDS = TimeValue.timeValueMinutes(15).getSeconds(); + /** * If the max_number_of_allocations is not set, use this value for now to prevent scaling up * to high numbers due to possible bugs or unexpected behaviour in the scaler. @@ -33,6 +40,7 @@ public class AdaptiveAllocationsScaler { private final String deploymentId; private final KalmanFilter1d requestRateEstimator; private final KalmanFilter1d inferenceTimeEstimator; + private double timeWithoutRequestsSeconds; private int numberOfAllocations; private int neededNumberOfAllocations; @@ -55,6 +63,7 @@ public class AdaptiveAllocationsScaler { // the number of allocations changes, which is passed explicitly to the estimator. requestRateEstimator = new KalmanFilter1d(deploymentId + ":rate", 100, true); inferenceTimeEstimator = new KalmanFilter1d(deploymentId + ":time", 100, false); + timeWithoutRequestsSeconds = 0.0; this.numberOfAllocations = numberOfAllocations; neededNumberOfAllocations = numberOfAllocations; minNumberOfAllocations = null; @@ -73,6 +82,11 @@ void setMinMaxNumberOfAllocations(Integer minNumberOfAllocations, Integer maxNum void process(AdaptiveAllocationsScalerService.Stats stats, double timeIntervalSeconds, int numberOfAllocations) { lastMeasuredQueueSize = stats.pendingCount(); + if (stats.requestCount() > 0) { + timeWithoutRequestsSeconds = 0.0; + } else { + timeWithoutRequestsSeconds += timeIntervalSeconds; + } // The request rate (per second) is the request count divided by the time. // Assuming a Poisson process for the requests, the variance in the request @@ -145,7 +159,7 @@ Integer scale() { numberOfAllocations--; } - this.neededNumberOfAllocations = numberOfAllocations; + neededNumberOfAllocations = numberOfAllocations; if (maxNumberOfAllocations == null) { numberOfAllocations = Math.min(numberOfAllocations, MAX_NUMBER_OF_ALLOCATIONS_SAFEGUARD); @@ -156,6 +170,13 @@ Integer scale() { if (maxNumberOfAllocations != null) { numberOfAllocations = Math.min(numberOfAllocations, maxNumberOfAllocations); } + if (ScaleToZeroFeatureFlag.isEnabled() + && (minNumberOfAllocations == null || minNumberOfAllocations == 0) + && timeWithoutRequestsSeconds > SCALE_TO_ZERO_AFTER_NO_REQUESTS_TIME_SECONDS) { + logger.debug("[{}] adaptive allocations scaler: scaling down to zero, because of no requests.", deploymentId); + numberOfAllocations = 0; + neededNumberOfAllocations = 0; + } if (numberOfAllocations != oldNumberOfAllocations) { logger.debug( diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerService.java index bbe90f769818b..775279a6b2553 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerService.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerService.java @@ -415,49 +415,60 @@ private void processDeploymentStats(GetDeploymentStatsAction.Response statsRespo if (newNumberOfAllocations > numberOfAllocations.get(deploymentId)) { lastScaleUpTimesMillis.put(deploymentId, now); } - UpdateTrainedModelDeploymentAction.Request updateRequest = new UpdateTrainedModelDeploymentAction.Request(deploymentId); - updateRequest.setNumberOfAllocations(newNumberOfAllocations); - updateRequest.setIsInternal(true); - ClientHelper.executeAsyncWithOrigin( - client, - ClientHelper.ML_ORIGIN, - UpdateTrainedModelDeploymentAction.INSTANCE, - updateRequest, - ActionListener.wrap(updateResponse -> { - logger.info("adaptive allocations scaler: scaled [{}] to [{}] allocations.", deploymentId, newNumberOfAllocations); - threadPool.executor(MachineLearning.UTILITY_THREAD_POOL_NAME) - .execute( - () -> inferenceAuditor.info( - deploymentId, - Strings.format( - "adaptive allocations scaler: scaled [%s] to [%s] allocations.", - deploymentId, - newNumberOfAllocations - ) - ) - ); - }, e -> { - logger.atLevel(Level.WARN) - .withThrowable(e) - .log( - "adaptive allocations scaler: scaling [{}] to [{}] allocations failed.", - deploymentId, - newNumberOfAllocations - ); - threadPool.executor(MachineLearning.UTILITY_THREAD_POOL_NAME) - .execute( - () -> inferenceAuditor.warning( - deploymentId, - Strings.format( - "adaptive allocations scaler: scaling [%s] to [%s] allocations failed.", - deploymentId, - newNumberOfAllocations - ) - ) - ); - }) - ); + updateNumberOfAllocations(deploymentId, newNumberOfAllocations); } } } + + public boolean maybeStartAllocation(TrainedModelAssignment assignment) { + if (ScaleToZeroFeatureFlag.isEnabled() + && assignment.getAdaptiveAllocationsSettings() != null + && assignment.getAdaptiveAllocationsSettings().getEnabled() == Boolean.TRUE) { + lastScaleUpTimesMillis.put(assignment.getDeploymentId(), System.currentTimeMillis()); + updateNumberOfAllocations(assignment.getDeploymentId(), 1); + return true; + } + return false; + } + + private void updateNumberOfAllocations(String deploymentId, int numberOfAllocations) { + UpdateTrainedModelDeploymentAction.Request updateRequest = new UpdateTrainedModelDeploymentAction.Request(deploymentId); + updateRequest.setNumberOfAllocations(numberOfAllocations); + updateRequest.setIsInternal(true); + ClientHelper.executeAsyncWithOrigin( + client, + ClientHelper.ML_ORIGIN, + UpdateTrainedModelDeploymentAction.INSTANCE, + updateRequest, + ActionListener.wrap(updateResponse -> { + logger.info("adaptive allocations scaler: scaled [{}] to [{}] allocations.", deploymentId, numberOfAllocations); + threadPool.executor(MachineLearning.UTILITY_THREAD_POOL_NAME) + .execute( + () -> inferenceAuditor.info( + deploymentId, + Strings.format( + "adaptive allocations scaler: scaled [%s] to [%s] allocations.", + deploymentId, + numberOfAllocations + ) + ) + ); + }, e -> { + logger.atLevel(Level.WARN) + .withThrowable(e) + .log("adaptive allocations scaler: scaling [{}] to [{}] allocations failed.", deploymentId, numberOfAllocations); + threadPool.executor(MachineLearning.UTILITY_THREAD_POOL_NAME) + .execute( + () -> inferenceAuditor.warning( + deploymentId, + Strings.format( + "adaptive allocations scaler: scaling [%s] to [%s] allocations failed.", + deploymentId, + numberOfAllocations + ) + ) + ); + }) + ); + } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/ScaleToZeroFeatureFlag.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/ScaleToZeroFeatureFlag.java new file mode 100644 index 0000000000000..072b8c5593c93 --- /dev/null +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/ScaleToZeroFeatureFlag.java @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.ml.inference.adaptiveallocations; + +import org.elasticsearch.common.util.FeatureFlag; + +public class ScaleToZeroFeatureFlag { + private ScaleToZeroFeatureFlag() {} + + private static final FeatureFlag FEATURE_FLAG = new FeatureFlag("inference_scale_to_zero"); + + public static boolean isEnabled() { + return FEATURE_FLAG.isEnabled(); + } +} diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlInitializationServiceTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlInitializationServiceTests.java index a5b9597886e15..80c957ecb7a09 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlInitializationServiceTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlInitializationServiceTests.java @@ -17,11 +17,9 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.DeterministicTaskQueue; -import org.elasticsearch.telemetry.metric.MeterRegistry; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.ml.inference.adaptiveallocations.AdaptiveAllocationsScalerService; -import org.elasticsearch.xpack.ml.notifications.InferenceAuditor; import org.junit.Before; import java.util.Map; @@ -40,8 +38,7 @@ public class MlInitializationServiceTests extends ESTestCase { private ThreadPool threadPool; private ClusterService clusterService; private Client client; - private InferenceAuditor inferenceAuditor; - private MeterRegistry meterRegistry; + private AdaptiveAllocationsScalerService adaptiveAllocationsScalerService; private MlAssignmentNotifier mlAssignmentNotifier; @Before @@ -50,8 +47,7 @@ public void setUpMocks() { threadPool = deterministicTaskQueue.getThreadPool(); clusterService = mock(ClusterService.class); client = mock(Client.class); - inferenceAuditor = mock(InferenceAuditor.class); - meterRegistry = mock(MeterRegistry.class); + adaptiveAllocationsScalerService = mock(AdaptiveAllocationsScalerService.class); mlAssignmentNotifier = mock(MlAssignmentNotifier.class); when(clusterService.getClusterName()).thenReturn(CLUSTER_NAME); @@ -77,8 +73,7 @@ public void testInitialize() { threadPool, clusterService, client, - inferenceAuditor, - meterRegistry, + adaptiveAllocationsScalerService, mlAssignmentNotifier, true, true, @@ -94,8 +89,7 @@ public void testInitialize_noMasterNode() { threadPool, clusterService, client, - inferenceAuditor, - meterRegistry, + adaptiveAllocationsScalerService, mlAssignmentNotifier, true, true, diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerTests.java index 08097357725d0..44aaba88c58a8 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerTests.java @@ -146,4 +146,47 @@ public void testAutoscaling_maxAllocationsSafeguard() { adaptiveAllocationsScaler.setMinMaxNumberOfAllocations(2, 77); assertThat(adaptiveAllocationsScaler.scale(), equalTo(77)); } + + public void testAutoscaling_scaleDownToZeroAllocations() { + AdaptiveAllocationsScaler adaptiveAllocationsScaler = new AdaptiveAllocationsScaler("test-deployment", 1); + // 1 hour with 1 request per 1 seconds, so don't scale. + for (int i = 0; i < 3600; i++) { + adaptiveAllocationsScaler.process(new AdaptiveAllocationsScalerService.Stats(1, 0, 0, 0.05), 1, 1); + assertThat(adaptiveAllocationsScaler.scale(), nullValue()); + } + // 15 minutes with no requests, so don't scale. + for (int i = 0; i < 900; i++) { + adaptiveAllocationsScaler.process(new AdaptiveAllocationsScalerService.Stats(0, 0, 0, 0.05), 1, 1); + assertThat(adaptiveAllocationsScaler.scale(), nullValue()); + } + // 1 second with a request, so don't scale. + adaptiveAllocationsScaler.process(new AdaptiveAllocationsScalerService.Stats(1, 0, 0, 0.05), 1, 1); + assertThat(adaptiveAllocationsScaler.scale(), nullValue()); + // 15 minutes with no requests, so don't scale. + for (int i = 0; i < 900; i++) { + adaptiveAllocationsScaler.process(new AdaptiveAllocationsScalerService.Stats(0, 0, 0, 0.05), 1, 1); + assertThat(adaptiveAllocationsScaler.scale(), nullValue()); + } + // another second with no requests, so scale to zero allocations. + adaptiveAllocationsScaler.process(new AdaptiveAllocationsScalerService.Stats(0, 0, 0, 0.05), 1, 1); + assertThat(adaptiveAllocationsScaler.scale(), equalTo(0)); + // 15 minutes with no requests, so don't scale. + for (int i = 0; i < 900; i++) { + adaptiveAllocationsScaler.process(new AdaptiveAllocationsScalerService.Stats(0, 0, 0, 0.05), 1, 0); + assertThat(adaptiveAllocationsScaler.scale(), nullValue()); + } + } + + public void testAutoscaling_dontScaleDownToZeroAllocationsWhenMinAllocationsIsSet() { + assumeTrue("Should only run if adaptive allocations feature flag is enabled", ScaleToZeroFeatureFlag.isEnabled()); + + AdaptiveAllocationsScaler adaptiveAllocationsScaler = new AdaptiveAllocationsScaler("test-deployment", 1); + adaptiveAllocationsScaler.setMinMaxNumberOfAllocations(1, null); + + // 1 hour with no requests, + for (int i = 0; i < 3600; i++) { + adaptiveAllocationsScaler.process(new AdaptiveAllocationsScalerService.Stats(1, 0, 0, 0.05), 1, 1); + assertThat(adaptiveAllocationsScaler.scale(), nullValue()); + } + } } From 052dbb4dacca29c3abf96e1b1579c569a2ec7095 Mon Sep 17 00:00:00 2001 From: Kushal-Dalasaniya <108124477+Kushal-Dalasaniya@users.noreply.github.com> Date: Thu, 26 Sep 2024 20:12:57 +0530 Subject: [PATCH 15/43] Optimize error handling after lazy rollovers (#111572) This commit improves the performance of the error-handling process after a lazy rollover or an index creation failed. --- .../action/bulk/BulkOperation.java | 20 --- .../action/bulk/TransportBulkAction.java | 113 ++++++------ .../action/bulk/BulkOperationTests.java | 6 - ...ActionIndicesThatCannotBeCreatedTests.java | 164 ------------------ .../bulk/TransportBulkActionIngestTests.java | 4 +- .../action/bulk/TransportBulkActionTests.java | 100 ++++++++++- .../bulk/TransportBulkActionTookTests.java | 12 +- 7 files changed, 164 insertions(+), 255 deletions(-) delete mode 100644 server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkOperation.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkOperation.java index 1789acc1cb7a6..f04d07fb690c4 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/BulkOperation.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkOperation.java @@ -87,7 +87,6 @@ final class BulkOperation extends ActionRunnable { private final ConcurrentLinkedQueue failureStoreRedirects = new ConcurrentLinkedQueue<>(); private final long startTimeNanos; private final ClusterStateObserver observer; - private final Map indicesThatCannotBeCreated; private final Executor executor; private final LongSupplier relativeTimeProvider; private final FailureStoreDocumentConverter failureStoreDocumentConverter; @@ -107,7 +106,6 @@ final class BulkOperation extends ActionRunnable { BulkRequest bulkRequest, NodeClient client, AtomicArray responses, - Map indicesThatCannotBeCreated, IndexNameExpressionResolver indexNameExpressionResolver, LongSupplier relativeTimeProvider, long startTimeNanos, @@ -122,7 +120,6 @@ final class BulkOperation extends ActionRunnable { bulkRequest, client, responses, - indicesThatCannotBeCreated, indexNameExpressionResolver, relativeTimeProvider, startTimeNanos, @@ -141,7 +138,6 @@ final class BulkOperation extends ActionRunnable { BulkRequest bulkRequest, NodeClient client, AtomicArray responses, - Map indicesThatCannotBeCreated, IndexNameExpressionResolver indexNameExpressionResolver, LongSupplier relativeTimeProvider, long startTimeNanos, @@ -158,7 +154,6 @@ final class BulkOperation extends ActionRunnable { this.bulkRequest = bulkRequest; this.listener = listener; this.startTimeNanos = startTimeNanos; - this.indicesThatCannotBeCreated = indicesThatCannotBeCreated; this.executor = executor; this.relativeTimeProvider = relativeTimeProvider; this.indexNameExpressionResolver = indexNameExpressionResolver; @@ -298,9 +293,6 @@ private Map> groupRequestsByShards( if (addFailureIfRequiresAliasAndAliasIsMissing(docWriteRequest, bulkItemRequest.id(), metadata)) { continue; } - if (addFailureIfIndexCannotBeCreated(docWriteRequest, bulkItemRequest.id())) { - continue; - } if (addFailureIfRequiresDataStreamAndNoParentDataStream(docWriteRequest, bulkItemRequest.id(), metadata)) { continue; } @@ -759,18 +751,6 @@ private boolean addFailureIfIndexIsClosed(DocWriteRequest request, Index conc return false; } - private boolean addFailureIfIndexCannotBeCreated(DocWriteRequest request, int idx) { - IndexNotFoundException cannotCreate = indicesThatCannotBeCreated.get(request.index()); - if (cannotCreate != null) { - var failureStoreStatus = isFailureStoreRequest(request) - ? IndexDocFailureStoreStatus.FAILED - : IndexDocFailureStoreStatus.NOT_APPLICABLE_OR_UNKNOWN; - addFailureAndDiscardRequest(request, idx, request.index(), cannotCreate, failureStoreStatus); - return true; - } - return false; - } - private static boolean isFailureStoreRequest(DocWriteRequest request) { return request instanceof IndexRequest ir && ir.isWriteToFailureStore(); } diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java index 03768af029141..61adf41a9a276 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java @@ -43,7 +43,6 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.util.concurrent.AtomicArray; import org.elasticsearch.features.FeatureService; -import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.IndexingPressure; import org.elasticsearch.index.VersionType; import org.elasticsearch.indices.SystemIndices; @@ -60,6 +59,7 @@ import java.util.Objects; import java.util.Set; import java.util.SortedMap; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executor; import java.util.function.Function; import java.util.function.LongSupplier; @@ -351,29 +351,36 @@ protected void createMissingIndicesAndIndexData( final AtomicArray responses = new AtomicArray<>(bulkRequest.requests.size()); // Optimizing when there are no prerequisite actions if (indicesToAutoCreate.isEmpty() && dataStreamsToBeRolledOver.isEmpty() && failureStoresToBeRolledOver.isEmpty()) { - executeBulk(task, bulkRequest, startTimeNanos, listener, executor, responses, Map.of()); + executeBulk(task, bulkRequest, startTimeNanos, listener, executor, responses); return; } - final Map indicesThatCannotBeCreated = new HashMap<>(); + Map indicesExceptions = new ConcurrentHashMap<>(); + Map dataStreamExceptions = new ConcurrentHashMap<>(); + Map failureStoreExceptions = new ConcurrentHashMap<>(); Runnable executeBulkRunnable = () -> executor.execute(new ActionRunnable<>(listener) { @Override protected void doRun() { - executeBulk(task, bulkRequest, startTimeNanos, listener, executor, responses, indicesThatCannotBeCreated); + failRequestsWhenPrerequisiteActionFailed( + indicesExceptions, + dataStreamExceptions, + failureStoreExceptions, + bulkRequest, + responses + ); + executeBulk(task, bulkRequest, startTimeNanos, listener, executor, responses); } }); try (RefCountingRunnable refs = new RefCountingRunnable(executeBulkRunnable)) { - createIndices(bulkRequest, indicesToAutoCreate, indicesThatCannotBeCreated, responses, refs); - rollOverDataStreams(bulkRequest, dataStreamsToBeRolledOver, false, responses, refs); - rollOverDataStreams(bulkRequest, failureStoresToBeRolledOver, true, responses, refs); + createIndices(indicesToAutoCreate, refs, indicesExceptions); + rollOverDataStreams(bulkRequest, dataStreamsToBeRolledOver, false, refs, dataStreamExceptions); + rollOverDataStreams(bulkRequest, failureStoresToBeRolledOver, true, refs, failureStoreExceptions); } } private void createIndices( - BulkRequest bulkRequest, Map indicesToAutoCreate, - Map indicesThatCannotBeCreated, - AtomicArray responses, - RefCountingRunnable refs + RefCountingRunnable refs, + final Map indicesExceptions ) { for (Map.Entry indexEntry : indicesToAutoCreate.entrySet()) { final String index = indexEntry.getKey(); @@ -384,25 +391,26 @@ public void onResponse(CreateIndexResponse createIndexResponse) {} @Override public void onFailure(Exception e) { final Throwable cause = ExceptionsHelper.unwrapCause(e); - if (cause instanceof IndexNotFoundException indexNotFoundException) { - synchronized (indicesThatCannotBeCreated) { - indicesThatCannotBeCreated.put(index, indexNotFoundException); - } - } else if ((cause instanceof ResourceAlreadyExistsException) == false) { + if ((cause instanceof ResourceAlreadyExistsException) == false) { // fail all requests involving this index, if create didn't work - failRequestsWhenPrerequisiteActionFailed(index, bulkRequest, responses, e); + indicesExceptions.put(index, e); } } }, refs.acquire())); } } + // Separate method to allow for overriding in tests. + void createIndex(CreateIndexRequest createIndexRequest, ActionListener listener) { + client.execute(AutoCreateAction.INSTANCE, createIndexRequest, listener); + } + private void rollOverDataStreams( BulkRequest bulkRequest, Set dataStreamsToBeRolledOver, boolean targetFailureStore, - AtomicArray responses, - RefCountingRunnable refs + RefCountingRunnable refs, + Map dataStreamExceptions ) { for (String dataStream : dataStreamsToBeRolledOver) { RolloverRequest rolloverRequest = new RolloverRequest(dataStream, null); @@ -416,7 +424,7 @@ private void rollOverDataStreams( } // We are executing a lazy rollover because it is an action specialised for this situation, when we want an // unconditional and performant rollover. - rolloverClient.execute(LazyRolloverAction.INSTANCE, rolloverRequest, ActionListener.releaseAfter(new ActionListener<>() { + rollOver(rolloverRequest, ActionListener.releaseAfter(new ActionListener<>() { @Override public void onResponse(RolloverResponse result) { @@ -431,26 +439,52 @@ public void onResponse(RolloverResponse result) { @Override public void onFailure(Exception e) { - failRequestsWhenPrerequisiteActionFailed(dataStream, bulkRequest, responses, e); + dataStreamExceptions.put(dataStream, e); } }, refs.acquire())); } } + // Separate method to allow for overriding in tests. + void rollOver(RolloverRequest rolloverRequest, ActionListener listener) { + rolloverClient.execute(LazyRolloverAction.INSTANCE, rolloverRequest, listener); + } + /** - * Fails all requests involving this index or data stream because the prerequisite action failed too. + * Mark all the requests for which the prerequisite action failed (i.e. index creation or data stream/failure store rollover) as failed. */ - private static void failRequestsWhenPrerequisiteActionFailed( - String target, + private void failRequestsWhenPrerequisiteActionFailed( + Map indicesExceptions, + Map dataStreamExceptions, + Map failureStoreExceptions, BulkRequest bulkRequest, - AtomicArray responses, - Exception error + AtomicArray responses ) { + if (indicesExceptions.isEmpty() && dataStreamExceptions.isEmpty() && failureStoreExceptions.isEmpty()) { + return; + } for (int i = 0; i < bulkRequest.requests.size(); i++) { DocWriteRequest request = bulkRequest.requests.get(i); - if (request != null && setResponseFailureIfIndexMatches(responses, i, request, target, error)) { - bulkRequest.requests.set(i, null); + if (request == null) { + continue; } + var exception = indicesExceptions.get(request.index()); + if (exception == null) { + if (request instanceof IndexRequest indexRequest && indexRequest.isWriteToFailureStore()) { + exception = failureStoreExceptions.get(request.index()); + } else { + exception = dataStreamExceptions.get(request.index()); + } + } + if (exception == null) { + continue; + } + var failureStoreStatus = request instanceof IndexRequest ir && ir.isWriteToFailureStore() + ? IndexDocFailureStoreStatus.FAILED + : IndexDocFailureStoreStatus.NOT_APPLICABLE_OR_UNKNOWN; + var failure = new BulkItemResponse.Failure(request.index(), request.id(), exception, failureStoreStatus); + responses.set(i, BulkItemResponse.failure(i, request.opType(), failure)); + bulkRequest.requests.set(i, null); } } @@ -532,33 +566,13 @@ private static boolean isSystemIndex(SortedMap indices } } - void createIndex(CreateIndexRequest createIndexRequest, ActionListener listener) { - client.execute(AutoCreateAction.INSTANCE, createIndexRequest, listener); - } - - private static boolean setResponseFailureIfIndexMatches( - AtomicArray responses, - int idx, - DocWriteRequest request, - String index, - Exception e - ) { - if (index.equals(request.index())) { - BulkItemResponse.Failure failure = new BulkItemResponse.Failure(request.index(), request.id(), e); - responses.set(idx, BulkItemResponse.failure(idx, request.opType(), failure)); - return true; - } - return false; - } - void executeBulk( Task task, BulkRequest bulkRequest, long startTimeNanos, ActionListener listener, Executor executor, - AtomicArray responses, - Map indicesThatCannotBeCreated + AtomicArray responses ) { new BulkOperation( task, @@ -568,7 +582,6 @@ void executeBulk( bulkRequest, client, responses, - indicesThatCannotBeCreated, indexNameExpressionResolver, relativeTimeNanosProvider, startTimeNanos, diff --git a/server/src/test/java/org/elasticsearch/action/bulk/BulkOperationTests.java b/server/src/test/java/org/elasticsearch/action/bulk/BulkOperationTests.java index 3be942bcd291e..b87dfd07181dc 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/BulkOperationTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/BulkOperationTests.java @@ -42,7 +42,6 @@ import org.elasticsearch.common.util.concurrent.AtomicArray; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.concurrent.ThreadContext; -import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.mapper.MapperException; import org.elasticsearch.index.shard.ShardId; @@ -1022,7 +1021,6 @@ private BulkOperation newBulkOperation(NodeClient client, BulkRequest request, A client, request, new AtomicArray<>(request.numberOfActions()), - Map.of(), mockObserver(DEFAULT_STATE), listener, new FailureStoreDocumentConverter() @@ -1040,7 +1038,6 @@ private BulkOperation newBulkOperation( client, request, new AtomicArray<>(request.numberOfActions()), - Map.of(), mockObserver(DEFAULT_STATE), listener, failureStoreDocumentConverter @@ -1059,7 +1056,6 @@ private BulkOperation newBulkOperation( client, request, new AtomicArray<>(request.numberOfActions()), - Map.of(), observer, listener, new FailureStoreDocumentConverter() @@ -1071,7 +1067,6 @@ private BulkOperation newBulkOperation( NodeClient client, BulkRequest request, AtomicArray existingResponses, - Map indicesThatCanNotBeCreated, ClusterStateObserver observer, ActionListener listener, FailureStoreDocumentConverter failureStoreDocumentConverter @@ -1100,7 +1095,6 @@ private BulkOperation newBulkOperation( request, client, existingResponses, - indicesThatCanNotBeCreated, indexNameExpressionResolver, () -> endTime, timeZero, diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java deleted file mode 100644 index 2f5e6b22e1a8c..0000000000000 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java +++ /dev/null @@ -1,164 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.action.bulk; - -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; -import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; -import org.elasticsearch.action.delete.DeleteRequest; -import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.action.support.ActionFilters; -import org.elasticsearch.action.support.ActionTestUtils; -import org.elasticsearch.action.update.UpdateRequest; -import org.elasticsearch.client.internal.node.NodeClient; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.block.ClusterBlocks; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.metadata.Metadata; -import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.concurrent.AtomicArray; -import org.elasticsearch.common.util.concurrent.ThreadContext; -import org.elasticsearch.features.FeatureService; -import org.elasticsearch.index.IndexNotFoundException; -import org.elasticsearch.index.IndexingPressure; -import org.elasticsearch.index.VersionType; -import org.elasticsearch.indices.EmptySystemIndices; -import org.elasticsearch.tasks.Task; -import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.MockUtils; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.transport.TransportService; - -import java.util.Map; -import java.util.Set; -import java.util.concurrent.Executor; -import java.util.function.Consumer; -import java.util.function.Function; - -import static java.util.Collections.emptySet; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -public class TransportBulkActionIndicesThatCannotBeCreatedTests extends ESTestCase { - private static final Consumer noop = index -> {}; - - public void testNonExceptional() { - BulkRequest bulkRequest = new BulkRequest(); - bulkRequest.add(new IndexRequest(randomAlphaOfLength(5))); - bulkRequest.add(new IndexRequest(randomAlphaOfLength(5))); - bulkRequest.add(new DeleteRequest(randomAlphaOfLength(5))); - bulkRequest.add(new UpdateRequest(randomAlphaOfLength(5), randomAlphaOfLength(5))); - // Test emulating that index can be auto-created - indicesThatCannotBeCreatedTestCase(emptySet(), bulkRequest, index -> true, noop); - // Test emulating that index cannot be auto-created - indicesThatCannotBeCreatedTestCase(emptySet(), bulkRequest, index -> false, noop); - // Test emulating auto_create_index=true with some indices already created. - indicesThatCannotBeCreatedTestCase(emptySet(), bulkRequest, index -> randomBoolean(), noop); - } - - public void testAllFail() { - BulkRequest bulkRequest = new BulkRequest(); - bulkRequest.add(new IndexRequest("no")); - bulkRequest.add(new IndexRequest("can't")); - bulkRequest.add(new DeleteRequest("do").version(0).versionType(VersionType.EXTERNAL)); - bulkRequest.add(new UpdateRequest("nothin", randomAlphaOfLength(5))); - indicesThatCannotBeCreatedTestCase(Set.of("no", "can't", "do", "nothin"), bulkRequest, index -> true, index -> { - throw new IndexNotFoundException("Can't make it because I say so"); - }); - } - - public void testSomeFail() { - BulkRequest bulkRequest = new BulkRequest(); - bulkRequest.add(new IndexRequest("ok")); - bulkRequest.add(new IndexRequest("bad")); - // Emulate auto_create_index=-bad,+* - indicesThatCannotBeCreatedTestCase(Set.of("bad"), bulkRequest, index -> true, index -> { - if (index.equals("bad")) { - throw new IndexNotFoundException("Can't make it because I say so"); - } - }); - } - - private void indicesThatCannotBeCreatedTestCase( - Set expected, - BulkRequest bulkRequest, - Function shouldAutoCreate, - Consumer simulateAutoCreate - ) { - ClusterService clusterService = mock(ClusterService.class); - ClusterState state = mock(ClusterState.class); - when(state.getMetadata()).thenReturn(Metadata.EMPTY_METADATA); - when(state.metadata()).thenReturn(Metadata.EMPTY_METADATA); - when(state.blocks()).thenReturn(mock(ClusterBlocks.class)); - when(clusterService.state()).thenReturn(state); - when(clusterService.getSettings()).thenReturn(Settings.EMPTY); - - DiscoveryNode localNode = mock(DiscoveryNode.class); - when(clusterService.localNode()).thenReturn(localNode); - when(localNode.isIngestNode()).thenReturn(randomBoolean()); - - final IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver( - new ThreadContext(Settings.EMPTY), - EmptySystemIndices.INSTANCE - ) { - @Override - public boolean hasIndexAbstraction(String indexAbstraction, ClusterState state) { - return shouldAutoCreate.apply(indexAbstraction) == false; - } - }; - - final ThreadPool threadPool = mock(ThreadPool.class); - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); - FeatureService mockFeatureService = mock(FeatureService.class); - when(mockFeatureService.clusterHasFeature(any(), any())).thenReturn(true); - TransportBulkAction action = new TransportBulkAction( - threadPool, - transportService, - clusterService, - null, - mockFeatureService, - new NodeClient(Settings.EMPTY, threadPool), - mock(ActionFilters.class), - indexNameExpressionResolver, - new IndexingPressure(Settings.EMPTY), - EmptySystemIndices.INSTANCE, - FailureStoreMetrics.NOOP - ) { - @Override - void executeBulk( - Task task, - BulkRequest bulkRequest, - long startTimeNanos, - ActionListener listener, - Executor executor, - AtomicArray responses, - Map indicesThatCannotBeCreated - ) { - assertEquals(expected, indicesThatCannotBeCreated.keySet()); - } - - @Override - void createIndex(CreateIndexRequest createIndexRequest, ActionListener listener) { - String index = createIndexRequest.index(); - try { - simulateAutoCreate.accept(index); - // If we try to create an index just immediately assume it worked - listener.onResponse(new CreateIndexResponse(true, true, index)); - } catch (Exception e) { - listener.onFailure(e); - } - } - }; - action.doExecute(null, bulkRequest, ActionTestUtils.assertNoFailureListener(bulkItemResponse -> {})); - } -} diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java index abdb02924e26c..112748cd73627 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java @@ -39,7 +39,6 @@ import org.elasticsearch.common.util.concurrent.AtomicArray; import org.elasticsearch.core.Nullable; import org.elasticsearch.features.FeatureService; -import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexingPressure; @@ -168,8 +167,7 @@ void executeBulk( long startTimeNanos, ActionListener listener, Executor executor, - AtomicArray responses, - Map indicesThatCannotBeCreated + AtomicArray responses ) { assertTrue(indexCreated); isExecuted = true; diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java index 5d55e22a080cd..eae6fbf9cbc7f 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java @@ -15,6 +15,8 @@ import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; +import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; +import org.elasticsearch.action.admin.indices.rollover.RolloverResponse; import org.elasticsearch.action.bulk.TransportBulkActionTookTests.Resolver; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.index.IndexRequest; @@ -39,6 +41,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.elasticsearch.features.FeatureService; +import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; @@ -90,7 +93,9 @@ public class TransportBulkActionTests extends ESTestCase { class TestTransportBulkAction extends TransportBulkAction { - volatile boolean failIndexCreation = false; + volatile Exception failIndexCreationException; + volatile Exception failDataStreamRolloverException; + volatile Exception failFailureStoreRolloverException; boolean indexCreated = false; // set when the "real" index is created Runnable beforeIndexCreation = null; @@ -116,12 +121,25 @@ void createIndex(CreateIndexRequest createIndexRequest, ActionListener listener) { + if (failDataStreamRolloverException != null && rolloverRequest.targetsFailureStore() == false) { + listener.onFailure(failDataStreamRolloverException); + } else if (failFailureStoreRolloverException != null && rolloverRequest.targetsFailureStore()) { + listener.onFailure(failFailureStoreRolloverException); + } else { + listener.onResponse( + new RolloverResponse(null, null, Map.of(), rolloverRequest.isDryRun(), true, true, true, rolloverRequest.isLazy()) + ); + } + } } @Before @@ -357,7 +375,7 @@ public void testRejectCoordination() { public void testRejectionAfterCreateIndexIsPropagated() { BulkRequest bulkRequest = new BulkRequest().add(new IndexRequest("index").id("id").source(Collections.emptyMap())); - bulkAction.failIndexCreation = randomBoolean(); + bulkAction.failIndexCreationException = randomBoolean() ? new ResourceAlreadyExistsException("index already exists") : null; final var blockingLatch = new CountDownLatch(1); try { bulkAction.beforeIndexCreation = () -> blockWriteThreadPool(blockingLatch); @@ -467,6 +485,76 @@ public void testResolveFailureStoreFromTemplate() throws Exception { assertThat(TransportBulkAction.resolveFailureInternal(indexTemplate + "-1", metadata, testTime), is(nullValue())); } + /** + * This test asserts that any failing prerequisite action that fails (i.e. index creation or data stream/failure store rollover) + * results in a failed response. + */ + public void testFailuresDuringPrerequisiteActions() throws InterruptedException { + // One request for testing a failure during index creation. + BulkRequest bulkRequest = new BulkRequest().add(new IndexRequest("index").source(Map.of())) + // One request for testing a failure during data stream rollover. + .add(new IndexRequest("data-stream").source(Map.of())) + // One request for testing a failure during failure store rollover. + .add(new IndexRequest("failure-store").source(Map.of()).setWriteToFailureStore(true)); + + // Construct a cluster state that contains the required data streams. + var state = clusterService.state() + .copyAndUpdateMetadata( + builder -> builder.put(indexMetadata(".ds-data-stream-01")) + .put(indexMetadata(".ds-failure-store-01")) + .put(indexMetadata(".fs-failure-store-01")) + .put( + DataStream.builder( + "data-stream", + DataStream.DataStreamIndices.backingIndicesBuilder(List.of(new Index(".ds-data-stream-01", randomUUID()))) + .setRolloverOnWrite(true) + .build() + ).build() + ) + .put( + DataStream.builder("failure-store", List.of(new Index(".ds-failure-store-01", randomUUID()))) + .setFailureIndices( + DataStream.DataStreamIndices.failureIndicesBuilder(List.of(new Index(".fs-failure-store-01", randomUUID()))) + .setRolloverOnWrite(true) + .build() + ) + .build() + ) + ); + + // Apply the cluster state. + CountDownLatch latch = new CountDownLatch(1); + clusterService.getClusterApplierService().onNewClusterState("set-state", () -> state, ActionListener.running(latch::countDown)); + // And wait for it to be applied. + latch.await(10L, TimeUnit.SECONDS); + + // Set the exceptions that the transport action should encounter. + bulkAction.failIndexCreationException = new IndexNotFoundException("index"); + bulkAction.failDataStreamRolloverException = new RuntimeException("data-stream-rollover-exception"); + bulkAction.failFailureStoreRolloverException = new RuntimeException("failure-store-rollover-exception"); + + // Execute the action and get the response. + PlainActionFuture future = new PlainActionFuture<>(); + ActionTestUtils.execute(bulkAction, null, bulkRequest, future); + BulkResponse response = future.actionGet(); + assertEquals(3, response.getItems().length); + + var indexFailure = response.getItems()[0]; + assertTrue(indexFailure.isFailed()); + assertTrue(indexFailure.getFailure().getCause() instanceof IndexNotFoundException); + assertNull(bulkRequest.requests.get(0)); + + var dataStreamFailure = response.getItems()[1]; + assertTrue(dataStreamFailure.isFailed()); + assertEquals("data-stream-rollover-exception", dataStreamFailure.getFailure().getCause().getMessage()); + assertNull(bulkRequest.requests.get(1)); + + var failureStoreFailure = response.getItems()[2]; + assertTrue(failureStoreFailure.isFailed()); + assertEquals("failure-store-rollover-exception", failureStoreFailure.getFailure().getCause().getMessage()); + assertNull(bulkRequest.requests.get(2)); + } + private BulkRequest buildBulkRequest(List indices) { BulkRequest request = new BulkRequest(); for (String index : indices) { @@ -488,4 +576,8 @@ private BulkRequest buildBulkStreamRequest(List indices) throws IOExcept StreamInput streamInput = out.bytes().streamInput(); return (new BulkRequest(streamInput)); } + + private static IndexMetadata.Builder indexMetadata(String index) { + return IndexMetadata.builder(index).settings(settings(IndexVersion.current())).numberOfShards(1).numberOfReplicas(1); + } } diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java index 7f15fd0ec2582..b3d3ebe5e1357 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java @@ -29,7 +29,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.AtomicArray; import org.elasticsearch.common.util.concurrent.ThreadContext; -import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.IndexingPressure; import org.elasticsearch.indices.EmptySystemIndices; @@ -50,7 +49,6 @@ import java.nio.charset.StandardCharsets; import java.util.Collections; import java.util.HashSet; -import java.util.Map; import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; @@ -143,11 +141,10 @@ void executeBulk( long startTimeNanos, ActionListener listener, Executor executor, - AtomicArray responses, - Map indicesThatCannotBeCreated + AtomicArray responses ) { expected.set(1000000); - super.executeBulk(task, bulkRequest, startTimeNanos, listener, executor, responses, indicesThatCannotBeCreated); + super.executeBulk(task, bulkRequest, startTimeNanos, listener, executor, responses); } }; } else { @@ -168,12 +165,11 @@ void executeBulk( long startTimeNanos, ActionListener listener, Executor executor, - AtomicArray responses, - Map indicesThatCannotBeCreated + AtomicArray responses ) { long elapsed = spinForAtLeastOneMillisecond(); expected.set(elapsed); - super.executeBulk(task, bulkRequest, startTimeNanos, listener, executor, responses, indicesThatCannotBeCreated); + super.executeBulk(task, bulkRequest, startTimeNanos, listener, executor, responses); } }; } From 457d99a7cc98c0804e256215deb09409bf194371 Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Thu, 26 Sep 2024 08:59:40 -0600 Subject: [PATCH 16/43] Allow configuring `ignore_dynamic_beyond_limit` in Serverless (#113548) This setting is already set for the Serverless templates, however, it's not marked as available on Serverless, so it's ending up ignored. This allows the setting. Co-authored-by: Elastic Machine --- .../java/org/elasticsearch/index/mapper/MapperService.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java index a7e1b3e122060..08461525526b9 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -125,7 +125,8 @@ public boolean isAutoUpdate() { "index.mapping.total_fields.ignore_dynamic_beyond_limit", false, Property.Dynamic, - Property.IndexScope + Property.IndexScope, + Property.ServerlessPublic ); public static final Setting INDEX_MAPPING_DEPTH_LIMIT_SETTING = Setting.longSetting( "index.mapping.depth.limit", From 0ea8a78ca7692c59615be11109ef3412f8690917 Mon Sep 17 00:00:00 2001 From: Stef Nestor <26751266+stefnestor@users.noreply.github.com> Date: Thu, 26 Sep 2024 09:05:21 -0600 Subject: [PATCH 17/43] (Doc+) Avoid search pile up by setting default timeout (#112846) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 👋! Mini doc PR to say can avoid search task pile-ups by setting [`search.default_search_timeout`](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-your-data.html#search-timeout) under [High JVM > avoid expensive searches](https://www.elastic.co/guide/en/elasticsearch/reference/master/high-jvm-memory-pressure.html#reduce-jvm-memory-pressure). --- .../common-issues/high-jvm-memory-pressure.asciidoc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/reference/troubleshooting/common-issues/high-jvm-memory-pressure.asciidoc b/docs/reference/troubleshooting/common-issues/high-jvm-memory-pressure.asciidoc index 267d6594b8025..3469a0ca5bf42 100644 --- a/docs/reference/troubleshooting/common-issues/high-jvm-memory-pressure.asciidoc +++ b/docs/reference/troubleshooting/common-issues/high-jvm-memory-pressure.asciidoc @@ -66,6 +66,8 @@ searches, consider the following setting changes: <> cluster setting. +* Set a default search timeout using the <> cluster setting. + [source,console] ---- PUT _settings From ade235bf0f032926ab611a17d553160bc2f254a9 Mon Sep 17 00:00:00 2001 From: David Turner Date: Thu, 26 Sep 2024 16:10:39 +0100 Subject: [PATCH 18/43] Wait for logs in tests without busy-waiting (#113589) Introduces `MockLog#awaitAllExpectationsMatched` to allow tests to wait until all the expected log messages have been seen without having to use `assertBusy()`. --- .../bootstrap/SpawnerNoBootstrapTests.java | 74 ++++-------- .../common/network/ThreadWatchdogIT.java | 42 +++---- .../single/SingleNodeDiscoveryIT.java | 2 +- .../indices/cluster/ShardLockFailureIT.java | 16 ++- .../search/fieldcaps/FieldCapabilitiesIT.java | 2 +- .../service/ClusterApplierServiceTests.java | 2 +- .../cluster/service/MasterServiceTests.java | 2 +- .../index/shard/IndexShardTests.java | 6 +- .../monitor/fs/FsHealthServiceTests.java | 2 +- .../tasks/BanFailureLoggingTests.java | 4 +- .../threadpool/ThreadPoolTests.java | 4 +- .../java/org/elasticsearch/test/MockLog.java | 108 +++++++++++++++--- .../AbstractSimpleTransportTestCase.java | 6 +- .../org/elasticsearch/test/MockLogTests.java | 63 +++++++++- .../SnapshotBasedIndexRecoveryIT.java | 2 +- 15 files changed, 224 insertions(+), 111 deletions(-) diff --git a/qa/no-bootstrap-tests/src/test/java/org/elasticsearch/bootstrap/SpawnerNoBootstrapTests.java b/qa/no-bootstrap-tests/src/test/java/org/elasticsearch/bootstrap/SpawnerNoBootstrapTests.java index 32416b0d0d77d..168493eb52f60 100644 --- a/qa/no-bootstrap-tests/src/test/java/org/elasticsearch/bootstrap/SpawnerNoBootstrapTests.java +++ b/qa/no-bootstrap-tests/src/test/java/org/elasticsearch/bootstrap/SpawnerNoBootstrapTests.java @@ -13,7 +13,6 @@ import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.core.LogEvent; import org.apache.lucene.tests.util.LuceneTestCase; import org.apache.lucene.util.Constants; import org.elasticsearch.Version; @@ -36,7 +35,6 @@ import java.util.HashSet; import java.util.List; import java.util.Set; -import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.function.Function; @@ -70,34 +68,6 @@ public class SpawnerNoBootstrapTests extends LuceneTestCase { MockLog.init(); } - static class ExpectedStreamMessage implements MockLog.LoggingExpectation { - final String expectedLogger; - final String expectedMessage; - final CountDownLatch matched; - volatile boolean saw; - - ExpectedStreamMessage(String logger, String message, CountDownLatch matched) { - this.expectedLogger = logger; - this.expectedMessage = message; - this.matched = matched; - } - - @Override - public void match(LogEvent event) { - if (event.getLoggerName().equals(expectedLogger) - && event.getLevel().equals(Level.WARN) - && event.getMessage().getFormattedMessage().equals(expectedMessage)) { - saw = true; - matched.countDown(); - } - } - - @Override - public void assertMatched() { - assertTrue("Expected to see message [" + expectedMessage + "] on logger [" + expectedLogger + "]", saw); - } - } - /** * Simplest case: a module with no controller daemon. */ @@ -209,32 +179,32 @@ private void assertControllerSpawns(final Function pluginsDir String stderrLoggerName = "test_plugin-controller-stderr"; Loggers.setLevel(LogManager.getLogger(stdoutLoggerName), Level.TRACE); Loggers.setLevel(LogManager.getLogger(stderrLoggerName), Level.TRACE); - CountDownLatch messagesLoggedLatch = new CountDownLatch(2); - try (var mockLog = MockLog.capture(stdoutLoggerName, stderrLoggerName)) { - if (expectSpawn) { - mockLog.addExpectation(new ExpectedStreamMessage(stdoutLoggerName, "I am alive", messagesLoggedLatch)); - mockLog.addExpectation(new ExpectedStreamMessage(stderrLoggerName, "I am an error", messagesLoggedLatch)); + if (expectSpawn) { + final Process process; + try (var mockLog = MockLog.capture(stdoutLoggerName, stderrLoggerName)) { + mockLog.addExpectation(new MockLog.SeenEventExpectation("stdout", stdoutLoggerName, Level.WARN, "I am alive")); + mockLog.addExpectation(new MockLog.SeenEventExpectation("stderr", stderrLoggerName, Level.WARN, "I am an error")); + + try (var spawner = new Spawner()) { + spawner.spawnNativeControllers(environment); + List processes = spawner.getProcesses(); + + // as there should only be a reference in the list for the module that had the controller daemon, we expect one here + assertThat(processes, hasSize(1)); + process = processes.get(0); + // fail if we don't get the expected log messages soonish + mockLog.awaitAllExpectationsMatched(); + } } - Spawner spawner = new Spawner(); - spawner.spawnNativeControllers(environment); - - List processes = spawner.getProcesses(); - - if (expectSpawn) { - // as there should only be a reference in the list for the module that had the controller daemon, we expect one here - assertThat(processes, hasSize(1)); - Process process = processes.get(0); - // fail if we don't get the expected log messages within one second; usually it will be even quicker - assertTrue(messagesLoggedLatch.await(1, TimeUnit.SECONDS)); - spawner.close(); - // fail if the process does not die within one second; usually it will be even quicker but it depends on OS scheduling - assertTrue(process.waitFor(1, TimeUnit.SECONDS)); - } else { - assertThat(processes, is(empty())); + // fail if the process does not die within one second; usually it will be even quicker but it depends on OS scheduling + assertTrue(process.waitFor(1, TimeUnit.SECONDS)); + } else { + try (var spawner = new Spawner()) { + spawner.spawnNativeControllers(environment); + assertThat(spawner.getProcesses(), is(empty())); } - mockLog.assertAllExpectationsMatched(); } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/common/network/ThreadWatchdogIT.java b/server/src/internalClusterTest/java/org/elasticsearch/common/network/ThreadWatchdogIT.java index 6d3ed45f4015a..f2441e43de8d8 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/common/network/ThreadWatchdogIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/common/network/ThreadWatchdogIT.java @@ -9,7 +9,7 @@ package org.elasticsearch.common.network; -import org.apache.logging.log4j.core.LogEvent; +import org.apache.logging.log4j.Level; import org.elasticsearch.action.ActionListenerResponseHandler; import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.client.Request; @@ -23,7 +23,6 @@ import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.common.util.CollectionUtils; import org.elasticsearch.common.util.concurrent.EsExecutors; -import org.elasticsearch.common.util.concurrent.RunOnce; import org.elasticsearch.features.NodeFeature; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; @@ -43,7 +42,6 @@ import java.io.IOException; import java.util.Collection; import java.util.List; -import java.util.concurrent.CountDownLatch; import java.util.function.Predicate; import java.util.function.Supplier; @@ -103,26 +101,24 @@ public void handleRequest(RestRequest request, RestChannel channel, NodeClient c } private static void blockAndWaitForWatchdogLogs() { - final var threadName = Thread.currentThread().getName(); - final var logsSeenLatch = new CountDownLatch(2); - final var warningSeen = new RunOnce(logsSeenLatch::countDown); - final var threadDumpSeen = new RunOnce(logsSeenLatch::countDown); - MockLog.assertThatLogger(() -> safeAwait(logsSeenLatch), ThreadWatchdog.class, new MockLog.LoggingExpectation() { - @Override - public void match(LogEvent event) { - final var formattedMessage = event.getMessage().getFormattedMessage(); - if (formattedMessage.contains("the following threads are active but did not make progress in the preceding [100ms]:") - && formattedMessage.contains(threadName)) { - warningSeen.run(); - } - if (formattedMessage.contains("hot threads dump due to active threads not making progress")) { - threadDumpSeen.run(); - } - } - - @Override - public void assertMatched() {} - }); + MockLog.awaitLogger( + () -> {}, + ThreadWatchdog.class, + new MockLog.SeenEventExpectation( + "warning", + ThreadWatchdog.class.getCanonicalName(), + Level.WARN, + "*the following threads are active but did not make progress in the preceding [100ms]:*" + + Thread.currentThread().getName() + + "*" + ), + new MockLog.SeenEventExpectation( + "thread dump", + ThreadWatchdog.class.getCanonicalName(), + Level.WARN, + "*hot threads dump due to active threads not making progress*" + ) + ); } public void testThreadWatchdogHttpLogging() throws IOException { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/discovery/single/SingleNodeDiscoveryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/discovery/single/SingleNodeDiscoveryIT.java index 8fd10cdf07310..10f13f6ab152f 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/discovery/single/SingleNodeDiscoveryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/discovery/single/SingleNodeDiscoveryIT.java @@ -160,7 +160,7 @@ public boolean innerMatch(final LogEvent event) { other.beforeTest(random()); final ClusterState first = internalCluster().getInstance(ClusterService.class).state(); assertThat(first.nodes().getSize(), equalTo(1)); - assertBusy(mockLog::assertAllExpectationsMatched); + mockLog.awaitAllExpectationsMatched(); } finally { other.close(); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/cluster/ShardLockFailureIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/cluster/ShardLockFailureIT.java index 9bcd8528acf9e..0ce3ca53e1c1f 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/cluster/ShardLockFailureIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/cluster/ShardLockFailureIT.java @@ -72,9 +72,9 @@ public void testShardLockFailure() throws Exception { var ignored1 = internalCluster().getInstance(NodeEnvironment.class, node).shardLock(shardId, "blocked for test"); var mockLog = MockLog.capture(IndicesClusterStateService.class); ) { - final CountDownLatch countDownLatch = new CountDownLatch(1); mockLog.addExpectation(new MockLog.LoggingExpectation() { + private final CountDownLatch countDownLatch = new CountDownLatch(1); int debugMessagesSeen = 0; int warnMessagesSeen = 0; @@ -101,14 +101,20 @@ public synchronized void match(LogEvent event) { } @Override - public void assertMatched() {} + public void assertMatched() { + fail("unused"); + } + + @Override + public void awaitMatched(long millis) throws InterruptedException { + assertTrue(countDownLatch.await(millis, TimeUnit.MILLISECONDS)); + } }); updateIndexSettings(Settings.builder().putNull(IndexMetadata.INDEX_ROUTING_EXCLUDE_GROUP_PREFIX + "._name"), indexName); ensureYellow(indexName); - assertTrue(countDownLatch.await(30, TimeUnit.SECONDS)); + mockLog.awaitAllExpectationsMatched(); assertEquals(ClusterHealthStatus.YELLOW, clusterAdmin().prepareHealth(TEST_REQUEST_TIMEOUT, indexName).get().getStatus()); - mockLog.assertAllExpectationsMatched(); } ensureGreen(indexName); @@ -153,7 +159,7 @@ public void testShardLockTimeout() throws Exception { ); updateIndexSettings(Settings.builder().putNull(IndexMetadata.INDEX_ROUTING_EXCLUDE_GROUP_PREFIX + "._name"), indexName); - assertBusy(mockLog::assertAllExpectationsMatched); + mockLog.awaitAllExpectationsMatched(); final var clusterHealthResponse = clusterAdmin().prepareHealth(TEST_REQUEST_TIMEOUT, indexName) .setWaitForEvents(Priority.LANGUID) .setTimeout(TimeValue.timeValueSeconds(10)) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java index ac68ff243166b..a754350c8faf7 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java @@ -693,7 +693,7 @@ public void testCancel() throws Exception { } }, 30, TimeUnit.SECONDS); cancellable.cancel(); - assertBusy(mockLog::assertAllExpectationsMatched); + mockLog.awaitAllExpectationsMatched(); logger.info("--> waiting for field-caps tasks to be cancelled"); assertBusy(() -> { List tasks = clusterAdmin().prepareListTasks() diff --git a/server/src/test/java/org/elasticsearch/cluster/service/ClusterApplierServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/service/ClusterApplierServiceTests.java index e37e92d553dcc..7c1c954e7b4e9 100644 --- a/server/src/test/java/org/elasticsearch/cluster/service/ClusterApplierServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/service/ClusterApplierServiceTests.java @@ -181,7 +181,7 @@ public void onFailure(Exception e) { fail(); } }); - assertBusy(mockLog::assertAllExpectationsMatched); + mockLog.awaitAllExpectationsMatched(); } } diff --git a/server/src/test/java/org/elasticsearch/cluster/service/MasterServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/service/MasterServiceTests.java index 498c04c005304..a540649582177 100644 --- a/server/src/test/java/org/elasticsearch/cluster/service/MasterServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/service/MasterServiceTests.java @@ -526,7 +526,7 @@ public void onFailure(Exception e) { fail(); } }); - assertBusy(mockLog::assertAllExpectationsMatched); + mockLog.awaitAllExpectationsMatched(); } } diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java index ec70f1f7adcfd..cddda8a76ae60 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java @@ -4191,7 +4191,7 @@ protected void commitIndexWriter(final IndexWriter writer, final Translog transl ); shard.flushOnIdle(0); assertFalse(shard.isActive()); - assertBusy(mockLog::assertAllExpectationsMatched); + mockLog.awaitAllExpectationsMatched(); // While the first flush is happening, index one more doc (to turn the shard's active flag to true), // and issue a second flushOnIdle request which should not wait for the ongoing flush @@ -4206,7 +4206,7 @@ protected void commitIndexWriter(final IndexWriter writer, final Translog transl ) ); shard.flushOnIdle(0); - assertBusy(mockLog::assertAllExpectationsMatched); + mockLog.awaitAllExpectationsMatched(); // A direct call to flush (with waitIfOngoing=false) should not wait and return false immediately assertFalse(shard.flush(new FlushRequest().waitIfOngoing(false).force(false))); @@ -4223,7 +4223,7 @@ protected void commitIndexWriter(final IndexWriter writer, final Translog transl "released flush lock" ) ); - assertBusy(mockLog::assertAllExpectationsMatched); + mockLog.awaitAllExpectationsMatched(); // The second flushOnIdle (that did not happen) should have turned the active flag to true assertTrue(shard.isActive()); diff --git a/server/src/test/java/org/elasticsearch/monitor/fs/FsHealthServiceTests.java b/server/src/test/java/org/elasticsearch/monitor/fs/FsHealthServiceTests.java index 05b71693a7fea..b644dfbc3a12c 100644 --- a/server/src/test/java/org/elasticsearch/monitor/fs/FsHealthServiceTests.java +++ b/server/src/test/java/org/elasticsearch/monitor/fs/FsHealthServiceTests.java @@ -146,7 +146,7 @@ public void testLoggingOnHungIO() throws Exception { disruptFileSystemProvider.injectIOException.set(true); fsHealthService.new FsHealthMonitor().run(); assertEquals(env.nodeDataPaths().length, disruptFileSystemProvider.getInjectedPathCount()); - assertBusy(mockLog::assertAllExpectationsMatched); + mockLog.awaitAllExpectationsMatched(); } finally { PathUtilsForTesting.teardown(); ThreadPool.terminate(testThreadPool, 500, TimeUnit.MILLISECONDS); diff --git a/server/src/test/java/org/elasticsearch/tasks/BanFailureLoggingTests.java b/server/src/test/java/org/elasticsearch/tasks/BanFailureLoggingTests.java index 276099e01c9f8..78d76476d06fc 100644 --- a/server/src/test/java/org/elasticsearch/tasks/BanFailureLoggingTests.java +++ b/server/src/test/java/org/elasticsearch/tasks/BanFailureLoggingTests.java @@ -184,8 +184,8 @@ public Task createTask(long id, String type, String action, TaskId parentTaskId, // acceptable; we mostly ignore the result of cancellation anyway } - // assert busy since failure to remove a ban may be logged after cancellation completed - assertBusy(mockLog::assertAllExpectationsMatched); + // await since failure to remove a ban may be logged after cancellation completed + mockLog.awaitAllExpectationsMatched(); } assertTrue("child tasks did not finish in time", childTaskLock.tryLock(15, TimeUnit.SECONDS)); diff --git a/server/src/test/java/org/elasticsearch/threadpool/ThreadPoolTests.java b/server/src/test/java/org/elasticsearch/threadpool/ThreadPoolTests.java index 458eeb900071d..310cf467a8391 100644 --- a/server/src/test/java/org/elasticsearch/threadpool/ThreadPoolTests.java +++ b/server/src/test/java/org/elasticsearch/threadpool/ThreadPoolTests.java @@ -128,7 +128,7 @@ public void testTimerThreadWarningLogging() throws Exception { final ThreadPool.CachedTimeThread thread = new ThreadPool.CachedTimeThread("[timer]", 200, 100); thread.start(); - assertBusy(mockLog::assertAllExpectationsMatched); + mockLog.awaitAllExpectationsMatched(); thread.interrupt(); thread.join(); @@ -297,7 +297,7 @@ public String toString() { } }; threadPool.schedule(runnable, TimeValue.timeValueMillis(randomLongBetween(0, 300)), EsExecutors.DIRECT_EXECUTOR_SERVICE); - assertBusy(mockLog::assertAllExpectationsMatched); + mockLog.awaitAllExpectationsMatched(); } finally { assertTrue(terminate(threadPool)); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/MockLog.java b/test/framework/src/main/java/org/elasticsearch/test/MockLog.java index 57acca08c23e8..4a012bb361e65 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/MockLog.java +++ b/test/framework/src/main/java/org/elasticsearch/test/MockLog.java @@ -16,6 +16,7 @@ import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.core.Releasable; +import org.elasticsearch.core.TimeValue; import java.util.Arrays; import java.util.List; @@ -23,10 +24,13 @@ import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import java.util.regex.Pattern; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.is; /** @@ -112,10 +116,46 @@ public void assertAllExpectationsMatched() { } } + public void awaitAllExpectationsMatched() { + awaitAllExpectationsMatched(ESTestCase.SAFE_AWAIT_TIMEOUT); + } + + // exposed for testing + void awaitAllExpectationsMatched(TimeValue waitTime) { + final var deadlineNanos = System.nanoTime() + waitTime.nanos(); + final var nanosPerMilli = TimeValue.timeValueMillis(1).nanos(); + try { + for (LoggingExpectation expectation : expectations) { + final var remainingMillis = (deadlineNanos - System.nanoTime() + nanosPerMilli - 1) / nanosPerMilli; // round up + assertThat(remainingMillis, greaterThan(0L)); + expectation.awaitMatched(remainingMillis); + } + } catch (InterruptedException interruptedException) { + Thread.currentThread().interrupt(); + throw new AssertionError("interrupted", interruptedException); + } + } + + /** + * Keeps track of whether the {@link LogEvent} instances it receives match the expected content. + */ public interface LoggingExpectation { + /** + * Called on every {@link LogEvent} received by the captured appenders. + */ void match(LogEvent event); + /** + * Returns if this expectation is matched, otherwise throws an {@link AssertionError}. + */ void assertMatched(); + + /** + * Returns if this expectation is matched within the given number of milliseconds, otherwise throws an {@link AssertionError}. + */ + default void awaitMatched(long millis) throws InterruptedException { + assertMatched(); + } } public abstract static class AbstractEventExpectation implements LoggingExpectation { @@ -123,14 +163,13 @@ public abstract static class AbstractEventExpectation implements LoggingExpectat protected final String logger; protected final Level level; protected final String message; - volatile boolean saw; + protected final CountDownLatch seenLatch = new CountDownLatch(1); public AbstractEventExpectation(String name, String logger, Level level, String message) { this.name = name; this.logger = logger; this.level = level; this.message = message; - this.saw = false; } @Override @@ -138,11 +177,11 @@ public void match(LogEvent event) { if (event.getLevel().equals(level) && event.getLoggerName().equals(logger) && innerMatch(event)) { if (Regex.isSimpleMatchPattern(message)) { if (Regex.simpleMatch(message, event.getMessage().getFormattedMessage())) { - saw = true; + seenLatch.countDown(); } } else { if (event.getMessage().getFormattedMessage().contains(message)) { - saw = true; + seenLatch.countDown(); } } } @@ -162,7 +201,7 @@ public UnseenEventExpectation(String name, String logger, Level level, String me @Override public void assertMatched() { - assertThat("expected not to see " + name + " but did", saw, equalTo(false)); + assertThat("expected not to see " + name + " but did", seenLatch.getCount(), equalTo(1L)); } } @@ -174,7 +213,12 @@ public SeenEventExpectation(String name, String logger, Level level, String mess @Override public void assertMatched() { - assertThat("expected to see " + name + " but did not", saw, equalTo(true)); + assertThat("expected to see " + name + " but did not", seenLatch.getCount(), equalTo(0L)); + } + + @Override + public void awaitMatched(long millis) throws InterruptedException { + assertThat("expected to see " + name + " but did not", seenLatch.await(millis, TimeUnit.MILLISECONDS), equalTo(true)); } } @@ -195,7 +239,17 @@ public void assertMatched() { if (expectSeen) { super.assertMatched(); } else { - assertThat("expected not to see " + name + " yet but did", saw, equalTo(false)); + assertThat("expected not to see " + name + " yet but did", seenLatch.getCount(), equalTo(1L)); + } + } + + @Override + public void awaitMatched(long millis) throws InterruptedException { + if (expectSeen) { + super.awaitMatched(millis); + } else { + // do not wait for negative expectation + assertThat("expected not to see " + name + " yet but did", seenLatch.getCount(), equalTo(1L)); } } } @@ -229,11 +283,11 @@ public boolean innerMatch(final LogEvent event) { public static class PatternSeenEventExpectation implements LoggingExpectation { - protected final String name; - protected final String logger; - protected final Level level; - protected final Pattern pattern; - volatile boolean saw; + private final String name; + private final String logger; + private final Level level; + private final Pattern pattern; + private final CountDownLatch seenLatch = new CountDownLatch(1); public PatternSeenEventExpectation(String name, String logger, Level level, String pattern) { this.name = name; @@ -246,16 +300,20 @@ public PatternSeenEventExpectation(String name, String logger, Level level, Stri public void match(LogEvent event) { if (event.getLevel().equals(level) && event.getLoggerName().equals(logger)) { if (pattern.matcher(event.getMessage().getFormattedMessage()).matches()) { - saw = true; + seenLatch.countDown(); } } } @Override public void assertMatched() { - assertThat(name, saw, equalTo(true)); + assertThat(name, seenLatch.getCount(), equalTo(0L)); } + @Override + public void awaitMatched(long millis) throws InterruptedException { + assertThat(name, seenLatch.await(millis, TimeUnit.MILLISECONDS), equalTo(true)); + } } /** @@ -284,6 +342,15 @@ public void assertMatched() { } } + @Override + public void awaitMatched(long millis) throws InterruptedException { + try { + delegate.awaitMatched(millis); + } finally { + assertMatchedCalled = true; + } + } + @Override public String toString() { return delegate.toString(); @@ -336,4 +403,17 @@ public static void assertThatLogger(Runnable action, Class loggerOwner, MockL mockLog.assertAllExpectationsMatched(); } } + + /** + * Executes an action and waits until the given logging expectations are satisfied. + */ + public static void awaitLogger(Runnable action, Class loggerOwner, MockLog.LoggingExpectation... expectations) { + try (var mockLog = MockLog.capture(loggerOwner)) { + for (var expectation : expectations) { + mockLog.addExpectation(expectation); + } + action.run(); + mockLog.awaitAllExpectationsMatched(); + } + } } diff --git a/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java b/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java index ba7aa9977b917..fe3de2218a493 100644 --- a/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java @@ -1373,7 +1373,7 @@ public void handleException(TransportException exp) {} serviceA.sendRequest(nodeB, "internal:test", new StringMessageRequest("", 10), noopResponseHandler); - assertBusy(mockLog::assertAllExpectationsMatched); + mockLog.awaitAllExpectationsMatched(); //////////////////////////////////////////////////////////////////////// // tests for included action type "internal:testError" which returns an error @@ -1420,7 +1420,7 @@ public void handleException(TransportException exp) {} serviceA.sendRequest(nodeB, "internal:testError", new StringMessageRequest(""), noopResponseHandler); - assertBusy(mockLog::assertAllExpectationsMatched); + mockLog.awaitAllExpectationsMatched(); //////////////////////////////////////////////////////////////////////// // tests for excluded action type "internal:testNotSeen" @@ -1467,7 +1467,7 @@ public void handleException(TransportException exp) {} submitRequest(serviceA, nodeB, "internal:testNotSeen", new StringMessageRequest(""), noopResponseHandler).get(); - assertBusy(mockLog::assertAllExpectationsMatched); + mockLog.awaitAllExpectationsMatched(); } } diff --git a/test/framework/src/test/java/org/elasticsearch/test/MockLogTests.java b/test/framework/src/test/java/org/elasticsearch/test/MockLogTests.java index 2720da55544d1..89d23c3b345b9 100644 --- a/test/framework/src/test/java/org/elasticsearch/test/MockLogTests.java +++ b/test/framework/src/test/java/org/elasticsearch/test/MockLogTests.java @@ -9,8 +9,11 @@ package org.elasticsearch.test; +import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.test.junit.annotations.TestLogging; import java.util.concurrent.atomic.AtomicBoolean; @@ -27,7 +30,7 @@ public void testConcurrentLogAndLifecycle() throws Exception { logThread.start(); for (int i = 0; i < 1000; i++) { - try (var mockLog = MockLog.capture(MockLogTests.class)) { + try (var ignored = MockLog.capture(MockLogTests.class)) { Thread.yield(); } } @@ -35,4 +38,62 @@ public void testConcurrentLogAndLifecycle() throws Exception { keepGoing.set(false); logThread.join(); } + + @TestLogging(reason = "checking log behaviour", value = "org.elasticsearch.test.MockLogTests:INFO") + public void testAwaitUnseenEvent() { + try (var mockLog = MockLog.capture(MockLogTests.class)) { + mockLog.addExpectation( + new MockLog.UnseenEventExpectation("unseen", MockLogTests.class.getCanonicalName(), Level.INFO, "unexpected") + ); + Thread.currentThread().interrupt(); // ensures no blocking calls + mockLog.awaitAllExpectationsMatched(); + mockLog.assertAllExpectationsMatched(); + + logger.info("unexpected"); + expectThrows(AssertionError.class, mockLog::awaitAllExpectationsMatched); + expectThrows(AssertionError.class, mockLog::assertAllExpectationsMatched); + + assertTrue(Thread.interrupted()); // clear interrupt flag again + } + } + + @TestLogging(reason = "checking log behaviour", value = "org.elasticsearch.test.MockLogTests:INFO") + public void testAwaitSeenEvent() throws InterruptedException { + try (var mockLog = MockLog.capture(MockLogTests.class)) { + mockLog.addExpectation(new MockLog.SeenEventExpectation("seen", MockLogTests.class.getCanonicalName(), Level.INFO, "expected")); + + expectThrows(AssertionError.class, () -> mockLog.awaitAllExpectationsMatched(TimeValue.timeValueMillis(10))); + expectThrows(AssertionError.class, mockLog::assertAllExpectationsMatched); + + final var logThread = new Thread(() -> { + logger.info("expected"); + mockLog.assertAllExpectationsMatched(); + }); + logThread.start(); + mockLog.awaitAllExpectationsMatched(); + mockLog.assertAllExpectationsMatched(); + logThread.join(); + } + } + + @TestLogging(reason = "checking log behaviour", value = "org.elasticsearch.test.MockLogTests:INFO") + public void testAwaitPatternEvent() throws InterruptedException { + try (var mockLog = MockLog.capture(MockLogTests.class)) { + mockLog.addExpectation( + new MockLog.PatternSeenEventExpectation("seen", MockLogTests.class.getCanonicalName(), Level.INFO, ".*expected.*") + ); + + expectThrows(AssertionError.class, () -> mockLog.awaitAllExpectationsMatched(TimeValue.timeValueMillis(10))); + expectThrows(AssertionError.class, mockLog::assertAllExpectationsMatched); + + final var logThread = new Thread(() -> { + logger.info("blah blah expected blah blah"); + mockLog.assertAllExpectationsMatched(); + }); + logThread.start(); + mockLog.awaitAllExpectationsMatched(); + mockLog.assertAllExpectationsMatched(); + logThread.join(); + } + } } diff --git a/x-pack/plugin/snapshot-based-recoveries/src/internalClusterTest/java/org/elasticsearch/xpack/snapshotbasedrecoveries/recovery/SnapshotBasedIndexRecoveryIT.java b/x-pack/plugin/snapshot-based-recoveries/src/internalClusterTest/java/org/elasticsearch/xpack/snapshotbasedrecoveries/recovery/SnapshotBasedIndexRecoveryIT.java index d1eaff1bef1b2..df8dc54bb7490 100644 --- a/x-pack/plugin/snapshot-based-recoveries/src/internalClusterTest/java/org/elasticsearch/xpack/snapshotbasedrecoveries/recovery/SnapshotBasedIndexRecoveryIT.java +++ b/x-pack/plugin/snapshot-based-recoveries/src/internalClusterTest/java/org/elasticsearch/xpack/snapshotbasedrecoveries/recovery/SnapshotBasedIndexRecoveryIT.java @@ -623,7 +623,7 @@ public void testRecoveryIsCancelledAfterDeletingTheIndex() throws Exception { assertAcked(indicesAdmin().prepareDelete(indexName).get()); - assertBusy(mockLog::assertAllExpectationsMatched); + mockLog.awaitAllExpectationsMatched(); } respondToRecoverSnapshotFile.countDown(); From 122e7288200ee03e9087c98dff6cebbc94e774aa Mon Sep 17 00:00:00 2001 From: Mark Tozzi Date: Thu, 26 Sep 2024 12:03:01 -0400 Subject: [PATCH 19/43] [ESQL] Add TO_DATE_NANOS conversion function (#112150) Resolves #111842 This adds a conversion function that yields DATE_NANOS. Mostly this is straight forward. It is worth noting that when converting a millisecond date into a nanosecond date, the conversion function truncates it to 0 nanoseconds (i.e. first nanosecond of that millisecond). This is, of course, a bit of an assumption, but I don't have a better assumption we can make. I'd thought about adding a second, optional, parameter to control this behavior, but it's important that TO_DATE_NANOS extend AbstractConvertFunction, which itself extends UnaryScalarFunction, so that it will work correctly with union types. Also, it's unlikely the user will have any better guess than we do for filling in the nanoseconds. Making that assumption does, however, create some weirdness. Consider two comparisons: TO_DATETIME("2023-03-23T12:15:03.360103847") == TO_DATETIME("2023-03-23T12:15:03.360") will return true while TO_DATE_NANOS("2023-03-23T12:15:03.360103847") == TO_DATE_NANOS("2023-03-23T12:15:03.360") will return false. This is akin to casting between longs and doubles, where things may compare equal in one type that are not equal in the other. This seems fine, and I can't think of a better way to do it, but it's worth being aware of. --------- Co-authored-by: Elastic Machine --- .../description/to_date_nanos.asciidoc | 7 + .../kibana/definition/to_date_nanos.json | 9 + .../esql/functions/kibana/docs/mv_avg.md | 2 +- .../esql/functions/kibana/docs/mv_sum.md | 2 +- .../functions/kibana/docs/to_date_nanos.md | 8 + .../esql/functions/kibana/inline_cast.json | 1 + .../functions/layout/to_date_nanos.asciidoc | 16 ++ .../parameters/to_date_nanos.asciidoc | 6 + .../esql/functions/signature/categorize.svg | 2 +- .../esql/functions/signature/qstr.svg | 2 +- .../functions/signature/to_date_nanos.svg | 1 + .../functions/types/to_date_nanos.asciidoc | 9 + .../src/main/resources/date_nanos.csv | 18 +- .../src/main/resources/date_nanos.csv-spec | 193 ++++++++++++++++++ .../main/resources/mapping-date_nanos.json | 3 + .../src/main/resources/meta.csv-spec | 10 +- .../ToDateNanosFromDatetimeEvaluator.java | 122 +++++++++++ .../ToDateNanosFromDoubleEvaluator.java | 124 +++++++++++ .../convert/ToDateNanosFromLongEvaluator.java | 122 +++++++++++ .../ToDateNanosFromStringEvaluator.java | 126 ++++++++++++ .../xpack/esql/action/EsqlCapabilities.java | 5 + .../function/EsqlFunctionRegistry.java | 2 + .../function/scalar/UnaryScalarFunction.java | 2 + .../function/scalar/convert/ToDateNanos.java | 134 ++++++++++++ .../esql/type/EsqlDataTypeConverter.java | 6 +- .../expression/function/TestCaseSupplier.java | 83 ++++++-- .../scalar/convert/ToDateNanosTests.java | 135 ++++++++++++ 27 files changed, 1115 insertions(+), 35 deletions(-) create mode 100644 docs/reference/esql/functions/description/to_date_nanos.asciidoc create mode 100644 docs/reference/esql/functions/kibana/definition/to_date_nanos.json create mode 100644 docs/reference/esql/functions/kibana/docs/to_date_nanos.md create mode 100644 docs/reference/esql/functions/layout/to_date_nanos.asciidoc create mode 100644 docs/reference/esql/functions/parameters/to_date_nanos.asciidoc create mode 100644 docs/reference/esql/functions/signature/to_date_nanos.svg create mode 100644 docs/reference/esql/functions/types/to_date_nanos.asciidoc create mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosFromDatetimeEvaluator.java create mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosFromDoubleEvaluator.java create mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosFromLongEvaluator.java create mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosFromStringEvaluator.java create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanos.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosTests.java diff --git a/docs/reference/esql/functions/description/to_date_nanos.asciidoc b/docs/reference/esql/functions/description/to_date_nanos.asciidoc new file mode 100644 index 0000000000000..3fac7295f1bed --- /dev/null +++ b/docs/reference/esql/functions/description/to_date_nanos.asciidoc @@ -0,0 +1,7 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Description* + +Converts an input to a nanosecond-resolution date value (aka date_nanos). + +NOTE: The range for date nanos is 1970-01-01T00:00:00.000000000Z to 2262-04-11T23:47:16.854775807Z. Additionally, integers cannot be converted into date nanos, as the range of integer nanoseconds only covers about 2 seconds after epoch. diff --git a/docs/reference/esql/functions/kibana/definition/to_date_nanos.json b/docs/reference/esql/functions/kibana/definition/to_date_nanos.json new file mode 100644 index 0000000000000..bafbcf2bc2038 --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/to_date_nanos.json @@ -0,0 +1,9 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "eval", + "name" : "to_date_nanos", + "description" : "Converts an input to a nanosecond-resolution date value (aka date_nanos).", + "note" : "The range for date nanos is 1970-01-01T00:00:00.000000000Z to 2262-04-11T23:47:16.854775807Z. Additionally, integers cannot be converted into date nanos, as the range of integer nanoseconds only covers about 2 seconds after epoch.", + "signatures" : [ ], + "preview" : true +} diff --git a/docs/reference/esql/functions/kibana/docs/mv_avg.md b/docs/reference/esql/functions/kibana/docs/mv_avg.md index c5163f36129bf..c3d7e5423f724 100644 --- a/docs/reference/esql/functions/kibana/docs/mv_avg.md +++ b/docs/reference/esql/functions/kibana/docs/mv_avg.md @@ -3,7 +3,7 @@ This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../READ --> ### MV_AVG -Converts a multivalued field into a single valued field containing the average of all the values. +Converts a multivalued field into a single valued field containing the average of all of the values. ``` ROW a=[3, 5, 1, 6] diff --git a/docs/reference/esql/functions/kibana/docs/mv_sum.md b/docs/reference/esql/functions/kibana/docs/mv_sum.md index 987017b34b743..16285d3c7229b 100644 --- a/docs/reference/esql/functions/kibana/docs/mv_sum.md +++ b/docs/reference/esql/functions/kibana/docs/mv_sum.md @@ -3,7 +3,7 @@ This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../READ --> ### MV_SUM -Converts a multivalued field into a single valued field containing the sum of all the values. +Converts a multivalued field into a single valued field containing the sum of all of the values. ``` ROW a=[3, 5, 6] diff --git a/docs/reference/esql/functions/kibana/docs/to_date_nanos.md b/docs/reference/esql/functions/kibana/docs/to_date_nanos.md new file mode 100644 index 0000000000000..0294802485ccb --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/to_date_nanos.md @@ -0,0 +1,8 @@ + + +### TO_DATE_NANOS +Converts an input to a nanosecond-resolution date value (aka date_nanos). + +Note: The range for date nanos is 1970-01-01T00:00:00.000000000Z to 2262-04-11T23:47:16.854775807Z. Additionally, integers cannot be converted into date nanos, as the range of integer nanoseconds only covers about 2 seconds after epoch. diff --git a/docs/reference/esql/functions/kibana/inline_cast.json b/docs/reference/esql/functions/kibana/inline_cast.json index f1aa283c52e95..81a1966773238 100644 --- a/docs/reference/esql/functions/kibana/inline_cast.json +++ b/docs/reference/esql/functions/kibana/inline_cast.json @@ -3,6 +3,7 @@ "boolean" : "to_boolean", "cartesian_point" : "to_cartesianpoint", "cartesian_shape" : "to_cartesianshape", + "date_nanos" : "to_date_nanos", "date_period" : "to_dateperiod", "datetime" : "to_datetime", "double" : "to_double", diff --git a/docs/reference/esql/functions/layout/to_date_nanos.asciidoc b/docs/reference/esql/functions/layout/to_date_nanos.asciidoc new file mode 100644 index 0000000000000..977a0ac969e5d --- /dev/null +++ b/docs/reference/esql/functions/layout/to_date_nanos.asciidoc @@ -0,0 +1,16 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +[discrete] +[[esql-to_date_nanos]] +=== `TO_DATE_NANOS` + +preview::["Do not use on production environments. This functionality is in technical preview and may be changed or removed in a future release. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features."] + +*Syntax* + +[.text-center] +image::esql/functions/signature/to_date_nanos.svg[Embedded,opts=inline] + +include::../parameters/to_date_nanos.asciidoc[] +include::../description/to_date_nanos.asciidoc[] +include::../types/to_date_nanos.asciidoc[] diff --git a/docs/reference/esql/functions/parameters/to_date_nanos.asciidoc b/docs/reference/esql/functions/parameters/to_date_nanos.asciidoc new file mode 100644 index 0000000000000..224f474fa64e3 --- /dev/null +++ b/docs/reference/esql/functions/parameters/to_date_nanos.asciidoc @@ -0,0 +1,6 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Parameters* + +`field`:: +Input value. The input can be a single- or multi-valued column or an expression. diff --git a/docs/reference/esql/functions/signature/categorize.svg b/docs/reference/esql/functions/signature/categorize.svg index 3f36f10382840..c52fd1763eea1 100644 --- a/docs/reference/esql/functions/signature/categorize.svg +++ b/docs/reference/esql/functions/signature/categorize.svg @@ -1 +1 @@ -CATEGORIZE(field) +CATEGORIZE(field) \ No newline at end of file diff --git a/docs/reference/esql/functions/signature/qstr.svg b/docs/reference/esql/functions/signature/qstr.svg index 0d3841b071cef..fb6114822ae63 100644 --- a/docs/reference/esql/functions/signature/qstr.svg +++ b/docs/reference/esql/functions/signature/qstr.svg @@ -1 +1 @@ -QSTR(query) +QSTR(query) \ No newline at end of file diff --git a/docs/reference/esql/functions/signature/to_date_nanos.svg b/docs/reference/esql/functions/signature/to_date_nanos.svg new file mode 100644 index 0000000000000..0b24b56429588 --- /dev/null +++ b/docs/reference/esql/functions/signature/to_date_nanos.svg @@ -0,0 +1 @@ +TO_DATE_NANOS(field) \ No newline at end of file diff --git a/docs/reference/esql/functions/types/to_date_nanos.asciidoc b/docs/reference/esql/functions/types/to_date_nanos.asciidoc new file mode 100644 index 0000000000000..1f50b65f25a77 --- /dev/null +++ b/docs/reference/esql/functions/types/to_date_nanos.asciidoc @@ -0,0 +1,9 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Supported types* + +[%header.monospaced.styled,format=dsv,separator=|] +|=== +field | result +date_nanos +|=== diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv index 7e857f5243f58..83a2f3cb1c281 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv @@ -1,9 +1,9 @@ -millis:date,nanos:date_nanos -2023-10-23T13:55:01.543Z,2023-10-23T13:55:01.543123456Z -2023-10-23T13:53:55.832Z,2023-10-23T13:53:55.832987654Z -2023-10-23T13:52:55.015Z,2023-10-23T13:52:55.015787878Z -2023-10-23T13:51:54.732Z,2023-10-23T13:51:54.732102837Z -2023-10-23T13:33:34.937Z,2023-10-23T13:33:34.937193000Z -2023-10-23T12:27:28.948Z,2023-10-23T12:27:28.948000000Z -2023-10-23T12:15:03.360Z,2023-10-23T12:15:03.360103847Z -1999-10-23T12:15:03.360Z,[2023-03-23T12:15:03.360103847Z, 2023-02-23T13:33:34.937193000Z, 2023-01-23T13:55:01.543123456Z] +millis:date,nanos:date_nanos,num:long +2023-10-23T13:55:01.543Z,2023-10-23T13:55:01.543123456Z,1698069301543123456 +2023-10-23T13:53:55.832Z,2023-10-23T13:53:55.832987654Z,1698069235832987654 +2023-10-23T13:52:55.015Z,2023-10-23T13:52:55.015787878Z,1698069175015787878 +2023-10-23T13:51:54.732Z,2023-10-23T13:51:54.732102837Z,1698069114732102837 +2023-10-23T13:33:34.937Z,2023-10-23T13:33:34.937193000Z,1698068014937193000 +2023-10-23T12:27:28.948Z,2023-10-23T12:27:28.948000000Z,1698064048948000000 +2023-10-23T12:15:03.360Z,2023-10-23T12:15:03.360103847Z,1698063303360103847 +1999-10-23T12:15:03.360Z,[2023-03-23T12:15:03.360103847Z, 2023-02-23T13:33:34.937193000Z, 2023-01-23T13:55:01.543123456Z], 0 diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec index b77689e1b5768..883010eb484db 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec @@ -70,3 +70,196 @@ FROM date_nanos | SORT millis asc | EVAL nanos = MV_LAST(nanos) | KEEP nanos | L nanos:date_nanos 2023-03-23T12:15:03.360103847Z ; + +string to date nanos +required_capability: to_date_nanos + +ROW d = TO_DATE_NANOS("2023-03-23T12:15:03.360103847"); + +d:date_nanos +2023-03-23T12:15:03.360103847Z +; + +string to date nanos, :: notation +required_capability: to_date_nanos + +ROW d = "2023-03-23T12:15:03.360103847"::date_nanos; + +d:date_nanos +2023-03-23T12:15:03.360103847Z +; + +string to date nanos, milliseconds only +required_capability: to_date_nanos + +ROW d = TO_DATE_NANOS("2023-03-23T12:15:03.360"); + +d:date_nanos +2023-03-23T12:15:03.360Z +; + +string to date nanos, out of range +required_capability: to_date_nanos + +ROW d = TO_DATE_NANOS("2262-04-12T00:00:00.000"); +warning:Line 1:9: evaluation of [TO_DATE_NANOS(\"2262-04-12T00:00:00.000\")] failed, treating result as null. Only first 20 failures recorded. +warning:Line 1:9: java.lang.IllegalArgumentException: date[2262-04-12T00:00:00Z] is after 2262-04-11T23:47:16.854775807 and cannot be stored in nanosecond resolution + +d:date_nanos +null +; + +string to date nanos, pre 1970 +required_capability: to_date_nanos + +ROW d = TO_DATE_NANOS("1969-04-12T00:00:00.000"); +warning:Line 1:9: evaluation of [TO_DATE_NANOS(\"1969-04-12T00:00:00.000\")] failed, treating result as null. Only first 20 failures recorded. +warning:Line 1:9: java.lang.IllegalArgumentException: date[1969-04-12T00:00:00Z] is before the epoch in 1970 and cannot be stored in nanosecond resolution + +d:date_nanos +null +; + +long to date nanos +required_capability: to_date_nanos + +ROW d = TO_DATE_NANOS(1724160894123456789); + +d:date_nanos +2024-08-20T13:34:54.123456789Z +; + +long to date nanos, :: notation +required_capability: to_date_nanos + +ROW d = 1724160894123456789::date_nanos; + +d:date_nanos +2024-08-20T13:34:54.123456789Z +; + + +long to date nanos, before 1970 +required_capability: to_date_nanos + +ROW d = TO_DATE_NANOS(TO_LONG(-1)); + +warning:Line 1:9: evaluation of [TO_DATE_NANOS(TO_LONG(-1))] failed, treating result as null. Only first 20 failures recorded. +warning:Line 1:9: java.lang.IllegalArgumentException: Nanosecond dates before 1970-01-01T00:00:00.000Z are not supported. +d:date_nanos +null +; + +unsigned long to date nanos +required_capability: to_date_nanos + +ROW d = TO_DATE_NANOS(TO_UNSIGNED_LONG(1724160894123456789)); + +d:date_nanos +2024-08-20T13:34:54.123456789Z +; + +double to date nanos +required_capability: to_date_nanos + +ROW d = TO_DATE_NANOS(1724160894123456789.0); + +d:date_nanos +# Note we've lost some precision here +2024-08-20T13:34:54.123456768Z +; + +datetime to date nanos, in range +required_capability: to_date_nanos + +ROW d = TO_DATE_NANOS(TO_DATETIME("2024-08-20T13:34:54.123Z")); + +d:date_nanos +2024-08-20T13:34:54.123000000Z +; + +datetime to date nanos, with overflow +required_capability: to_date_nanos + +ROW d = TO_DATE_NANOS(TO_DATETIME("2262-04-12T00:00:00.000")); +warning:Line 1:9: evaluation of [TO_DATE_NANOS(TO_DATETIME(\"2262-04-12T00:00:00.000\"))] failed, treating result as null. Only first 20 failures recorded. +warning:Line 1:9: java.lang.IllegalArgumentException: milliSeconds [9223372800000] are after 2262-04-11T23:47:16.854775807 and cannot be converted to nanoseconds + +d:date_nanos +null +; + +datetime to date nanos, pre 1970 +required_capability: to_date_nanos + +ROW d = TO_DATE_NANOS(TO_DATETIME("1969-04-12T00:00:00.000")); +warning:Line 1:9: evaluation of [TO_DATE_NANOS(TO_DATETIME(\"1969-04-12T00:00:00.000\"))] failed, treating result as null. Only first 20 failures recorded. +warning:Line 1:9: java.lang.IllegalArgumentException: milliSeconds [-22809600000] are before the epoch in 1970 and cannot be converted to nanoseconds + +d:date_nanos +null +; + + +date nanos to long, index version +required_capability: to_date_nanos + +FROM date_nanos | WHERE millis > "2020-02-02" | EVAL l = TO_LONG(nanos) | KEEP l; + +l:long +1698069301543123456 +1698069235832987654 +1698069175015787878 +1698069114732102837 +1698068014937193000 +1698064048948000000 +1698063303360103847 +; + +long to date nanos, index version +required_capability: to_date_nanos + +FROM date_nanos | WHERE millis > "2020-02-02" | EVAL d = TO_DATE_NANOS(num) | KEEP d; + +d:date_nanos +2023-10-23T13:55:01.543123456Z +2023-10-23T13:53:55.832987654Z +2023-10-23T13:52:55.015787878Z +2023-10-23T13:51:54.732102837Z +2023-10-23T13:33:34.937193000Z +2023-10-23T12:27:28.948000000Z +2023-10-23T12:15:03.360103847Z +; + +date_nanos to date nanos, index version +required_capability: to_date_nanos + +FROM date_nanos | WHERE millis > "2020-02-02" | EVAL d = TO_DATE_NANOS(nanos) | KEEP d; + +d:date_nanos +2023-10-23T13:55:01.543123456Z +2023-10-23T13:53:55.832987654Z +2023-10-23T13:52:55.015787878Z +2023-10-23T13:51:54.732102837Z +2023-10-23T13:33:34.937193000Z +2023-10-23T12:27:28.948000000Z +2023-10-23T12:15:03.360103847Z +; + +attempt to cast the result of a fold to date nanos +required_capability: to_date_nanos + +ROW d = TO_DATE_NANOS(CONCAT("2023-01-01","T12:12:12")); + +d:date_nanos +2023-01-01T12:12:12.000000000Z +; + +attempt to cast nulls to date nanos +required_capability: to_date_nanos + +ROW a = TO_DATE_NANOS(null), b = TO_DATE_NANOS(null + 1::long), c = TO_DATE_NANOS(CONCAT("2024", null)); + +a:date_nanos | b:date_nanos | c:date_nanos +null | null | null +; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-date_nanos.json b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-date_nanos.json index 506290d90b4b0..a07f9eeeca7b8 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-date_nanos.json +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-date_nanos.json @@ -5,6 +5,9 @@ }, "nanos": { "type": "date_nanos" + }, + "num": { + "type": "long" } } } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec index 2b3fa9dec797d..13c3857a5c497 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec @@ -97,6 +97,8 @@ double tau() "boolean to_boolean(field:boolean|keyword|text|double|long|unsigned_long|integer)" "cartesian_point to_cartesianpoint(field:cartesian_point|keyword|text)" "cartesian_shape to_cartesianshape(field:cartesian_point|cartesian_shape|keyword|text)" +"date_nanos to_date_nanos(field:date|date_nanos|keyword|text|double|long|unsigned_long)" +"date_nanos to_datenanos(field:date|date_nanos|keyword|text|double|long|unsigned_long)" "date_period to_dateperiod(field:date_period|keyword|text)" "date to_datetime(field:date|date_nanos|keyword|text|double|long|unsigned_long|integer)" "double to_dbl(field:boolean|date|keyword|text|double|long|unsigned_long|integer|counter_double|counter_integer|counter_long)" @@ -227,6 +229,8 @@ to_bool |field |"boolean|keyword|text|double to_boolean |field |"boolean|keyword|text|double|long|unsigned_long|integer" |Input value. The input can be a single- or multi-valued column or an expression. to_cartesianpo|field |"cartesian_point|keyword|text" |Input value. The input can be a single- or multi-valued column or an expression. to_cartesiansh|field |"cartesian_point|cartesian_shape|keyword|text" |Input value. The input can be a single- or multi-valued column or an expression. +to_date_nanos |field |"date|date_nanos|keyword|text|double|long|unsigned_long" |Input value. The input can be a single- or multi-valued column or an expression. +to_datenanos |field |"date|date_nanos|keyword|text|double|long|unsigned_long" |Input value. The input can be a single- or multi-valued column or an expression. to_dateperiod |field |"date_period|keyword|text" |Input value. The input is a valid constant date period expression. to_datetime |field |"date|date_nanos|keyword|text|double|long|unsigned_long|integer" |Input value. The input can be a single- or multi-valued column or an expression. to_dbl |field |"boolean|date|keyword|text|double|long|unsigned_long|integer|counter_double|counter_integer|counter_long" |Input value. The input can be a single- or multi-valued column or an expression. @@ -357,6 +361,8 @@ to_bool |Converts an input value to a boolean value. A string value of *tr to_boolean |Converts an input value to a boolean value. A string value of *true* will be case-insensitive converted to the Boolean *true*. For anything else, including the empty string, the function will return *false*. The numerical value of *0* will be converted to *false*, anything else will be converted to *true*. to_cartesianpo|Converts an input value to a `cartesian_point` value. A string will only be successfully converted if it respects the {wikipedia}/Well-known_text_representation_of_geometry[WKT Point] format. to_cartesiansh|Converts an input value to a `cartesian_shape` value. A string will only be successfully converted if it respects the {wikipedia}/Well-known_text_representation_of_geometry[WKT] format. +to_date_nanos |Converts an input to a nanosecond-resolution date value (aka date_nanos). +to_datenanos |Converts an input to a nanosecond-resolution date value (aka date_nanos). to_dateperiod |Converts an input value into a `date_period` value. to_datetime |Converts an input value to a date value. A string will only be successfully converted if it's respecting the format `yyyy-MM-dd'T'HH:mm:ss.SSS'Z'`. To convert dates in other formats, use <>. to_dbl |Converts an input value to a double value. If the input parameter is of a date type, its value will be interpreted as milliseconds since the {wikipedia}/Unix_time[Unix epoch], converted to double. Boolean *true* will be converted to double *1.0*, *false* to *0.0*. @@ -489,6 +495,8 @@ to_bool |boolean to_boolean |boolean |false |false |false to_cartesianpo|cartesian_point |false |false |false to_cartesiansh|cartesian_shape |false |false |false +to_date_nanos |date_nanos |false |false |false +to_datenanos |date_nanos |false |false |false to_dateperiod |date_period |false |false |false to_datetime |date |false |false |false to_dbl |double |false |false |false @@ -536,5 +544,5 @@ required_capability: meta meta functions | stats a = count(*), b = count(*), c = count(*) | mv_expand c; a:long | b:long | c:long -119 | 119 | 119 +121 | 121 | 121 ; diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosFromDatetimeEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosFromDatetimeEvaluator.java new file mode 100644 index 0000000000000..e00e7e044ae12 --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosFromDatetimeEvaluator.java @@ -0,0 +1,122 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.convert; + +import java.lang.IllegalArgumentException; +import java.lang.Override; +import java.lang.String; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.LongVector; +import org.elasticsearch.compute.data.Vector; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.xpack.esql.core.tree.Source; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link ToDateNanos}. + * This class is generated. Do not edit it. + */ +public final class ToDateNanosFromDatetimeEvaluator extends AbstractConvertFunction.AbstractEvaluator { + public ToDateNanosFromDatetimeEvaluator(EvalOperator.ExpressionEvaluator field, Source source, + DriverContext driverContext) { + super(driverContext, field, source); + } + + @Override + public String name() { + return "ToDateNanosFromDatetime"; + } + + @Override + public Block evalVector(Vector v) { + LongVector vector = (LongVector) v; + int positionCount = v.getPositionCount(); + if (vector.isConstant()) { + try { + return driverContext.blockFactory().newConstantLongBlockWith(evalValue(vector, 0), positionCount); + } catch (IllegalArgumentException e) { + registerException(e); + return driverContext.blockFactory().newConstantNullBlock(positionCount); + } + } + try (LongBlock.Builder builder = driverContext.blockFactory().newLongBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + try { + builder.appendLong(evalValue(vector, p)); + } catch (IllegalArgumentException e) { + registerException(e); + builder.appendNull(); + } + } + return builder.build(); + } + } + + private static long evalValue(LongVector container, int index) { + long value = container.getLong(index); + return ToDateNanos.fromDatetime(value); + } + + @Override + public Block evalBlock(Block b) { + LongBlock block = (LongBlock) b; + int positionCount = block.getPositionCount(); + try (LongBlock.Builder builder = driverContext.blockFactory().newLongBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + int valueCount = block.getValueCount(p); + int start = block.getFirstValueIndex(p); + int end = start + valueCount; + boolean positionOpened = false; + boolean valuesAppended = false; + for (int i = start; i < end; i++) { + try { + long value = evalValue(block, i); + if (positionOpened == false && valueCount > 1) { + builder.beginPositionEntry(); + positionOpened = true; + } + builder.appendLong(value); + valuesAppended = true; + } catch (IllegalArgumentException e) { + registerException(e); + } + } + if (valuesAppended == false) { + builder.appendNull(); + } else if (positionOpened) { + builder.endPositionEntry(); + } + } + return builder.build(); + } + } + + private static long evalValue(LongBlock container, int index) { + long value = container.getLong(index); + return ToDateNanos.fromDatetime(value); + } + + public static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory field; + + public Factory(EvalOperator.ExpressionEvaluator.Factory field, Source source) { + this.field = field; + this.source = source; + } + + @Override + public ToDateNanosFromDatetimeEvaluator get(DriverContext context) { + return new ToDateNanosFromDatetimeEvaluator(field.get(context), source, context); + } + + @Override + public String toString() { + return "ToDateNanosFromDatetimeEvaluator[field=" + field + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosFromDoubleEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosFromDoubleEvaluator.java new file mode 100644 index 0000000000000..23b30e669241b --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosFromDoubleEvaluator.java @@ -0,0 +1,124 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.convert; + +import java.lang.IllegalArgumentException; +import java.lang.Override; +import java.lang.String; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.data.DoubleVector; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.Vector; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.xpack.esql.core.InvalidArgumentException; +import org.elasticsearch.xpack.esql.core.tree.Source; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link ToDateNanos}. + * This class is generated. Do not edit it. + */ +public final class ToDateNanosFromDoubleEvaluator extends AbstractConvertFunction.AbstractEvaluator { + public ToDateNanosFromDoubleEvaluator(EvalOperator.ExpressionEvaluator field, Source source, + DriverContext driverContext) { + super(driverContext, field, source); + } + + @Override + public String name() { + return "ToDateNanosFromDouble"; + } + + @Override + public Block evalVector(Vector v) { + DoubleVector vector = (DoubleVector) v; + int positionCount = v.getPositionCount(); + if (vector.isConstant()) { + try { + return driverContext.blockFactory().newConstantLongBlockWith(evalValue(vector, 0), positionCount); + } catch (IllegalArgumentException | InvalidArgumentException e) { + registerException(e); + return driverContext.blockFactory().newConstantNullBlock(positionCount); + } + } + try (LongBlock.Builder builder = driverContext.blockFactory().newLongBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + try { + builder.appendLong(evalValue(vector, p)); + } catch (IllegalArgumentException | InvalidArgumentException e) { + registerException(e); + builder.appendNull(); + } + } + return builder.build(); + } + } + + private static long evalValue(DoubleVector container, int index) { + double value = container.getDouble(index); + return ToDateNanos.fromDouble(value); + } + + @Override + public Block evalBlock(Block b) { + DoubleBlock block = (DoubleBlock) b; + int positionCount = block.getPositionCount(); + try (LongBlock.Builder builder = driverContext.blockFactory().newLongBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + int valueCount = block.getValueCount(p); + int start = block.getFirstValueIndex(p); + int end = start + valueCount; + boolean positionOpened = false; + boolean valuesAppended = false; + for (int i = start; i < end; i++) { + try { + long value = evalValue(block, i); + if (positionOpened == false && valueCount > 1) { + builder.beginPositionEntry(); + positionOpened = true; + } + builder.appendLong(value); + valuesAppended = true; + } catch (IllegalArgumentException | InvalidArgumentException e) { + registerException(e); + } + } + if (valuesAppended == false) { + builder.appendNull(); + } else if (positionOpened) { + builder.endPositionEntry(); + } + } + return builder.build(); + } + } + + private static long evalValue(DoubleBlock container, int index) { + double value = container.getDouble(index); + return ToDateNanos.fromDouble(value); + } + + public static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory field; + + public Factory(EvalOperator.ExpressionEvaluator.Factory field, Source source) { + this.field = field; + this.source = source; + } + + @Override + public ToDateNanosFromDoubleEvaluator get(DriverContext context) { + return new ToDateNanosFromDoubleEvaluator(field.get(context), source, context); + } + + @Override + public String toString() { + return "ToDateNanosFromDoubleEvaluator[field=" + field + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosFromLongEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosFromLongEvaluator.java new file mode 100644 index 0000000000000..cc52208ce5a25 --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosFromLongEvaluator.java @@ -0,0 +1,122 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.convert; + +import java.lang.IllegalArgumentException; +import java.lang.Override; +import java.lang.String; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.LongVector; +import org.elasticsearch.compute.data.Vector; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.xpack.esql.core.tree.Source; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link ToDateNanos}. + * This class is generated. Do not edit it. + */ +public final class ToDateNanosFromLongEvaluator extends AbstractConvertFunction.AbstractEvaluator { + public ToDateNanosFromLongEvaluator(EvalOperator.ExpressionEvaluator field, Source source, + DriverContext driverContext) { + super(driverContext, field, source); + } + + @Override + public String name() { + return "ToDateNanosFromLong"; + } + + @Override + public Block evalVector(Vector v) { + LongVector vector = (LongVector) v; + int positionCount = v.getPositionCount(); + if (vector.isConstant()) { + try { + return driverContext.blockFactory().newConstantLongBlockWith(evalValue(vector, 0), positionCount); + } catch (IllegalArgumentException e) { + registerException(e); + return driverContext.blockFactory().newConstantNullBlock(positionCount); + } + } + try (LongBlock.Builder builder = driverContext.blockFactory().newLongBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + try { + builder.appendLong(evalValue(vector, p)); + } catch (IllegalArgumentException e) { + registerException(e); + builder.appendNull(); + } + } + return builder.build(); + } + } + + private static long evalValue(LongVector container, int index) { + long value = container.getLong(index); + return ToDateNanos.fromLong(value); + } + + @Override + public Block evalBlock(Block b) { + LongBlock block = (LongBlock) b; + int positionCount = block.getPositionCount(); + try (LongBlock.Builder builder = driverContext.blockFactory().newLongBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + int valueCount = block.getValueCount(p); + int start = block.getFirstValueIndex(p); + int end = start + valueCount; + boolean positionOpened = false; + boolean valuesAppended = false; + for (int i = start; i < end; i++) { + try { + long value = evalValue(block, i); + if (positionOpened == false && valueCount > 1) { + builder.beginPositionEntry(); + positionOpened = true; + } + builder.appendLong(value); + valuesAppended = true; + } catch (IllegalArgumentException e) { + registerException(e); + } + } + if (valuesAppended == false) { + builder.appendNull(); + } else if (positionOpened) { + builder.endPositionEntry(); + } + } + return builder.build(); + } + } + + private static long evalValue(LongBlock container, int index) { + long value = container.getLong(index); + return ToDateNanos.fromLong(value); + } + + public static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory field; + + public Factory(EvalOperator.ExpressionEvaluator.Factory field, Source source) { + this.field = field; + this.source = source; + } + + @Override + public ToDateNanosFromLongEvaluator get(DriverContext context) { + return new ToDateNanosFromLongEvaluator(field.get(context), source, context); + } + + @Override + public String toString() { + return "ToDateNanosFromLongEvaluator[field=" + field + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosFromStringEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosFromStringEvaluator.java new file mode 100644 index 0000000000000..c5a20ac298da7 --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosFromStringEvaluator.java @@ -0,0 +1,126 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.convert; + +import java.lang.IllegalArgumentException; +import java.lang.Override; +import java.lang.String; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.Vector; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.xpack.esql.core.tree.Source; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link ToDateNanos}. + * This class is generated. Do not edit it. + */ +public final class ToDateNanosFromStringEvaluator extends AbstractConvertFunction.AbstractEvaluator { + public ToDateNanosFromStringEvaluator(EvalOperator.ExpressionEvaluator field, Source source, + DriverContext driverContext) { + super(driverContext, field, source); + } + + @Override + public String name() { + return "ToDateNanosFromString"; + } + + @Override + public Block evalVector(Vector v) { + BytesRefVector vector = (BytesRefVector) v; + int positionCount = v.getPositionCount(); + BytesRef scratchPad = new BytesRef(); + if (vector.isConstant()) { + try { + return driverContext.blockFactory().newConstantLongBlockWith(evalValue(vector, 0, scratchPad), positionCount); + } catch (IllegalArgumentException e) { + registerException(e); + return driverContext.blockFactory().newConstantNullBlock(positionCount); + } + } + try (LongBlock.Builder builder = driverContext.blockFactory().newLongBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + try { + builder.appendLong(evalValue(vector, p, scratchPad)); + } catch (IllegalArgumentException e) { + registerException(e); + builder.appendNull(); + } + } + return builder.build(); + } + } + + private static long evalValue(BytesRefVector container, int index, BytesRef scratchPad) { + BytesRef value = container.getBytesRef(index, scratchPad); + return ToDateNanos.fromKeyword(value); + } + + @Override + public Block evalBlock(Block b) { + BytesRefBlock block = (BytesRefBlock) b; + int positionCount = block.getPositionCount(); + try (LongBlock.Builder builder = driverContext.blockFactory().newLongBlockBuilder(positionCount)) { + BytesRef scratchPad = new BytesRef(); + for (int p = 0; p < positionCount; p++) { + int valueCount = block.getValueCount(p); + int start = block.getFirstValueIndex(p); + int end = start + valueCount; + boolean positionOpened = false; + boolean valuesAppended = false; + for (int i = start; i < end; i++) { + try { + long value = evalValue(block, i, scratchPad); + if (positionOpened == false && valueCount > 1) { + builder.beginPositionEntry(); + positionOpened = true; + } + builder.appendLong(value); + valuesAppended = true; + } catch (IllegalArgumentException e) { + registerException(e); + } + } + if (valuesAppended == false) { + builder.appendNull(); + } else if (positionOpened) { + builder.endPositionEntry(); + } + } + return builder.build(); + } + } + + private static long evalValue(BytesRefBlock container, int index, BytesRef scratchPad) { + BytesRef value = container.getBytesRef(index, scratchPad); + return ToDateNanos.fromKeyword(value); + } + + public static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory field; + + public Factory(EvalOperator.ExpressionEvaluator.Factory field, Source source) { + this.field = field; + this.source = source; + } + + @Override + public ToDateNanosFromStringEvaluator get(DriverContext context) { + return new ToDateNanosFromStringEvaluator(field.get(context), source, context); + } + + @Override + public String toString() { + return "ToDateNanosFromStringEvaluator[field=" + field + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index f714d4d1808c1..f0fa89dedd9ab 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -273,6 +273,11 @@ public enum Cap { */ DATE_NANOS_TYPE(EsqlCorePlugin.DATE_NANOS_FEATURE_FLAG), + /** + * Support for to_date_nanos function + */ + TO_DATE_NANOS(EsqlCorePlugin.DATE_NANOS_FEATURE_FLAG), + /** * Support CIDRMatch in CombineDisjunctions rule. */ diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java index f96742b5a4d91..96ccf8b01e5bc 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java @@ -43,6 +43,7 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToBoolean; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToCartesianPoint; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToCartesianShape; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDateNanos; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDatePeriod; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDatetime; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDegrees; @@ -349,6 +350,7 @@ private FunctionDefinition[][] functions() { def(ToCartesianShape.class, ToCartesianShape::new, "to_cartesianshape"), def(ToDatePeriod.class, ToDatePeriod::new, "to_dateperiod"), def(ToDatetime.class, ToDatetime::new, "to_datetime", "to_dt"), + def(ToDateNanos.class, ToDateNanos::new, "to_date_nanos", "to_datenanos"), def(ToDegrees.class, ToDegrees::new, "to_degrees"), def(ToDouble.class, ToDouble::new, "to_double", "to_dbl"), def(ToGeoPoint.class, ToGeoPoint::new, "to_geopoint"), diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/UnaryScalarFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/UnaryScalarFunction.java index bdbc9b649c101..4d34033286f52 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/UnaryScalarFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/UnaryScalarFunction.java @@ -22,6 +22,7 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToBoolean; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToCartesianPoint; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToCartesianShape; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDateNanos; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDatetime; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDegrees; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDouble; @@ -107,6 +108,7 @@ public static List getNamedWriteables() { entries.add(ToBoolean.ENTRY); entries.add(ToCartesianPoint.ENTRY); entries.add(ToDatetime.ENTRY); + entries.add(ToDateNanos.ENTRY); entries.add(ToDegrees.ENTRY); entries.add(ToDouble.ENTRY); entries.add(ToGeoShape.ENTRY); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanos.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanos.java new file mode 100644 index 0000000000000..9a6a91b7ccedd --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanos.java @@ -0,0 +1,134 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.convert; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.time.DateFormatters; +import org.elasticsearch.common.time.DateUtils; +import org.elasticsearch.compute.ann.ConvertEvaluator; +import org.elasticsearch.xpack.esql.core.InvalidArgumentException; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.NodeInfo; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.core.type.DataTypeConverter; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.Param; + +import java.io.IOException; +import java.time.Instant; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME; +import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_NANOS; +import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE; +import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD; +import static org.elasticsearch.xpack.esql.core.type.DataType.LONG; +import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT; +import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG; +import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.DEFAULT_DATE_NANOS_FORMATTER; + +public class ToDateNanos extends AbstractConvertFunction { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( + Expression.class, + "ToDateNanos", + ToDateNanos::new + ); + + private static final Map EVALUATORS = Map.ofEntries( + Map.entry(DATETIME, ToDateNanosFromDatetimeEvaluator.Factory::new), + Map.entry(DATE_NANOS, (field, source) -> field), + Map.entry(LONG, ToDateNanosFromLongEvaluator.Factory::new), + Map.entry(KEYWORD, ToDateNanosFromStringEvaluator.Factory::new), + Map.entry(TEXT, ToDateNanosFromStringEvaluator.Factory::new), + Map.entry(DOUBLE, ToDateNanosFromDoubleEvaluator.Factory::new), + Map.entry(UNSIGNED_LONG, ToLongFromUnsignedLongEvaluator.Factory::new) + /* + NB: not including an integer conversion, because max int in nanoseconds is like 2 seconds after epoch, and it seems more likely + a user who tries to convert an int to a nanosecond date has made a mistake that we should catch that at parse time. + TO_DATE_NANOS(TO_LONG(intVal)) is still possible if someone really needs to do this. + */ + ); + + @FunctionInfo( + returnType = "date_nanos", + description = "Converts an input to a nanosecond-resolution date value (aka date_nanos).", + note = "The range for date nanos is 1970-01-01T00:00:00.000000000Z to 2262-04-11T23:47:16.854775807Z. Additionally, integers " + + "cannot be converted into date nanos, as the range of integer nanoseconds only covers about 2 seconds after epoch.", + preview = true + ) + public ToDateNanos( + Source source, + @Param( + name = "field", + type = { "date", "date_nanos", "keyword", "text", "double", "long", "unsigned_long" }, + description = "Input value. The input can be a single- or multi-valued column or an expression." + ) Expression field + ) { + super(source, field); + } + + protected ToDateNanos(StreamInput in) throws IOException { + super(in); + } + + @Override + public DataType dataType() { + return DATE_NANOS; + } + + @Override + protected Map factories() { + return EVALUATORS; + } + + @Override + public Expression replaceChildren(List newChildren) { + return new ToDateNanos(source(), newChildren.get(0)); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, ToDateNanos::new, field()); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + + @ConvertEvaluator(extraName = "FromLong", warnExceptions = { IllegalArgumentException.class }) + static long fromLong(long in) { + if (in < 0L) { + throw new IllegalArgumentException("Nanosecond dates before 1970-01-01T00:00:00.000Z are not supported."); + } + return in; + } + + @ConvertEvaluator(extraName = "FromDouble", warnExceptions = { IllegalArgumentException.class, InvalidArgumentException.class }) + static long fromDouble(double in) { + if (in < 0d) { + throw new IllegalArgumentException("Nanosecond dates before 1970-01-01T00:00:00.000Z are not supported."); + } + return DataTypeConverter.safeDoubleToLong(in); + } + + @ConvertEvaluator(extraName = "FromString", warnExceptions = { IllegalArgumentException.class }) + static long fromKeyword(BytesRef in) { + Instant parsed = DateFormatters.from(DEFAULT_DATE_NANOS_FORMATTER.parse(in.utf8ToString())).toInstant(); + return DateUtils.toLong(parsed); + } + + @ConvertEvaluator(extraName = "FromDatetime", warnExceptions = { IllegalArgumentException.class }) + static long fromDatetime(long in) { + return DateUtils.toNanoSeconds(in); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java index 0c530bd0eb273..edc3081a33681 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java @@ -28,6 +28,7 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToBoolean; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToCartesianPoint; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToCartesianShape; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDateNanos; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDatePeriod; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDatetime; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDouble; @@ -63,6 +64,7 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_POINT; import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_SHAPE; import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME; +import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_NANOS; import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_PERIOD; import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE; import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_POINT; @@ -96,6 +98,7 @@ public class EsqlDataTypeConverter { public static final DateFormatter DEFAULT_DATE_TIME_FORMATTER = DateFormatter.forPattern("strict_date_optional_time"); + public static final DateFormatter DEFAULT_DATE_NANOS_FORMATTER = DateFormatter.forPattern("strict_date_optional_time_nanos"); public static final DateFormatter HOUR_MINUTE_SECOND = DateFormatter.forPattern("strict_hour_minute_second_fraction"); @@ -104,6 +107,7 @@ public class EsqlDataTypeConverter { entry(CARTESIAN_POINT, ToCartesianPoint::new), entry(CARTESIAN_SHAPE, ToCartesianShape::new), entry(DATETIME, ToDatetime::new), + entry(DATE_NANOS, ToDateNanos::new), // ToDegrees, typeless entry(DOUBLE, ToDouble::new), entry(GEO_POINT, ToGeoPoint::new), @@ -499,7 +503,7 @@ public static String dateTimeToString(long dateTime) { } public static String nanoTimeToString(long dateTime) { - return DateFormatter.forPattern("strict_date_optional_time_nanos").formatNanos(dateTime); + return DEFAULT_DATE_NANOS_FORMATTER.formatNanos(dateTime); } public static String dateTimeToString(long dateTime, DateFormatter formatter) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java index e44ea907518b4..b3942a71edadb 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java @@ -623,6 +623,7 @@ public static void forUnaryBoolean( /** * Generate positive test cases for a unary function operating on an {@link DataType#DATETIME}. + * This variant defaults to maximum range of possible values */ public static void forUnaryDatetime( List suppliers, @@ -641,6 +642,29 @@ public static void forUnaryDatetime( ); } + /** + * Generate positive test cases for a unary function operating on an {@link DataType#DATETIME}. + * This variant accepts a range of values + */ + public static void forUnaryDatetime( + List suppliers, + String expectedEvaluatorToString, + DataType expectedType, + long min, + long max, + Function expectedValue, + List warnings + ) { + unaryNumeric( + suppliers, + expectedEvaluatorToString, + dateCases(min, max), + expectedType, + n -> expectedValue.apply(Instant.ofEpochMilli(n.longValue())), + warnings + ); + } + /** * Generate positive test cases for a unary function operating on an {@link DataType#DATE_NANOS}. */ @@ -1044,26 +1068,45 @@ public static List booleanCases() { *

*/ public static List dateCases() { - return List.of( - new TypedDataSupplier("<1970-01-01T00:00:00Z>", () -> 0L, DataType.DATETIME), - new TypedDataSupplier( - "", - () -> ESTestCase.randomLongBetween(0, 10 * (long) 10e11), // 1970-01-01T00:00:00Z - 2286-11-20T17:46:40Z - DataType.DATETIME - ), - new TypedDataSupplier( - "", - // 2286-11-20T17:46:40Z - +292278994-08-17T07:12:55.807Z - () -> ESTestCase.randomLongBetween(10 * (long) 10e11, Long.MAX_VALUE), - DataType.DATETIME - ), - new TypedDataSupplier( - "", - // very close to +292278994-08-17T07:12:55.807Z, the maximum supported millis since epoch - () -> ESTestCase.randomLongBetween(Long.MAX_VALUE / 100 * 99, Long.MAX_VALUE), - DataType.DATETIME - ) - ); + return dateCases(Long.MIN_VALUE, Long.MAX_VALUE); + } + + /** + * Generate cases for {@link DataType#DATETIME}. + *

+ * For multi-row parameters, see {@link MultiRowTestCaseSupplier#dateCases}. + *

+ */ + public static List dateCases(long min, long max) { + List cases = new ArrayList<>(); + if (min <= 0 && max >= 0) { + cases.add(new TypedDataSupplier("<1970-01-01T00:00:00Z>", () -> 0L, DataType.DATETIME)); + } + + // 1970-01-01T00:00:00Z - 2286-11-20T17:46:40Z + long lower1 = Math.max(min, 0); + long upper1 = Math.min(max, 10 * (long) 10e11); + if (lower1 < upper1) { + cases.add(new TypedDataSupplier("", () -> ESTestCase.randomLongBetween(lower1, upper1), DataType.DATETIME)); + } + + // 2286-11-20T17:46:40Z - +292278994-08-17T07:12:55.807Z + long lower2 = Math.max(min, 10 * (long) 10e11); + long upper2 = Math.min(max, Long.MAX_VALUE); + if (lower2 < upper2) { + cases.add(new TypedDataSupplier("", () -> ESTestCase.randomLongBetween(lower2, upper2), DataType.DATETIME)); + } + + // very close to +292278994-08-17T07:12:55.807Z, the maximum supported millis since epoch + long lower3 = Math.max(min, Long.MAX_VALUE / 100 * 99); + long upper3 = Math.min(max, Long.MAX_VALUE); + if (lower3 < upper3) { + cases.add( + new TypedDataSupplier("", () -> ESTestCase.randomLongBetween(lower3, upper3), DataType.DATETIME) + ); + } + + return cases; } /** diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosTests.java new file mode 100644 index 0000000000000..e91a5cc1ebca4 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosTests.java @@ -0,0 +1,135 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.convert; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.common.time.DateUtils; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; + +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.List; +import java.util.function.Supplier; + +public class ToDateNanosTests extends AbstractScalarFunctionTestCase { + public ToDateNanosTests(@Name("TestCase") Supplier testCaseSupplier) { + this.testCase = testCaseSupplier.get(); + } + + @ParametersFactory + public static Iterable parameters() { + final String read = "Attribute[channel=0]"; + final List suppliers = new ArrayList<>(); + + TestCaseSupplier.forUnaryDateNanos(suppliers, read, DataType.DATE_NANOS, DateUtils::toLong, List.of()); + TestCaseSupplier.forUnaryDatetime( + suppliers, + "ToDateNanosFromDatetimeEvaluator[field=" + read + "]", + DataType.DATE_NANOS, + 0, + DateUtils.MAX_NANOSECOND_INSTANT.toEpochMilli(), + i -> DateUtils.toNanoSeconds(i.toEpochMilli()), + List.of() + ); + TestCaseSupplier.forUnaryLong( + suppliers, + "ToDateNanosFromLongEvaluator[field=" + read + "]", + DataType.DATE_NANOS, + l -> l, + 0, + Long.MAX_VALUE, + List.of() + ); + TestCaseSupplier.forUnaryLong( + suppliers, + "ToDateNanosFromLongEvaluator[field=" + read + "]", + DataType.DATE_NANOS, + l -> null, + Long.MIN_VALUE, + -1L, + List.of( + "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", + "Line -1:-1: java.lang.IllegalArgumentException: Nanosecond dates before 1970-01-01T00:00:00.000Z are not supported." + ) + ); + TestCaseSupplier.forUnaryUnsignedLong( + suppliers, + "ToLongFromUnsignedLongEvaluator[field=" + read + "]", + DataType.DATE_NANOS, + BigInteger::longValueExact, + BigInteger.ZERO, + BigInteger.valueOf(Long.MAX_VALUE), + List.of() + ); + TestCaseSupplier.forUnaryUnsignedLong( + suppliers, + "ToLongFromUnsignedLongEvaluator[field=" + read + "]", + DataType.DATE_NANOS, + bi -> null, + BigInteger.valueOf(Long.MAX_VALUE).add(BigInteger.TWO), + UNSIGNED_LONG_MAX, + bi -> List.of( + "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", + "Line -1:-1: org.elasticsearch.xpack.esql.core.InvalidArgumentException: [" + bi + "] out of [long] range" + ) + ); + TestCaseSupplier.forUnaryDouble( + suppliers, + "ToDateNanosFromDoubleEvaluator[field=" + read + "]", + DataType.DATE_NANOS, + d -> null, + Double.NEGATIVE_INFINITY, + -Double.MIN_VALUE, + d -> List.of( + "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", + "Line -1:-1: java.lang.IllegalArgumentException: Nanosecond dates before 1970-01-01T00:00:00.000Z are not supported." + ) + ); + TestCaseSupplier.forUnaryDouble( + suppliers, + "ToDateNanosFromDoubleEvaluator[field=" + read + "]", + DataType.DATE_NANOS, + d -> null, + 9.223372036854777E18, // a "convenient" value larger than `(double) Long.MAX_VALUE` (== ...776E18) + Double.POSITIVE_INFINITY, + d -> List.of( + "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", + "Line -1:-1: org.elasticsearch.xpack.esql.core.InvalidArgumentException: [" + d + "] out of [long] range" + ) + ); + TestCaseSupplier.forUnaryStrings( + suppliers, + "ToDateNanosFromStringEvaluator[field=" + read + "]", + DataType.DATE_NANOS, + bytesRef -> null, + bytesRef -> List.of( + "Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.", + "Line -1:-1: java.lang.IllegalArgumentException: " + + (bytesRef.utf8ToString().isEmpty() + ? "cannot parse empty datetime" + : ("failed to parse date field [" + bytesRef.utf8ToString() + "] with format [strict_date_optional_time_nanos]")) + ) + ); + return parameterSuppliersFromTypedDataWithDefaultChecks( + true, + suppliers, + (v, p) -> "date_nanos or datetime or double or long or string or unsigned_long" + ); + } + + @Override + protected Expression build(Source source, List args) { + return new ToDateNanos(source, args.get(0)); + } +} From 3516b9a67529bd9d2509f252c01079bceb3ed20a Mon Sep 17 00:00:00 2001 From: Fang Xing <155562079+fang-xing-esql@users.noreply.github.com> Date: Thu, 26 Sep 2024 12:06:35 -0400 Subject: [PATCH 20/43] [ES|QL] Check expression resolved before checking its data type in ImplicitCasting (#113314) * check resolved before check data type --- docs/changelog/113314.yaml | 6 +++ .../xpack/esql/analysis/Analyzer.java | 37 ++++++++++--------- .../xpack/esql/analysis/VerifierTests.java | 11 ++++++ 3 files changed, 37 insertions(+), 17 deletions(-) create mode 100644 docs/changelog/113314.yaml diff --git a/docs/changelog/113314.yaml b/docs/changelog/113314.yaml new file mode 100644 index 0000000000000..c496ad3dd86f1 --- /dev/null +++ b/docs/changelog/113314.yaml @@ -0,0 +1,6 @@ +pr: 113314 +summary: "[ES|QL] Check expression resolved before checking its data type in `ImplicitCasting`" +area: ES|QL +type: bug +issues: + - 113242 diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index 9288e1cf81a15..63462d1721f71 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -1070,12 +1070,12 @@ private static Expression processBinaryOperator(BinaryOperator o) { private static Expression processIn(In in) { Expression left = in.value(); List right = in.list(); - DataType targetDataType = left.dataType(); - if (left.resolved() == false || supportsStringImplicitCasting(targetDataType) == false) { + if (left.resolved() == false || supportsStringImplicitCasting(left.dataType()) == false) { return in; } + DataType targetDataType = left.dataType(); List newChildren = new ArrayList<>(right.size() + 1); boolean childrenChanged = false; @@ -1107,23 +1107,26 @@ private static Expression castMixedNumericTypes(EsqlScalarFunction f, DataType t DataType childDataType; for (Expression e : f.children()) { - childDataType = e.dataType(); - if (childDataType.isNumeric() == false - || childDataType == targetNumericType - || canCastNumeric(childDataType, targetNumericType) == false) { + if (e.resolved()) { + childDataType = e.dataType(); + if (childDataType.isNumeric() == false + || childDataType == targetNumericType + || canCastNumeric(childDataType, targetNumericType) == false) { + newChildren.add(e); + continue; + } + childrenChanged = true; + // add a casting function + switch (targetNumericType) { + case INTEGER -> newChildren.add(new ToInteger(e.source(), e)); + case LONG -> newChildren.add(new ToLong(e.source(), e)); + case DOUBLE -> newChildren.add(new ToDouble(e.source(), e)); + case UNSIGNED_LONG -> newChildren.add(new ToUnsignedLong(e.source(), e)); + default -> throw new EsqlIllegalArgumentException("unexpected data type: " + targetNumericType); + } + } else { newChildren.add(e); - continue; } - childrenChanged = true; - // add a casting function - switch (targetNumericType) { - case INTEGER -> newChildren.add(new ToInteger(e.source(), e)); - case LONG -> newChildren.add(new ToLong(e.source(), e)); - case DOUBLE -> newChildren.add(new ToDouble(e.source(), e)); - case UNSIGNED_LONG -> newChildren.add(new ToUnsignedLong(e.source(), e)); - default -> throw new EsqlIllegalArgumentException("unexpected data type: " + targetNumericType); - } - } return childrenChanged ? f.replaceChildren(newChildren) : f; } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java index 2012e319510af..35e553de61a78 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java @@ -283,6 +283,17 @@ public void testImplicitCastingErrorMessages() { "1:42: Cannot convert string [a] to [DOUBLE], error [Cannot parse number [a]]", error("ROW a=[3, 5, 1, 6] | EVAL avg_a = MV_AVG(\"a\")") ); + assertEquals( + "1:19: Unknown column [languages.*], did you mean any of [languages, languages.byte, languages.long, languages.short]?", + error("from test | where `languages.*` in (1, 2)") + ); + assertEquals("1:22: Unknown function [func]", error("from test | eval x = func(languages) | where x in (1, 2)")); + assertEquals( + "1:32: Unknown column [languages.*], did you mean any of [languages, languages.byte, languages.long, languages.short]?", + error("from test | eval x = coalesce( `languages.*`, languages, 0 )") + ); + String error = error("from test | eval x = func(languages) | eval y = coalesce(x, languages, 0 )"); + assertThat(error, containsString("function [func]")); } public void testAggsExpressionsInStatsAggs() { From d8a32157e555edf536cfbfc46cc1ebed0227f7e1 Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Thu, 26 Sep 2024 10:07:33 -0600 Subject: [PATCH 21/43] Add allowed warnings to the dot-prefix yaml tests (#113560) This adds the missing optional warning from the global legacy template Relates to #113529 (doesn't close it yet until this has been unmuted, which won't happen until this change is backported to 8.16) --- .../resources/rest-api-spec/test/dot_prefix/10_basic.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/modules/dot-prefix-validation/src/yamlRestTest/resources/rest-api-spec/test/dot_prefix/10_basic.yml b/modules/dot-prefix-validation/src/yamlRestTest/resources/rest-api-spec/test/dot_prefix/10_basic.yml index b160af4ee8290..ae256daeb8abb 100644 --- a/modules/dot-prefix-validation/src/yamlRestTest/resources/rest-api-spec/test/dot_prefix/10_basic.yml +++ b/modules/dot-prefix-validation/src/yamlRestTest/resources/rest-api-spec/test/dot_prefix/10_basic.yml @@ -164,11 +164,13 @@ teardown: --- "Deprecated index template with a dot prefix index pattern": - requires: - test_runner_features: ["warnings", "headers"] + test_runner_features: ["warnings", "headers", "allowed_warnings"] - do: warnings: - "Index [.data-*] name begins with a dot (.), which is deprecated, and will not be allowed in a future Elasticsearch version." + allowed_warnings: + - "index template [my-template] has index patterns [regular, .data-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template] will take precedence during new index creation" indices.put_index_template: name: my-template body: @@ -177,7 +179,8 @@ teardown: - do: headers: { X-elastic-product-origin: kibana } - warnings: + allowed_warnings: + - "index template [my-template2] has index patterns [other, .data2-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template2] will take precedence during new index creation" indices.put_index_template: name: my-template2 body: From 1e2c19fb0be8f73caec9b7bb693b89d120020127 Mon Sep 17 00:00:00 2001 From: Pat Whelan Date: Thu, 26 Sep 2024 12:18:42 -0400 Subject: [PATCH 22/43] [ML] Add stream flag to inference providers (#113424) Pass the stream flag from the REST request through to the inference providers via the InferenceInputs. Co-authored-by: Elastic Machine --- .../inference/InferenceService.java | 2 ++ .../TestDenseInferenceServiceExtension.java | 1 + .../mock/TestRerankingServiceExtension.java | 1 + .../TestSparseInferenceServiceExtension.java | 1 + ...stStreamingCompletionServiceExtension.java | 1 + .../action/TransportInferenceAction.java | 1 + .../inference/external/http/HttpUtils.java | 2 +- .../http/sender/DocumentsOnlyInput.java | 14 +++++++++++-- .../http/sender/QueryAndDocsInputs.java | 21 +++++++++++++------ .../inference/services/SenderService.java | 5 +++-- .../inference/services/ServiceUtils.java | 1 + .../AlibabaCloudSearchService.java | 1 + .../ElasticsearchInternalService.java | 1 + .../services/elser/ElserInternalService.java | 1 + .../SimpleServiceIntegrationValidator.java | 1 + .../inference/services/ServiceUtilsTests.java | 21 ++++++++----------- .../AmazonBedrockServiceTests.java | 4 ++++ .../anthropic/AnthropicServiceTests.java | 2 ++ .../AzureAiStudioServiceTests.java | 3 +++ .../azureopenai/AzureOpenAiServiceTests.java | 3 +++ .../services/cohere/CohereServiceTests.java | 6 ++++++ .../elastic/ElasticInferenceServiceTests.java | 2 ++ .../GoogleAiStudioServiceTests.java | 4 ++++ .../HuggingFaceBaseServiceTests.java | 1 + .../huggingface/HuggingFaceServiceTests.java | 2 ++ .../ibmwatsonx/IbmWatsonxServiceTests.java | 3 +++ .../services/mistral/MistralServiceTests.java | 2 ++ .../services/openai/OpenAiServiceTests.java | 3 +++ ...impleServiceIntegrationValidatorTests.java | 5 ++++- 29 files changed, 91 insertions(+), 24 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/inference/InferenceService.java b/server/src/main/java/org/elasticsearch/inference/InferenceService.java index f677f75dfb5ae..854c58b4f57ad 100644 --- a/server/src/main/java/org/elasticsearch/inference/InferenceService.java +++ b/server/src/main/java/org/elasticsearch/inference/InferenceService.java @@ -85,6 +85,7 @@ void parseRequestConfig( * @param model The model * @param query Inference query, mainly for re-ranking * @param input Inference input + * @param stream Stream inference results * @param taskSettings Settings in the request to override the model's defaults * @param inputType For search, ingest etc * @param timeout The timeout for the request @@ -94,6 +95,7 @@ void infer( Model model, @Nullable String query, List input, + boolean stream, Map taskSettings, InputType inputType, TimeValue timeout, diff --git a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestDenseInferenceServiceExtension.java b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestDenseInferenceServiceExtension.java index 10d8f90efef5b..daa29d33699ef 100644 --- a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestDenseInferenceServiceExtension.java +++ b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestDenseInferenceServiceExtension.java @@ -94,6 +94,7 @@ public void infer( Model model, @Nullable String query, List input, + boolean stream, Map taskSettings, InputType inputType, TimeValue timeout, diff --git a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestRerankingServiceExtension.java b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestRerankingServiceExtension.java index fae11d5b53ca3..1894db6db8df6 100644 --- a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestRerankingServiceExtension.java +++ b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestRerankingServiceExtension.java @@ -85,6 +85,7 @@ public void infer( Model model, @Nullable String query, List input, + boolean stream, Map taskSettings, InputType inputType, TimeValue timeout, diff --git a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestSparseInferenceServiceExtension.java b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestSparseInferenceServiceExtension.java index fee9855b188c2..1a5df146a3aa4 100644 --- a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestSparseInferenceServiceExtension.java +++ b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestSparseInferenceServiceExtension.java @@ -88,6 +88,7 @@ public void infer( Model model, @Nullable String query, List input, + boolean stream, Map taskSettings, InputType inputType, TimeValue timeout, diff --git a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestStreamingCompletionServiceExtension.java b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestStreamingCompletionServiceExtension.java index 3d72b1f2729b0..4313026e92521 100644 --- a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestStreamingCompletionServiceExtension.java +++ b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestStreamingCompletionServiceExtension.java @@ -85,6 +85,7 @@ public void infer( Model model, String query, List input, + boolean stream, Map taskSettings, InputType inputType, TimeValue timeout, diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportInferenceAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportInferenceAction.java index 803e8f1e07612..4186b281a35b5 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportInferenceAction.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportInferenceAction.java @@ -114,6 +114,7 @@ private void inferOnService( model, request.getQuery(), request.getInput(), + request.isStreaming(), request.getTaskSettings(), request.getInputType(), request.getInferenceTimeout(), diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpUtils.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpUtils.java index 9f2ceddc92a2e..4282e5d1e7cb9 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpUtils.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpUtils.java @@ -46,7 +46,7 @@ private static String getStatusCodeErrorMessage(Request request, HttpResult resu } public static void checkForEmptyBody(ThrottlerManager throttlerManager, Logger logger, Request request, HttpResult result) { - if (result.isBodyEmpty()) { + if (result.isBodyEmpty() && (request.isStreaming() == false)) { String message = format("Response body was empty for request from inference entity id [%s]", request.getInferenceEntityId()); throttlerManager.warn(logger, message); throw new IllegalStateException(message); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/DocumentsOnlyInput.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/DocumentsOnlyInput.java index a32e2018117f8..8cf411d84c932 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/DocumentsOnlyInput.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/DocumentsOnlyInput.java @@ -21,13 +21,23 @@ public static DocumentsOnlyInput of(InferenceInputs inferenceInputs) { } private final List input; + private final boolean stream; - public DocumentsOnlyInput(List chunks) { + public DocumentsOnlyInput(List input) { + this(input, false); + } + + public DocumentsOnlyInput(List input, boolean stream) { super(); - this.input = Objects.requireNonNull(chunks); + this.input = Objects.requireNonNull(input); + this.stream = stream; } public List getInputs() { return this.input; } + + public boolean stream() { + return stream; + } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/QueryAndDocsInputs.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/QueryAndDocsInputs.java index 0d5f98c180ba9..50bb77b307db3 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/QueryAndDocsInputs.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/QueryAndDocsInputs.java @@ -21,6 +21,19 @@ public static QueryAndDocsInputs of(InferenceInputs inferenceInputs) { } private final String query; + private final List chunks; + private final boolean stream; + + public QueryAndDocsInputs(String query, List chunks) { + this(query, chunks, false); + } + + public QueryAndDocsInputs(String query, List chunks, boolean stream) { + super(); + this.query = Objects.requireNonNull(query); + this.chunks = Objects.requireNonNull(chunks); + this.stream = stream; + } public String getQuery() { return query; @@ -30,12 +43,8 @@ public List getChunks() { return chunks; } - List chunks; - - public QueryAndDocsInputs(String query, List chunks) { - super(); - this.query = Objects.requireNonNull(query); - this.chunks = Objects.requireNonNull(chunks); + public boolean stream() { + return stream; } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/SenderService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/SenderService.java index 864aebcef124f..21b2df6af1ab6 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/SenderService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/SenderService.java @@ -51,6 +51,7 @@ public void infer( Model model, @Nullable String query, List input, + boolean stream, Map taskSettings, InputType inputType, TimeValue timeout, @@ -58,9 +59,9 @@ public void infer( ) { init(); if (query != null) { - doInfer(model, new QueryAndDocsInputs(query, input), taskSettings, inputType, timeout, listener); + doInfer(model, new QueryAndDocsInputs(query, input, stream), taskSettings, inputType, timeout, listener); } else { - doInfer(model, new DocumentsOnlyInput(input), taskSettings, inputType, timeout, listener); + doInfer(model, new DocumentsOnlyInput(input, stream), taskSettings, inputType, timeout, listener); } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ServiceUtils.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ServiceUtils.java index 6eb0331913009..32c1d17373e53 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ServiceUtils.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ServiceUtils.java @@ -666,6 +666,7 @@ public static void getEmbeddingSize(Model model, InferenceService service, Actio model, null, List.of(TEST_EMBEDDING_INPUT), + false, Map.of(), InputType.INGEST, InferenceAction.Request.DEFAULT_TIMEOUT, diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchService.java index 8f0c9896c6642..994bad194aef6 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchService.java @@ -309,6 +309,7 @@ private void checkAlibabaCloudSearchServiceConfig(Model model, InferenceService model, query, List.of(input), + false, Map.of(), InputType.INGEST, DEFAULT_TIMEOUT, diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java index cca8ae63e974c..93408c067098b 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java @@ -323,6 +323,7 @@ public void infer( Model model, @Nullable String query, List input, + boolean stream, Map taskSettings, InputType inputType, TimeValue timeout, diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalService.java index 948117954a63f..746cb6e89fad0 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalService.java @@ -149,6 +149,7 @@ public void infer( Model model, @Nullable String query, List inputs, + boolean stream, Map taskSettings, InputType inputType, TimeValue timeout, diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/validation/SimpleServiceIntegrationValidator.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/validation/SimpleServiceIntegrationValidator.java index 6233a7e0b6b29..70f01e77b9369 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/validation/SimpleServiceIntegrationValidator.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/validation/SimpleServiceIntegrationValidator.java @@ -31,6 +31,7 @@ public void validate(InferenceService service, Model model, ActionListener { - @SuppressWarnings("unchecked") - ActionListener listener = (ActionListener) invocation.getArguments()[6]; + ActionListener listener = invocation.getArgument(7); listener.onResponse(new InferenceTextEmbeddingFloatResults(List.of())); return Void.TYPE; - }).when(service).infer(any(), any(), any(), any(), any(), any(), any()); + }).when(service).infer(any(), any(), any(), anyBoolean(), any(), any(), any(), any()); PlainActionFuture listener = new PlainActionFuture<>(); getEmbeddingSize(model, service, listener); @@ -878,12 +878,11 @@ public void testGetEmbeddingSize_ReturnsError_WhenTextEmbeddingByteResults_IsEmp when(model.getTaskType()).thenReturn(TaskType.TEXT_EMBEDDING); doAnswer(invocation -> { - @SuppressWarnings("unchecked") - ActionListener listener = (ActionListener) invocation.getArguments()[6]; + ActionListener listener = invocation.getArgument(7); listener.onResponse(new InferenceTextEmbeddingByteResults(List.of())); return Void.TYPE; - }).when(service).infer(any(), any(), any(), any(), any(), any(), any()); + }).when(service).infer(any(), any(), any(), anyBoolean(), any(), any(), any(), any()); PlainActionFuture listener = new PlainActionFuture<>(); getEmbeddingSize(model, service, listener); @@ -903,12 +902,11 @@ public void testGetEmbeddingSize_ReturnsSize_ForTextEmbeddingResults() { var textEmbedding = TextEmbeddingResultsTests.createRandomResults(); doAnswer(invocation -> { - @SuppressWarnings("unchecked") - ActionListener listener = (ActionListener) invocation.getArguments()[6]; + ActionListener listener = invocation.getArgument(7); listener.onResponse(textEmbedding); return Void.TYPE; - }).when(service).infer(any(), any(), any(), any(), any(), any(), any()); + }).when(service).infer(any(), any(), any(), anyBoolean(), any(), any(), any(), any()); PlainActionFuture listener = new PlainActionFuture<>(); getEmbeddingSize(model, service, listener); @@ -927,12 +925,11 @@ public void testGetEmbeddingSize_ReturnsSize_ForTextEmbeddingByteResults() { var textEmbedding = InferenceTextEmbeddingByteResultsTests.createRandomResults(); doAnswer(invocation -> { - @SuppressWarnings("unchecked") - ActionListener listener = (ActionListener) invocation.getArguments()[6]; + ActionListener listener = invocation.getArgument(7); listener.onResponse(textEmbedding); return Void.TYPE; - }).when(service).infer(any(), any(), any(), any(), any(), any(), any()); + }).when(service).infer(any(), any(), any(), anyBoolean(), any(), any(), any(), any()); PlainActionFuture listener = new PlainActionFuture<>(); getEmbeddingSize(model, service, listener); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockServiceTests.java index bbf34354e1818..297a42f9d1fa7 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockServiceTests.java @@ -671,6 +671,7 @@ public void testInfer_ThrowsErrorWhenModelIsNotAmazonBedrockModel() throws IOExc mockModel, null, List.of(""), + false, new HashMap<>(), InputType.INGEST, InferenceAction.Request.DEFAULT_TIMEOUT, @@ -721,6 +722,7 @@ public void testInfer_SendsRequest_ForEmbeddingsModel() throws IOException { model, null, List.of("abc"), + false, new HashMap<>(), InputType.INGEST, InferenceAction.Request.DEFAULT_TIMEOUT, @@ -762,6 +764,7 @@ public void testInfer_SendsRequest_ForChatCompletionModel() throws IOException { model, null, List.of("abc"), + false, new HashMap<>(), InputType.INGEST, InferenceAction.Request.DEFAULT_TIMEOUT, @@ -1025,6 +1028,7 @@ public void testInfer_UnauthorizedResponse() throws IOException { model, null, List.of("abc"), + false, new HashMap<>(), InputType.INGEST, InferenceAction.Request.DEFAULT_TIMEOUT, diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/anthropic/AnthropicServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/anthropic/AnthropicServiceTests.java index 5e32344ab3840..c3693c227c435 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/anthropic/AnthropicServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/anthropic/AnthropicServiceTests.java @@ -452,6 +452,7 @@ public void testInfer_ThrowsErrorWhenModelIsNotAValidModel() throws IOException mockModel, null, List.of(""), + false, new HashMap<>(), InputType.INGEST, InferenceAction.Request.DEFAULT_TIMEOUT, @@ -506,6 +507,7 @@ public void testInfer_SendsCompletionRequest() throws IOException { model, null, List.of("input"), + false, new HashMap<>(), InputType.INGEST, InferenceAction.Request.DEFAULT_TIMEOUT, diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioServiceTests.java index 6f33c36f42db4..bb736f592fbdb 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioServiceTests.java @@ -825,6 +825,7 @@ public void testInfer_ThrowsErrorWhenModelIsNotAzureAiStudioModel() throws IOExc mockModel, null, List.of(""), + false, new HashMap<>(), InputType.INGEST, InferenceAction.Request.DEFAULT_TIMEOUT, @@ -954,6 +955,7 @@ public void testInfer_WithChatCompletionModel() throws IOException { model, null, List.of("abc"), + false, new HashMap<>(), InputType.INGEST, InferenceAction.Request.DEFAULT_TIMEOUT, @@ -1004,6 +1006,7 @@ public void testInfer_UnauthorisedResponse() throws IOException { model, null, List.of("abc"), + false, new HashMap<>(), InputType.INGEST, InferenceAction.Request.DEFAULT_TIMEOUT, diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiServiceTests.java index b3fbd6fc9b425..142877c09180f 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiServiceTests.java @@ -601,6 +601,7 @@ public void testInfer_ThrowsErrorWhenModelIsNotAzureOpenAiModel() throws IOExcep mockModel, null, List.of(""), + false, new HashMap<>(), InputType.INGEST, InferenceAction.Request.DEFAULT_TIMEOUT, @@ -656,6 +657,7 @@ public void testInfer_SendsRequest() throws IOException, URISyntaxException { model, null, List.of("abc"), + false, new HashMap<>(), InputType.INGEST, InferenceAction.Request.DEFAULT_TIMEOUT, @@ -1051,6 +1053,7 @@ public void testInfer_UnauthorisedResponse() throws IOException, URISyntaxExcept model, null, List.of("abc"), + false, new HashMap<>(), InputType.INGEST, InferenceAction.Request.DEFAULT_TIMEOUT, diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceTests.java index aebc3e3776c40..a577a6664d39d 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceTests.java @@ -622,6 +622,7 @@ public void testInfer_ThrowsErrorWhenModelIsNotCohereModel() throws IOException mockModel, null, List.of(""), + false, new HashMap<>(), InputType.INGEST, InferenceAction.Request.DEFAULT_TIMEOUT, @@ -689,6 +690,7 @@ public void testInfer_SendsRequest() throws IOException { model, null, List.of("abc"), + false, new HashMap<>(), InputType.INGEST, InferenceAction.Request.DEFAULT_TIMEOUT, @@ -932,6 +934,7 @@ public void testInfer_UnauthorisedResponse() throws IOException { model, null, List.of("abc"), + false, new HashMap<>(), InputType.INGEST, InferenceAction.Request.DEFAULT_TIMEOUT, @@ -991,6 +994,7 @@ public void testInfer_SetsInputTypeToIngest_FromInferParameter_WhenTaskSettingsA model, null, List.of("abc"), + false, new HashMap<>(), InputType.INGEST, InferenceAction.Request.DEFAULT_TIMEOUT, @@ -1064,6 +1068,7 @@ public void testInfer_SetsInputTypeToIngestFromInferParameter_WhenModelSettingIs model, null, List.of("abc"), + false, CohereEmbeddingsTaskSettingsTests.getTaskSettingsMap(InputType.SEARCH, null), InputType.INGEST, InferenceAction.Request.DEFAULT_TIMEOUT, @@ -1135,6 +1140,7 @@ public void testInfer_DoesNotSetInputType_WhenNotPresentInTaskSettings_AndUnspec model, null, List.of("abc"), + false, new HashMap<>(), InputType.UNSPECIFIED, InferenceAction.Request.DEFAULT_TIMEOUT, diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceTests.java index 38124b3401aaa..0bbf2be7301d8 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceTests.java @@ -346,6 +346,7 @@ public void testInfer_ThrowsErrorWhenModelIsNotAValidModel() throws IOException mockModel, null, List.of(""), + false, new HashMap<>(), InputType.INGEST, InferenceAction.Request.DEFAULT_TIMEOUT, @@ -397,6 +398,7 @@ public void testInfer_SendsEmbeddingsRequest() throws IOException { model, null, List.of("input text"), + false, new HashMap<>(), InputType.INGEST, InferenceAction.Request.DEFAULT_TIMEOUT, diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioServiceTests.java index 89d6a010bbc07..5d79d0e01f401 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioServiceTests.java @@ -503,6 +503,7 @@ public void testInfer_ThrowsErrorWhenModelIsNotGoogleAiStudioModel() throws IOEx mockModel, null, List.of(""), + false, new HashMap<>(), InputType.INGEST, InferenceAction.Request.DEFAULT_TIMEOUT, @@ -578,6 +579,7 @@ public void testInfer_SendsCompletionRequest() throws IOException { model, null, List.of("input"), + false, new HashMap<>(), InputType.INGEST, InferenceAction.Request.DEFAULT_TIMEOUT, @@ -634,6 +636,7 @@ public void testInfer_SendsEmbeddingsRequest() throws IOException { model, null, List.of(input), + false, new HashMap<>(), InputType.INGEST, InferenceAction.Request.DEFAULT_TIMEOUT, @@ -775,6 +778,7 @@ public void testInfer_ResourceNotFound() throws IOException { model, null, List.of("abc"), + false, new HashMap<>(), InputType.INGEST, InferenceAction.Request.DEFAULT_TIMEOUT, diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceBaseServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceBaseServiceTests.java index 22c3b7895460a..168110ae8f7c7 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceBaseServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceBaseServiceTests.java @@ -69,6 +69,7 @@ public void testInfer_ThrowsErrorWhenModelIsNotHuggingFaceModel() throws IOExcep mockModel, null, List.of(""), + false, new HashMap<>(), InputType.INGEST, InferenceAction.Request.DEFAULT_TIMEOUT, diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceServiceTests.java index 5ea9f82e5b60c..d13dea2ab6b4c 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceServiceTests.java @@ -438,6 +438,7 @@ public void testInfer_SendsEmbeddingsRequest() throws IOException { model, null, List.of("abc"), + false, new HashMap<>(), InputType.INGEST, InferenceAction.Request.DEFAULT_TIMEOUT, @@ -481,6 +482,7 @@ public void testInfer_SendsElserRequest() throws IOException { model, null, List.of("abc"), + false, new HashMap<>(), InputType.INGEST, InferenceAction.Request.DEFAULT_TIMEOUT, diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/IbmWatsonxServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/IbmWatsonxServiceTests.java index e0936c778c7a7..a2de7c15d54da 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/IbmWatsonxServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/IbmWatsonxServiceTests.java @@ -409,6 +409,7 @@ public void testInfer_ThrowsErrorWhenModelIsNotIbmWatsonxModel() throws IOExcept mockModel, null, List.of(""), + false, new HashMap<>(), InputType.INGEST, InferenceAction.Request.DEFAULT_TIMEOUT, @@ -465,6 +466,7 @@ public void testInfer_SendsEmbeddingsRequest() throws IOException { model, null, List.of(input), + false, new HashMap<>(), InputType.INGEST, InferenceAction.Request.DEFAULT_TIMEOUT, @@ -588,6 +590,7 @@ public void testInfer_ResourceNotFound() throws IOException { model, null, List.of("abc"), + false, new HashMap<>(), InputType.INGEST, InferenceAction.Request.DEFAULT_TIMEOUT, diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/mistral/MistralServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/mistral/MistralServiceTests.java index 9d0fd954c44f9..33a2b43caf174 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/mistral/MistralServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/mistral/MistralServiceTests.java @@ -446,6 +446,7 @@ public void testInfer_ThrowsErrorWhenModelIsNotMistralEmbeddingsModel() throws I mockModel, null, List.of(""), + false, new HashMap<>(), InputType.INGEST, InferenceAction.Request.DEFAULT_TIMEOUT, @@ -571,6 +572,7 @@ public void testInfer_UnauthorisedResponse() throws IOException { model, null, List.of("abc"), + false, new HashMap<>(), InputType.INGEST, InferenceAction.Request.DEFAULT_TIMEOUT, diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/OpenAiServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/OpenAiServiceTests.java index a5e8c1d7eb26e..32099c4bd0be9 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/OpenAiServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/OpenAiServiceTests.java @@ -936,6 +936,7 @@ public void testInfer_ThrowsErrorWhenModelIsNotOpenAiModel() throws IOException mockModel, null, List.of(""), + false, new HashMap<>(), InputType.INGEST, InferenceAction.Request.DEFAULT_TIMEOUT, @@ -990,6 +991,7 @@ public void testInfer_SendsRequest() throws IOException { model, null, List.of("abc"), + false, new HashMap<>(), InputType.INGEST, InferenceAction.Request.DEFAULT_TIMEOUT, @@ -1470,6 +1472,7 @@ public void testInfer_UnauthorisedResponse() throws IOException { model, null, List.of("abc"), + false, new HashMap<>(), InputType.INGEST, InferenceAction.Request.DEFAULT_TIMEOUT, diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/validation/SimpleServiceIntegrationValidatorTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/validation/SimpleServiceIntegrationValidatorTests.java index ef295e4070cc3..767dd4d64a7d3 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/validation/SimpleServiceIntegrationValidatorTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/validation/SimpleServiceIntegrationValidatorTests.java @@ -64,6 +64,7 @@ public void testValidate_ServiceThrowsException() { eq(mockModel), eq(null), eq(TEST_INPUT), + eq(false), eq(Map.of()), eq(InputType.INGEST), eq(InferenceAction.Request.DEFAULT_TIMEOUT), @@ -94,7 +95,7 @@ public void testValidate_SuccessfulCallToServiceForReRankTaskType() { private void mockSuccessfulCallToService(String query, InferenceServiceResults result) { doAnswer(ans -> { - ActionListener responseListener = ans.getArgument(6); + ActionListener responseListener = ans.getArgument(7); responseListener.onResponse(result); return null; }).when(mockInferenceService) @@ -102,6 +103,7 @@ private void mockSuccessfulCallToService(String query, InferenceServiceResults r eq(mockModel), eq(query), eq(TEST_INPUT), + eq(false), eq(Map.of()), eq(InputType.INGEST), eq(InferenceAction.Request.DEFAULT_TIMEOUT), @@ -117,6 +119,7 @@ private void verifyCallToService(boolean withQuery) { eq(mockModel), eq(withQuery ? TEST_QUERY : null), eq(TEST_INPUT), + eq(false), eq(Map.of()), eq(InputType.INGEST), eq(InferenceAction.Request.DEFAULT_TIMEOUT), From f412de7e8ec93631b224e7d350907852e4433121 Mon Sep 17 00:00:00 2001 From: David Turner Date: Thu, 26 Sep 2024 17:32:17 +0100 Subject: [PATCH 23/43] Handle rejections in `IncrementalBulkIT` (#113599) Submitting a bare runnable to a threadpool risks an exception being thrown if the queue is full. This commit moves to submitting `AbstractRunnable` instances that won't be rejected. Closes #113365 --- muted-tests.yml | 3 - .../action/bulk/IncrementalBulkIT.java | 91 ++++++++++++------- 2 files changed, 60 insertions(+), 34 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index 5d7474af06d86..7a21957a044c0 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -257,9 +257,6 @@ tests: - class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT method: test {yaml=reference/ccr/apis/follow/post-resume-follow/line_84} issue: https://github.com/elastic/elasticsearch/issues/113343 -- class: org.elasticsearch.action.bulk.IncrementalBulkIT - method: testBulkLevelBulkFailureAfterFirstIncrementalRequest - issue: https://github.com/elastic/elasticsearch/issues/113365 - class: org.elasticsearch.xpack.ml.integration.MlJobIT method: testDeleteJob_TimingStatsDocumentIsDeleted issue: https://github.com/elastic/elasticsearch/issues/113370 diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/IncrementalBulkIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/IncrementalBulkIT.java index d7a5d4e2ac973..75f914f76dd77 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/IncrementalBulkIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/IncrementalBulkIT.java @@ -15,6 +15,7 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.elasticsearch.core.AbstractRefCounted; import org.elasticsearch.core.Releasable; @@ -37,6 +38,7 @@ import java.util.Map; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.CyclicBarrier; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.atomic.AtomicBoolean; @@ -214,14 +216,8 @@ public void testGlobalBulkFailure() throws InterruptedException { IncrementalBulkService incrementalBulkService = internalCluster().getInstance(IncrementalBulkService.class, randomNodeName); ThreadPool threadPool = internalCluster().getInstance(ThreadPool.class, randomNodeName); - int threadCount = threadPool.info(ThreadPool.Names.WRITE).getMax(); - long queueSize = threadPool.info(ThreadPool.Names.WRITE).getQueueSize().singles(); - blockWritePool(threadCount, threadPool, blockingLatch); - - Runnable runnable = () -> {}; - for (int i = 0; i < queueSize; i++) { - threadPool.executor(ThreadPool.Names.WRITE).execute(runnable); - } + blockWritePool(threadPool, blockingLatch); + fillWriteQueue(threadPool); IncrementalBulkService.Handler handler = incrementalBulkService.newBulkRequest(); if (randomBoolean()) { @@ -253,35 +249,32 @@ public void testBulkLevelBulkFailureAfterFirstIncrementalRequest() throws Except AbstractRefCounted refCounted = AbstractRefCounted.of(() -> {}); PlainActionFuture future = new PlainActionFuture<>(); - int threadCount = threadPool.info(ThreadPool.Names.WRITE).getMax(); - long queueSize = threadPool.info(ThreadPool.Names.WRITE).getQueueSize().singles(); - CountDownLatch blockingLatch1 = new CountDownLatch(1); AtomicBoolean nextRequested = new AtomicBoolean(true); AtomicLong hits = new AtomicLong(0); - try (Releasable ignored2 = blockingLatch1::countDown;) { - blockWritePool(threadCount, threadPool, blockingLatch1); + try { + blockWritePool(threadPool, blockingLatch1); while (nextRequested.get()) { nextRequested.set(false); refCounted.incRef(); handler.addItems(List.of(indexRequest(index)), refCounted::decRef, () -> nextRequested.set(true)); hits.incrementAndGet(); } + } finally { + blockingLatch1.countDown(); } assertBusy(() -> assertTrue(nextRequested.get())); CountDownLatch blockingLatch2 = new CountDownLatch(1); - try (Releasable ignored3 = blockingLatch2::countDown;) { - blockWritePool(threadCount, threadPool, blockingLatch2); - Runnable runnable = () -> {}; - // Fill Queue - for (int i = 0; i < queueSize; i++) { - threadPool.executor(ThreadPool.Names.WRITE).execute(runnable); - } + try { + blockWritePool(threadPool, blockingLatch2); + fillWriteQueue(threadPool); handler.lastItems(List.of(indexRequest(index)), refCounted::decRef, future); + } finally { + blockingLatch2.countDown(); } // Should not throw because some succeeded @@ -459,19 +452,55 @@ public void testShortCircuitShardLevelFailureWithIngestNodeHop() throws Exceptio } } - private static void blockWritePool(int threadCount, ThreadPool threadPool, CountDownLatch blockingLatch) throws InterruptedException { - CountDownLatch startedLatch = new CountDownLatch(threadCount); + private static void blockWritePool(ThreadPool threadPool, CountDownLatch finishLatch) { + final var threadCount = threadPool.info(ThreadPool.Names.WRITE).getMax(); + final var startBarrier = new CyclicBarrier(threadCount + 1); + final var blockingTask = new AbstractRunnable() { + @Override + public void onFailure(Exception e) { + fail(e); + } + + @Override + protected void doRun() { + safeAwait(startBarrier); + safeAwait(finishLatch); + } + + @Override + public boolean isForceExecution() { + return true; + } + }; for (int i = 0; i < threadCount; i++) { - threadPool.executor(ThreadPool.Names.WRITE).execute(() -> { - startedLatch.countDown(); - try { - blockingLatch.await(); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - }); + threadPool.executor(ThreadPool.Names.WRITE).execute(blockingTask); + } + safeAwait(startBarrier); + } + + private static void fillWriteQueue(ThreadPool threadPool) { + final var queueSize = Math.toIntExact(threadPool.info(ThreadPool.Names.WRITE).getQueueSize().singles()); + final var queueFilled = new AtomicBoolean(false); + final var queueFillingTask = new AbstractRunnable() { + @Override + public void onFailure(Exception e) { + fail(e); + } + + @Override + protected void doRun() { + assertTrue("thread pool not blocked", queueFilled.get()); + } + + @Override + public boolean isForceExecution() { + return true; + } + }; + for (int i = 0; i < queueSize; i++) { + threadPool.executor(ThreadPool.Names.WRITE).execute(queueFillingTask); } - startedLatch.await(); + queueFilled.set(true); } private BulkResponse executeBulk(long docs, String index, IncrementalBulkService.Handler handler, ExecutorService executorService) { From 80cc765090e4874a0f420d2b0026c801bd7554cb Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Fri, 27 Sep 2024 02:56:06 +1000 Subject: [PATCH 24/43] Mute org.elasticsearch.xpack.esql.qa.mixed.MixedClusterEsqlSpecIT test {date_nanos.Date_nanos to date nanos, index version SYNC} #113632 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 7a21957a044c0..df82cda4277b0 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -302,6 +302,9 @@ tests: - class: org.elasticsearch.integration.KibanaUserRoleIntegTests method: testSearchAndMSearch issue: https://github.com/elastic/elasticsearch/issues/113593 +- class: org.elasticsearch.xpack.esql.qa.mixed.MixedClusterEsqlSpecIT + method: test {date_nanos.Date_nanos to date nanos, index version SYNC} + issue: https://github.com/elastic/elasticsearch/issues/113632 # Examples: # From 861aca0524de1a82d373606fda5ccb0fd83ab122 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Thu, 26 Sep 2024 19:15:33 +0200 Subject: [PATCH 25/43] Also enable LogsdbIndexModeSettingsProvider in case of stateless. (#113624) --- .../main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java index 833555a7884ea..5cb7bf9e75252 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java @@ -49,7 +49,7 @@ public Collection createComponents(PluginServices services) { @Override public Collection getAdditionalIndexSettingProviders(IndexSettingProvider.Parameters parameters) { if (DiscoveryNode.isStateless(settings)) { - return List.of(); + return List.of(logsdbIndexModeSettingsProvider); } return List.of(new SyntheticSourceIndexSettingsProvider(licenseService), logsdbIndexModeSettingsProvider); } From 239cafe299b14781bfbdaab8b6adbde627dad7ac Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Thu, 26 Sep 2024 17:39:19 +0000 Subject: [PATCH 26/43] Bump versions after 8.15.2 release --- .buildkite/pipelines/intake.yml | 2 +- .buildkite/pipelines/periodic-packaging.yml | 9 ++++----- .buildkite/pipelines/periodic.yml | 12 ++++++------ .ci/bwcVersions | 2 +- .ci/snapshotBwcVersions | 2 +- server/src/main/java/org/elasticsearch/Version.java | 1 + .../org/elasticsearch/TransportVersions.csv | 1 + .../org/elasticsearch/index/IndexVersions.csv | 1 + 8 files changed, 16 insertions(+), 14 deletions(-) diff --git a/.buildkite/pipelines/intake.yml b/.buildkite/pipelines/intake.yml index e7ba4ba7610cd..a6af8bd35c7a0 100644 --- a/.buildkite/pipelines/intake.yml +++ b/.buildkite/pipelines/intake.yml @@ -62,7 +62,7 @@ steps: timeout_in_minutes: 300 matrix: setup: - BWC_VERSION: ["8.15.2", "8.16.0", "9.0.0"] + BWC_VERSION: ["8.15.3", "8.16.0", "9.0.0"] agents: provider: gcp image: family/elasticsearch-ubuntu-2004 diff --git a/.buildkite/pipelines/periodic-packaging.yml b/.buildkite/pipelines/periodic-packaging.yml index 76cc543a6898e..1c33337dc821f 100644 --- a/.buildkite/pipelines/periodic-packaging.yml +++ b/.buildkite/pipelines/periodic-packaging.yml @@ -30,8 +30,7 @@ steps: image: family/elasticsearch-{{matrix.image}} diskSizeGb: 350 machineType: n1-standard-8 - env: - USE_PROD_DOCKER_CREDENTIALS: "true" + env: {} - group: packaging-tests-upgrade steps: - label: "{{matrix.image}} / 8.0.1 / packaging-tests-upgrade" @@ -289,8 +288,8 @@ steps: env: BWC_VERSION: 8.14.3 - - label: "{{matrix.image}} / 8.15.2 / packaging-tests-upgrade" - command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.15.2 + - label: "{{matrix.image}} / 8.15.3 / packaging-tests-upgrade" + command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.15.3 timeout_in_minutes: 300 matrix: setup: @@ -304,7 +303,7 @@ steps: buildDirectory: /dev/shm/bk diskSizeGb: 250 env: - BWC_VERSION: 8.15.2 + BWC_VERSION: 8.15.3 - label: "{{matrix.image}} / 8.16.0 / packaging-tests-upgrade" command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.16.0 diff --git a/.buildkite/pipelines/periodic.yml b/.buildkite/pipelines/periodic.yml index 6afa69f8bf015..fa7e84fae160b 100644 --- a/.buildkite/pipelines/periodic.yml +++ b/.buildkite/pipelines/periodic.yml @@ -302,8 +302,8 @@ steps: - signal_reason: agent_stop limit: 3 - - label: 8.15.2 / bwc - command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.15.2#bwcTest + - label: 8.15.3 / bwc + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.15.3#bwcTest timeout_in_minutes: 300 agents: provider: gcp @@ -313,7 +313,7 @@ steps: preemptible: true diskSizeGb: 250 env: - BWC_VERSION: 8.15.2 + BWC_VERSION: 8.15.3 retry: automatic: - exit_status: "-1" @@ -431,7 +431,7 @@ steps: setup: ES_RUNTIME_JAVA: - openjdk17 - BWC_VERSION: ["8.15.2", "8.16.0", "9.0.0"] + BWC_VERSION: ["8.15.3", "8.16.0", "9.0.0"] agents: provider: gcp image: family/elasticsearch-ubuntu-2004 @@ -475,7 +475,7 @@ steps: ES_RUNTIME_JAVA: - openjdk21 - openjdk23 - BWC_VERSION: ["8.15.2", "8.16.0", "9.0.0"] + BWC_VERSION: ["8.15.3", "8.16.0", "9.0.0"] agents: provider: gcp image: family/elasticsearch-ubuntu-2004 @@ -584,7 +584,7 @@ steps: machineType: n2-standard-8 buildDirectory: /dev/shm/bk diskSizeGb: 250 - if: build.branch == "main" || build.branch == "8.x" || build.branch == "7.17" + if: build.branch == "main" || build.branch == "7.17" - label: check-branch-consistency command: .ci/scripts/run-gradle.sh branchConsistency timeout_in_minutes: 15 diff --git a/.ci/bwcVersions b/.ci/bwcVersions index 498727b3ecd39..de0505c61a251 100644 --- a/.ci/bwcVersions +++ b/.ci/bwcVersions @@ -14,6 +14,6 @@ BWC_VERSION: - "8.12.2" - "8.13.4" - "8.14.3" - - "8.15.2" + - "8.15.3" - "8.16.0" - "9.0.0" diff --git a/.ci/snapshotBwcVersions b/.ci/snapshotBwcVersions index a2f1e0c675ea5..24f58abc72493 100644 --- a/.ci/snapshotBwcVersions +++ b/.ci/snapshotBwcVersions @@ -1,4 +1,4 @@ BWC_VERSION: - - "8.15.2" + - "8.15.3" - "8.16.0" - "9.0.0" diff --git a/server/src/main/java/org/elasticsearch/Version.java b/server/src/main/java/org/elasticsearch/Version.java index 23436786126d8..fea27255e1328 100644 --- a/server/src/main/java/org/elasticsearch/Version.java +++ b/server/src/main/java/org/elasticsearch/Version.java @@ -185,6 +185,7 @@ public class Version implements VersionId, ToXContentFragment { public static final Version V_8_15_0 = new Version(8_15_00_99); public static final Version V_8_15_1 = new Version(8_15_01_99); public static final Version V_8_15_2 = new Version(8_15_02_99); + public static final Version V_8_15_3 = new Version(8_15_03_99); public static final Version V_8_16_0 = new Version(8_16_00_99); public static final Version V_9_0_0 = new Version(9_00_00_99); public static final Version CURRENT = V_9_0_0; diff --git a/server/src/main/resources/org/elasticsearch/TransportVersions.csv b/server/src/main/resources/org/elasticsearch/TransportVersions.csv index 19528a9719e22..44c752def351e 100644 --- a/server/src/main/resources/org/elasticsearch/TransportVersions.csv +++ b/server/src/main/resources/org/elasticsearch/TransportVersions.csv @@ -129,3 +129,4 @@ 8.14.3,8636001 8.15.0,8702002 8.15.1,8702002 +8.15.2,8702003 diff --git a/server/src/main/resources/org/elasticsearch/index/IndexVersions.csv b/server/src/main/resources/org/elasticsearch/index/IndexVersions.csv index 2e684719688f9..971940041f9b1 100644 --- a/server/src/main/resources/org/elasticsearch/index/IndexVersions.csv +++ b/server/src/main/resources/org/elasticsearch/index/IndexVersions.csv @@ -129,3 +129,4 @@ 8.14.3,8505000 8.15.0,8512000 8.15.1,8512000 +8.15.2,8512000 From 8d6ecdc75d67030b3ad89bdf232808ce216ad3eb Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Thu, 26 Sep 2024 17:40:46 +0000 Subject: [PATCH 27/43] Prune changelogs after 8.15.2 release --- docs/changelog/111519.yaml | 5 ----- docs/changelog/111535.yaml | 5 ----- docs/changelog/111548.yaml | 6 ------ docs/changelog/111932.yaml | 6 ------ docs/changelog/112400.yaml | 5 ----- docs/changelog/112444.yaml | 6 ------ docs/changelog/112581.yaml | 5 ----- docs/changelog/112610.yaml | 6 ------ docs/changelog/112649.yaml | 5 ----- docs/changelog/112703.yaml | 5 ----- docs/changelog/112713.yaml | 5 ----- docs/changelog/112720.yaml | 5 ----- docs/changelog/112872.yaml | 6 ------ 13 files changed, 70 deletions(-) delete mode 100644 docs/changelog/111519.yaml delete mode 100644 docs/changelog/111535.yaml delete mode 100644 docs/changelog/111548.yaml delete mode 100644 docs/changelog/111932.yaml delete mode 100644 docs/changelog/112400.yaml delete mode 100644 docs/changelog/112444.yaml delete mode 100644 docs/changelog/112581.yaml delete mode 100644 docs/changelog/112610.yaml delete mode 100644 docs/changelog/112649.yaml delete mode 100644 docs/changelog/112703.yaml delete mode 100644 docs/changelog/112713.yaml delete mode 100644 docs/changelog/112720.yaml delete mode 100644 docs/changelog/112872.yaml diff --git a/docs/changelog/111519.yaml b/docs/changelog/111519.yaml deleted file mode 100644 index 8cc62fb8ed903..0000000000000 --- a/docs/changelog/111519.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 111519 -summary: "ESQL: Don't mutate the `BoolQueryBuilder` in plan" -area: ES|QL -type: bug -issues: [] diff --git a/docs/changelog/111535.yaml b/docs/changelog/111535.yaml deleted file mode 100644 index 4beebbf28d4e1..0000000000000 --- a/docs/changelog/111535.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 111535 -summary: Fix remote cluster credential secure settings reload -area: Authorization -type: bug -issues: [] diff --git a/docs/changelog/111548.yaml b/docs/changelog/111548.yaml deleted file mode 100644 index ca9e5ae622894..0000000000000 --- a/docs/changelog/111548.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 111548 -summary: Json parsing exceptions should not cause 500 errors -area: Infra/Core -type: bug -issues: - - 111542 diff --git a/docs/changelog/111932.yaml b/docs/changelog/111932.yaml deleted file mode 100644 index ce840ecebcff0..0000000000000 --- a/docs/changelog/111932.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 111932 -summary: Fix union-types where one index is missing the field -area: ES|QL -type: bug -issues: - - 111912 diff --git a/docs/changelog/112400.yaml b/docs/changelog/112400.yaml deleted file mode 100644 index 6d622e5fb5248..0000000000000 --- a/docs/changelog/112400.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 112400 -summary: Make sure file accesses in `DnRoleMapper` are done in stack frames with permissions -area: Infra/Core -type: bug -issues: [] diff --git a/docs/changelog/112444.yaml b/docs/changelog/112444.yaml deleted file mode 100644 index bfa4fd693f0e0..0000000000000 --- a/docs/changelog/112444.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 112444 -summary: Full coverage of ECS by ecs@mappings when `date_detection` is disabled -area: Mapping -type: bug -issues: - - 112398 diff --git a/docs/changelog/112581.yaml b/docs/changelog/112581.yaml deleted file mode 100644 index 489b4780c06fb..0000000000000 --- a/docs/changelog/112581.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 112581 -summary: Fix missing header in `put_geoip_database` JSON spec -area: Ingest Node -type: bug -issues: [] diff --git a/docs/changelog/112610.yaml b/docs/changelog/112610.yaml deleted file mode 100644 index 3d67a80a8f0b3..0000000000000 --- a/docs/changelog/112610.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 112610 -summary: Support widening of numeric types in union-types -area: ES|QL -type: bug -issues: - - 111277 diff --git a/docs/changelog/112649.yaml b/docs/changelog/112649.yaml deleted file mode 100644 index e3cf1e8e34881..0000000000000 --- a/docs/changelog/112649.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 112649 -summary: Allowlist `tracestate` header on remote server port -area: Security -type: bug -issues: [] diff --git a/docs/changelog/112703.yaml b/docs/changelog/112703.yaml deleted file mode 100644 index a428e8c4e2339..0000000000000 --- a/docs/changelog/112703.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 112703 -summary: JSON parse failures should be 4xx codes -area: Infra/Core -type: bug -issues: [] diff --git a/docs/changelog/112713.yaml b/docs/changelog/112713.yaml deleted file mode 100644 index 1ccf451b13f82..0000000000000 --- a/docs/changelog/112713.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 112713 -summary: Fix encoding of dynamic arrays in ignored source -area: Logs -type: bug -issues: [] diff --git a/docs/changelog/112720.yaml b/docs/changelog/112720.yaml deleted file mode 100644 index a44ea5a699520..0000000000000 --- a/docs/changelog/112720.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 112720 -summary: Fix NPE in `dense_vector` stats -area: Vector Search -type: bug -issues: [] diff --git a/docs/changelog/112872.yaml b/docs/changelog/112872.yaml deleted file mode 100644 index 5a6f3af03961d..0000000000000 --- a/docs/changelog/112872.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 112872 -summary: Fix parsing error in `_terms_enum` API -area: Search -type: bug -issues: - - 94378 From 6040e4605442c5ab34e722578c3ac258bb3eddb1 Mon Sep 17 00:00:00 2001 From: Joe Gallo Date: Thu, 26 Sep 2024 14:08:42 -0400 Subject: [PATCH 28/43] Tidy up database_type parsing (#113633) --- .../elasticsearch/ingest/geoip/Database.java | 45 +++++++++++------- .../ingest/geoip/GeoIpProcessor.java | 38 +++++++-------- ...ansportGetDatabaseConfigurationAction.java | 2 +- .../ingest/geoip/MMDBUtilTests.java | 35 ++++++++++++++ .../test/resources/GeoIP2-Country-Test.mmdb | Bin 0 -> 19744 bytes .../src/test/resources/GeoLite2-ASN-Test.mmdb | Bin 0 -> 12653 bytes .../test/resources/GeoLite2-Country-Test.mmdb | Bin 0 -> 18041 bytes .../upgrades/SystemIndexMigrationInfo.java | 2 +- 8 files changed, 83 insertions(+), 39 deletions(-) create mode 100644 modules/ingest-geoip/src/test/resources/GeoIP2-Country-Test.mmdb create mode 100644 modules/ingest-geoip/src/test/resources/GeoLite2-ASN-Test.mmdb create mode 100644 modules/ingest-geoip/src/test/resources/GeoLite2-Country-Test.mmdb diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/Database.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/Database.java index 6bf38fbd34cfe..dccda0d58cfbf 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/Database.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/Database.java @@ -9,6 +9,7 @@ package org.elasticsearch.ingest.geoip; +import org.elasticsearch.common.Strings; import org.elasticsearch.core.Nullable; import java.util.Arrays; @@ -150,6 +151,29 @@ enum Database { private static final String ENTERPRISE_DB_SUFFIX = "-Enterprise"; private static final String ISP_DB_SUFFIX = "-ISP"; + @Nullable + private static Database getMaxmindDatabase(final String databaseType) { + if (databaseType.endsWith(Database.CITY_DB_SUFFIX)) { + return Database.City; + } else if (databaseType.endsWith(Database.COUNTRY_DB_SUFFIX)) { + return Database.Country; + } else if (databaseType.endsWith(Database.ASN_DB_SUFFIX)) { + return Database.Asn; + } else if (databaseType.endsWith(Database.ANONYMOUS_IP_DB_SUFFIX)) { + return Database.AnonymousIp; + } else if (databaseType.endsWith(Database.CONNECTION_TYPE_DB_SUFFIX)) { + return Database.ConnectionType; + } else if (databaseType.endsWith(Database.DOMAIN_DB_SUFFIX)) { + return Database.Domain; + } else if (databaseType.endsWith(Database.ENTERPRISE_DB_SUFFIX)) { + return Database.Enterprise; + } else if (databaseType.endsWith(Database.ISP_DB_SUFFIX)) { + return Database.Isp; + } else { + return null; // no match was found + } + } + /** * Parses the passed-in databaseType (presumably from the passed-in databaseFile) and return the Database instance that is * associated with that databaseType. @@ -161,24 +185,9 @@ enum Database { */ public static Database getDatabase(final String databaseType, final String databaseFile) { Database database = null; - if (databaseType != null) { - if (databaseType.endsWith(Database.CITY_DB_SUFFIX)) { - database = Database.City; - } else if (databaseType.endsWith(Database.COUNTRY_DB_SUFFIX)) { - database = Database.Country; - } else if (databaseType.endsWith(Database.ASN_DB_SUFFIX)) { - database = Database.Asn; - } else if (databaseType.endsWith(Database.ANONYMOUS_IP_DB_SUFFIX)) { - database = Database.AnonymousIp; - } else if (databaseType.endsWith(Database.CONNECTION_TYPE_DB_SUFFIX)) { - database = Database.ConnectionType; - } else if (databaseType.endsWith(Database.DOMAIN_DB_SUFFIX)) { - database = Database.Domain; - } else if (databaseType.endsWith(Database.ENTERPRISE_DB_SUFFIX)) { - database = Database.Enterprise; - } else if (databaseType.endsWith(Database.ISP_DB_SUFFIX)) { - database = Database.Isp; - } + + if (Strings.hasText(databaseType)) { + database = getMaxmindDatabase(databaseType); } if (database == null) { diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java index 3e4f2be5be8d5..ce160b060ae4c 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java @@ -106,14 +106,14 @@ boolean isIgnoreMissing() { } @Override - public IngestDocument execute(IngestDocument ingestDocument) throws IOException { - Object ip = ingestDocument.getFieldValue(field, Object.class, ignoreMissing); + public IngestDocument execute(IngestDocument document) throws IOException { + Object ip = document.getFieldValue(field, Object.class, ignoreMissing); if (isValid.get() == false) { - ingestDocument.appendFieldValue("tags", "_geoip_expired_database", false); - return ingestDocument; + document.appendFieldValue("tags", "_geoip_expired_database", false); + return document; } else if (ip == null && ignoreMissing) { - return ingestDocument; + return document; } else if (ip == null) { throw new IllegalArgumentException("field [" + field + "] is null, cannot extract geoip information."); } @@ -121,44 +121,44 @@ public IngestDocument execute(IngestDocument ingestDocument) throws IOException try (IpDatabase ipDatabase = this.supplier.get()) { if (ipDatabase == null) { if (ignoreMissing == false) { - tag(ingestDocument, databaseFile); + tag(document, databaseFile); } - return ingestDocument; + return document; } if (ip instanceof String ipString) { - Map geoData = getGeoData(ipDatabase, ipString); - if (geoData.isEmpty() == false) { - ingestDocument.setFieldValue(targetField, geoData); + Map data = getGeoData(ipDatabase, ipString); + if (data.isEmpty() == false) { + document.setFieldValue(targetField, data); } } else if (ip instanceof List ipList) { boolean match = false; - List> geoDataList = new ArrayList<>(ipList.size()); + List> dataList = new ArrayList<>(ipList.size()); for (Object ipAddr : ipList) { if (ipAddr instanceof String == false) { throw new IllegalArgumentException("array in field [" + field + "] should only contain strings"); } - Map geoData = getGeoData(ipDatabase, (String) ipAddr); - if (geoData.isEmpty()) { - geoDataList.add(null); + Map data = getGeoData(ipDatabase, (String) ipAddr); + if (data.isEmpty()) { + dataList.add(null); continue; } if (firstOnly) { - ingestDocument.setFieldValue(targetField, geoData); - return ingestDocument; + document.setFieldValue(targetField, data); + return document; } match = true; - geoDataList.add(geoData); + dataList.add(data); } if (match) { - ingestDocument.setFieldValue(targetField, geoDataList); + document.setFieldValue(targetField, dataList); } } else { throw new IllegalArgumentException("field [" + field + "] should contain only string or array of strings"); } } - return ingestDocument; + return document; } private Map getGeoData(IpDatabase ipDatabase, String ipAddress) throws IOException { diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportGetDatabaseConfigurationAction.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportGetDatabaseConfigurationAction.java index 730ae6d8b8ae5..0660a9ff0491d 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportGetDatabaseConfigurationAction.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportGetDatabaseConfigurationAction.java @@ -74,7 +74,7 @@ protected void doExecute( * TransportGetDatabaseConfigurationAction used to be a TransportMasterNodeAction, and not all nodes in the cluster have been * updated. So we don't want to send node requests to the other nodes because they will blow up. Instead, we just return * the information that we used to return from the master node (it doesn't make any difference that this might not be the master - * node, because we're only reading the clsuter state). + * node, because we're only reading the cluster state). */ newResponseAsync(task, request, createActionContext(task, request), List.of(), List.of(), listener); } else { diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MMDBUtilTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MMDBUtilTests.java index 5473eab35aa2c..d441b749f4225 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MMDBUtilTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MMDBUtilTests.java @@ -66,4 +66,39 @@ public void testSmallFileWithALongDescription() throws IOException { // it was once the case that we couldn't process an mmdb that was smaller than 512 bytes assertThat(Files.size(database), is(444L)); // 444 is <512 } + + public void testDatabaseTypeParsing() throws IOException { + // this test is a little bit overloaded -- it's testing that we're getting the expected sorts of + // database_type strings from these files, *and* it's also testing that we dispatch on those strings + // correctly and associated those files with the correct high-level Elasticsearch Database type. + // down the road it would probably make sense to split these out and find a better home for some of the + // logic, but for now it's probably more valuable to have the test *somewhere* than to get especially + // pedantic about where precisely it should be. + + copyDatabase("GeoLite2-City-Test.mmdb", tmpDir); + copyDatabase("GeoLite2-Country-Test.mmdb", tmpDir); + copyDatabase("GeoLite2-ASN-Test.mmdb", tmpDir); + copyDatabase("GeoIP2-Anonymous-IP-Test.mmdb", tmpDir); + copyDatabase("GeoIP2-City-Test.mmdb", tmpDir); + copyDatabase("GeoIP2-Country-Test.mmdb", tmpDir); + copyDatabase("GeoIP2-Connection-Type-Test.mmdb", tmpDir); + copyDatabase("GeoIP2-Domain-Test.mmdb", tmpDir); + copyDatabase("GeoIP2-Enterprise-Test.mmdb", tmpDir); + copyDatabase("GeoIP2-ISP-Test.mmdb", tmpDir); + + assertThat(parseDatabaseFromType("GeoLite2-City-Test.mmdb"), is(Database.City)); + assertThat(parseDatabaseFromType("GeoLite2-Country-Test.mmdb"), is(Database.Country)); + assertThat(parseDatabaseFromType("GeoLite2-ASN-Test.mmdb"), is(Database.Asn)); + assertThat(parseDatabaseFromType("GeoIP2-Anonymous-IP-Test.mmdb"), is(Database.AnonymousIp)); + assertThat(parseDatabaseFromType("GeoIP2-City-Test.mmdb"), is(Database.City)); + assertThat(parseDatabaseFromType("GeoIP2-Country-Test.mmdb"), is(Database.Country)); + assertThat(parseDatabaseFromType("GeoIP2-Connection-Type-Test.mmdb"), is(Database.ConnectionType)); + assertThat(parseDatabaseFromType("GeoIP2-Domain-Test.mmdb"), is(Database.Domain)); + assertThat(parseDatabaseFromType("GeoIP2-Enterprise-Test.mmdb"), is(Database.Enterprise)); + assertThat(parseDatabaseFromType("GeoIP2-ISP-Test.mmdb"), is(Database.Isp)); + } + + private Database parseDatabaseFromType(String databaseFile) throws IOException { + return Database.getDatabase(MMDBUtil.getDatabaseType(tmpDir.resolve(databaseFile)), null); + } } diff --git a/modules/ingest-geoip/src/test/resources/GeoIP2-Country-Test.mmdb b/modules/ingest-geoip/src/test/resources/GeoIP2-Country-Test.mmdb new file mode 100644 index 0000000000000000000000000000000000000000..0b1f6cf50b2a035d73e426268649a650d989e390 GIT binary patch literal 19744 zcmZXa2Ygh;_Q%iNy>~&H2w1V)&K_W;miXejY zDk5DvsB5o_y}l=gBw+WcPkpxkZ{~MzcH>_^@V#?p&eSvK%*-NUi1_x2c!1|by!fgV zF`2RmAJL3xjw08KoWfKKq9xIaXic;sP9@qBrxERl_CyEbbm9!+OyVr!Y~mcEBXKTq z9&tW#0bvtKL?@y%(S_(rbR#Y#l8NrbMMMvxC((=OO*1q1Tt3VNeNL(lo91374edHQ5Y79#D$e2RjhLyF`k$}Ow`Z~O``hU zBKO3F_iCZiO7&&&yvIeB$5SgrR>o7S zsJEI}BeIt2>*7V$Gj9X2QDl?OJ7qgdZjP6HQe+EywvQ#BBDRWbix+{?4(jbBcE$5{ zi#$!?9%64}UW>n3*?!`H$ic=uIV5tJ3P*@%;(5=Cz`)cLMTzI*c`vZC7e!uT-pj-* z#H++>#OuTx#GAxhL`(yH)D^;xzUU@jIMFLZem;*PbG`d zJ)XLV3O&T=sf9+bcwTQYE{>-z5#!Q$>N4tGE=C{fT@lZ_QjDu8yqdU%xR$t1qty>g z-JqeTHARdYF$TXFH^qgiY$%OLC;Ac@L?)3%WNWB->4(Y&ijl*Ha*6%~%qKB}#}=qKqgfDhS$=5oRhvR1#IF`ff4CF*Tl; zKujbiX(Y@UK8<@Dg~ojx-~D1trot4Wny87Fe1IjViZP9O4-(VkMHv573$xOR*SKQ3Txwe>%>^!m_pUY#>SMfNsK4Zm#10hX5vXPwy>6`h^=vjZDMSva7SFY zlX|;|-O`%d(AdM0dx?F-e&PUekT^sz{uzvadIq0iozD{Je3T`hCtlES=FE6W#G?gb zye!5m7+9?sud?K8#OsX;#v5Y18Be{%7GlIv;%(v`;$7lBG2V~seIUk%@zh5w^0647 z#D$-V@mW0eFEJ4N)H-}l978|9(0Z-El-A!-c$}ys>QUq;F&dac{8Q>Di7$z-G}KHO zUyJb#h2IiRos55mb3qwjGrm|Z==xoMfc43mH15z#y{itc#%Jt_b2fe z@wc>Tu8VjW|2$k7k5|NVmM-Ek#bY(r;_->685JD8QFOIb*gyU zvdC$1VLLX|UOXL`cX~XJ@z2BfhdAIlJDzurcsf#n@y~M}Q|A*G#7o-ZNusb*T-ce- zb`eik=5-@3B$A2l#6?68qNhfi(HeS~y;<4C#3jU~1mmCQa;EwaR}fbcR}ohe*AUkd z*J;R|;<5gUBSZh-|`7^doYJT%tcQfEY*&B5o!a|2&L; zo*@(tC2k=Y|2)H(x{Vl4o5&>c)QAiXKL1GlaRq+%v6(YtEV~G-? zlqe(0i3;K_g7MG8_@`D{O?eeD4(}uniih#fGl9a11mmCQZl>-b?j`Oc?k6UrmT7v4 zJyXO}O<@i305O%AhLU%S=RqJKp6Mc<1>%{(yqUx-Vm2{{m`ltf=4-&5^iUom5dXYg z#e?|g#SVW2RWtB;7O`iG#q%gjF41W75>rc2@^A4hW9o5YIkAFRDRL^RUPY`X)(~rn zb;NpN1F@0VL_9%kCY~g=5Kj?XiEYGoVh6F4*hTCno+c3gR0sDG`v}B8B|JbJBn}aW zi6g`_#Ipn}MCnC|=ZP1H7YR(AD)KV%3h^rO8u2>u2Jt5G77-(k5^odl5bqN25$_Wp z5FZjB5g!wu5T6pC5&t6oO?*xqBfcPNiQ_~aQBO1wCy0~8m&8}Z*Tgr(x5RhE_rwpx ze~2H6pNOA{{}R6tzY@O@|08}U{viG&{?fpWPOXDiVDnz_8roXD9u3Hq-Fr>Kq7Vz9 zgw3FEhIpF;t;Bl@g)NAd8qV(UwnmZm;%x)86?HrFo~og0ISr|HZpyj2c{{L{)0@F0_1|x5{c!vPDigzfB+@jGI6QUM;7;&41+fY6f?hxtP5~7sAMo=9rCn^Y7s}hEZ2vJE?5#uy;C&r6+0)-Qa zNgB@KQWv>Ly!XZxc>nX>FW$*<;S|-Rc&mvT@jk%HrZyH)`J52%^hTj~hInVPM&=!O=E_n~;|VK%!^ypQ0Qk1gU|gud`S(z}>>j}l9W$Hco7MOHAiOuUb? z1;)R&Y{LVz8F5ZuU&&2x)g`aAi@flNye`=EOM5HGrQ(F6n4~bD?5+E^EJ*YXUaw`7mAq#bP=-? zg`G8Ay{;5?(>RU$$V`Sp4>7wl?;;In!Ofl&_M!qU#Jm^^Wnx|e3=|Xb&%8;@%YbXd zyd1bv%s$k+LgOr~1MJMailEW9o66L+P`E+N>zKM;!_7;f@J0=1HTro3~!YqxmTQcQ`Lavzon95<^ET;Ms1DXg2L7_lQ#6NSGn0dfZF$c5c5Dj-) zZ=vv34R_&SQs(Vq4rgAz#yQQI8bREl;g0rBD2x^}02GNil0^zN+=hY_j?!?awHOLx z#SAfTjD}lh35BIigs>NLu9y|T{bC~inG?he165*1s8HENVH}0yHO_YCY$6oy5pxpr z?rx%RFNODMINrvb421{9oB})`W;Kh{Xqdm3hQJ)8eb0x4w%vCJ1TEpE(Ybji(;m+U&C_EwNM&@nOa0k7a!Y4J{S`h!N9b#?;u*bGR z{YNpk12Hjo0MClK6WA~2E?|$CyIITA8m`mXOW{6^jtiMO0ENS19%Sl}#<}zb<`D{? z(Qv)Sb5M9i%qZ}pn9sAw3mWbiUZU`2jduTK>QyMbA?9mLy{_RZyh-6(8g9>yLgBcW zZv&r+`3~^0nC}7~i1{8B-q&z9*M}5-q;ak@zE3FpRKwj)|3WRt#QZn&K5vru1%W;;@NZj)Nhr|^O%C6iD~7qL1qud~K^G&id&h21pVz9d88VzIgd zJ;kEUTRn7<^XU$)UexQY;mode2^20D>r&=jrr~UDs}F@&XgH>CT?K`RSXTo>#JUE^ z5DW3oN)_umAVsX}sds~h+t7^^-o(nB^-H5Lop9oxlb4Cg`iqqX^b;$aMf@5UzTO2irkgv1)nBE>A?jC~A+V>R4{N}*68RvGik zn<(5xVOYa8w@N5HELIipfLP;zd&L?LOcHAX6(%-OxSPUzG+fKN4+>Mnx}SNIHJtOO zRZU@yhMme3#$e4BYZ@>^tOqHauHkN{nH0{_aLsKF6y}RHmwEFvT!*|E!f_EvLc?4cCV-{#mQ3;P?>4KWnpC z>wt}7t*71w7I7Ai@y~jqi4gJ6+A7u-DmIf8G6zdrvD%P_UKBwW%662rsf)=_~ z_YzCKtl_$#S5eDRv0ek-6pQiCdZUTnThxnbxXbu96y6gH@lOr2{c4Yx}l zLE(g09|On4`ULowSf8@wXBuwR|EBPB4d<+3eF23!v1*xjT*Gazp27wV*CtLv;d`;Z z1ilgLD;D`$!?oRSDf~{uZT$x*{3O${Y~%QT!d^7TOqKlNR~yepZ<`Se{);WZR;ZGG1vHA#He1Nq{+0mv0!3V?sk zawCu?zMBB73SFh!>vRhHYPe(1M9MF|Eaox(Ip*f;M`4bJ+ui<14H4e};AZg+WD&+c zXFvGzC>*SD<|w9y0>i|23sbi;?>43!3~wU59Vz^?%n?A5`0fA#;w#Wae0OR%9@;mO z!a@zl<@th0;a_|@t&i4u?!ZD+7^C41tOTiu_)38a@s+VixrVEE7lmOB*IFx)8ZW*o z=8e;E4y?Wj6i(D|8@d}Q{E~JLFiU**0{G`F_W^k7ydS{AP1bOZ4!&v%Ycx7dWl916 z)N>kB4{ErZat4JnHQavAMhefFbAb8co692eG~Dqmpzt9IorV@7g{RI(n72s7?Zl%L zF41r<6~3iNJt4kjz#8#A4y+X4asUHcq2U@IC#h<~cPdo>b%)A#LH za`7D?4iblm!^9EsJwtU>@>vS;+4R3!e_?r9WvDDzR$1G(u)HXkmNB&7=wNwSpfp$z zDvB0_duFKb$ceO~U}i>DxV$2e7A)geFfACdd!>yE+gGKH2@I^Q*;re1u(oDPZB0#W z&4P@I%3f)C$PZWL#vY07I666YICdnqEn`w~uk-<#^-pf7-?byFu)M0QGCZ-??hsAB zVnuc+QeL3Dk(Fk5K4D*U>n*`>FjN*AU5Tc0qEF)s>B3^W&w!z2p~_&9of9e>T~uCb zr{v_JV7U!(QMsL-lUF`5P*oc2g|wZWTU))hw)&~snkQ*2V+Ue; zWBbvs?daX!qt&r(u|wcIJCb=~*7mxkhw{R~(V<8sI#Gmv>8>6}S9`ZuKO__>2$dBC z)szGSWd&7bq4F}LnLRHpGf#KDUuHN^HrDBFrb-kB?UXG25enGpNyT>d(At_uYHK#t z*6c*T>ZWe5duTQqwexkU=!Sq@I3TtbT|>8yPIEd|zw;54skMz-yH(nNRP?4G+)KBZ zGoUCd8IiLEk&cKXc}-c=Q}i>j2c$j%;E zH+yky_3GN1ReHWQ+xbo@)Pkv{X5k1=Y!xPOM{GCxcPO?ElNOsFJE)UtPPf?+KPqZC zJQvGsPgVZHht*eRCLl=#le*UwZbU_G9s0MqVforF@klB&OlZ~En`H@ zS3#^%=CY{i^orz4+VC&xlS@*@f8{)E;N%kMDo1aV+i6I|pM)8=|YsOe?Oc44}@= znRZHMT2*CqRhgZRM0Q?n%_6las*hle8|Kc^z11bwl-q^eR2eyz2kQ=$m6uH{ zEw73cRD{bXOf;HtHj|Fn9g?2col1Jc?vwP1ouW-G7;x|~{Dee@4Dw?&&|*FDM_A9H-*K0DI&?OPZ;C)Wa+fjFTR^3yF zlk9F|28V+c;a~)Du8Ak_RASHT6t+8e8sE2aVnr~!6zx_9!V`~mY1`#Y{B^9$c%zwO zKfNI)76u}f+DyA#qL;#%vhG+?%ZHs zYgYrGjSS4o!f+qT6HC_i&_GU{IzU&!dW!RJwbzd^LBet`<*#ge5p+!zwkop56r z_$V)ft<*1GSHJ8@J74E0_mvylis?S=I+A4Rf!hB_tlgW6+H)8W2RJ4=*3Kyp2VuN9 zgUhQBpqNS-lCGpZl_9%Ei!GeqAL8gH9i0YOAvopMt=({P#l!X67HGS@MQ4Zpp@%&+ zH1<^NX*VuC7CVeN*sbIG$rYQRnPex+@PwY8U3m~a9dfIlhs+^?(0Dx&#_5RIDy1Dd zzM@-IMs{Z1ylwUS4%_+3C;^wc8WX$^UPQ&|*>)t^xC#vzW;95exh#U6xH|(34=55`yP$^b2)qdNH~BKMxSKQV2lZWrDrKobaRO| zRUE|jDO0V+X>qe&L{$cZd^EO4TPk+4nneHc1M5!A*ikoKjn$Lpj5X=XeN1c)}xGFS>ks%a`VBiCFWKdn9W5$sS0bz?ia^*1~ z#P%U1V8=KkS7BkzX4RF$PU~IPB-vfoBu+!=kLXI8)0MP9F=DCOPTt`N5Gonngmk+8 z3`V!Yju5Hm>(KP1{(C1D9EC9#3()dZWwS7=LpU4M?b%hoTup@6pD_0GUmlJI^M=x> z@~R>c#-&X4iP)Bsfe$d90*u?Qj_1{T^r0XtFS=hB!$w|-m7a2R3dia{e&hih3hEba zRSoWU8sxELkP{CFaj`Q7m6vEU164>&$-+muI%uTpFE>EpebhpoRFE+mB;g33`s{cvK(C5KxrM@Yg>u4_>Ek`X<$x#OY|! zu9wq(COkA^Kz3%RtjswZ;FD8=V5Yy+q^JT)ilxYRW7O@=X`$#QB$e5qYY4YXoM@uXu6q(@slV3i)o40{V5;gGg^hxv>=d7HE)E@n<5aM)IHcenW5(ismp%znS5RoD3`&R$>Ciy8{X+_iqnqxk zQhmlrdx|>fRVN+}l6{-)!V$jM1~~ueFmGH-)IDV-l2c+wa1P$?+yG&41MBBtptBms zipBPoqUY}6#rW$*Ost>5Y`7oyIXl^p4OiN<=iPC$(3vT>`@<@1Bq#J6<)f=gaUkn% zr;qTkj=bn5eRNZIu#m1#-^PO7?d%bzK0I>U9ME@I?K*2#Huloazx^NeuU7S|iyE3= zqb@Ly!jB*@*^yz#56-M#wpjhWXe&&ou@Vfm*@X!<)MDmn3>AkGWz$(h$0|P-gd=O- z85?|PA%?DuMkUn|8?j@ncIY_g&TOpH&d)|kSgEtqpmW>XC;Z(D-tJIe|daJLnz5`{cU>bHTc%Hsh z!?EJbcrc+`S~JXEsV4wLRP>BHy>9mOh8at^)0;0nrowB2?RFcu-RVsR@32{eabBXk z!54gcN>)asA`sn<+0kFx{o%BSReMY?B|N;&C{yQtggj-2YH^(QPpmuAFmDUGfLS@E zdqUGKdrx+nPRr7(;!mq8!T5FK+0Hi{e)1_Vr~u>S>79&oAr6Il9-8<&T|;!0o(Z*! z;d>G08v|bM@gtAd&0LAOz_n0yXRXtn&I^<5&I=RzcF}YD+`jRWq`b^%-c`RI|lZ3}TI;i~~wq~)uLE7Gk2 z{mA7LURG`h{lE>PPgZV_UnyC+(Jkr|w+o%CPogkUbOYzn#3e~?HD29Jm*O6ykJkvt zClAb3EpJxgxK&=lT6F*A1!pY=a0+q;gz!e7&wy}WIRk=~#Y`ap!dIw3tn6`6U}B_K z8t$6vCWS?zzn$#bI=~+4D<8d0S)Q8fo3C|VNaT-Jc97i5Qq*ABw zeMhS?{)P<;a0YLffv2M6)&&VI8S~ypOL$gOZoB{BKvhwwS86ygQg6jvowjuvhA_bS z(9%lG!a$a^BY|@0;cZY^=x*H|^&+Ncsif%Q9tBOm_C9m4`cekmW}|xG z(Q;EwwhbFYB>zH;lM6Bvo>=Qrp^1By1n#s-r4%8 zH&ik13NtvDr>1-VK(WUFUzY^ljA{?wp*^VEU6W<_nyGoGN_3i_$uc~dhH4GY2->7~s~9mOYJ{Q;Mnk`kdBpS7%E(KJ}!Y2y<5YnJ_~`pen7J>jiO`weX%8Kt2xp1tfoL!4KA zm1R3pxZGR$2^Y$xjlc2_iO$fv5$&JgYWYrv!zCThqLb?voqR;Sh`QHx)}?+a_(xrZ zBRa;G54)NpB#E+dPd%2Bl+q~d)TkK)Km2(|cC<^P3R zlyYC^RTuFqihCr?g;lHHGxX~`)}(zO^uj$JC;P5;thz5Bmi&G38v!1l!?-|Y^$nKk zuM~e@yrkk&`G@Sj0jGd^{82CBwKe#e0Oydcu6IgqU#E83!UTte-z84&R}QJYd;L{! zPQAx&$0YbU34VWB1g}2*mHFo2o)c6r(JGaqQn-)=bqYV|%tfA)QtuIHeP`Xyr|tY4 ze+f>-PEKL}*s`N@VuyO#vBSM$JJfr?(t~w7mZA?GuG##?cl`qs`iIJjdKZ?Lp3EPK z@r5TAj4H=#d}TptU`%yVsQDOv2OZIi`L>g94+%yp?Y>S8 zcJipe*r4qcQT6t;(MLNtAyA4}d%L14Tv4tL7Ug9n6T9~dRWx?m=Z8xgT@@G|j2Nap dp`3fH>`d%J z>`LrLAXJC%PV7PKNmLS5L^V-E)DqK)I${P9v)JYsKRKCyt0&4OGPe&bcM*}d0WB8v|TB9WnNwlmm8?OhPalvj<{Y5cx3b> zK;^BJR)}CT1k=3(cwFR8;6A$CMcgd~I_*6o_nJv)Ne_QNln;wMke{wR$bg5WU^P8| zh07`tNb;D-Bfz7^Wzg`o=Q(H(#yuhOq=_5U!3<9$;8l@lfR{y{1zx1*b0W`&Xz8857%6CLw2i`Es!Vw(vO_8^l_HBd0tth=qyhpr`xDOb& ztH>IoEW|6lAs^v1ZY1)th=rwo0-(}A1-29U3>Yc$Il_Mu`GNso5?>jZC-$|-H&lL0 zd>7ij=RAKPekA@){Kuehw#d&&w28GCl+MF|Ui7liM#ia)u3(sj{Z5euQ zCDyo5+FGpfp)`Tv+Yl4Qn#57cLzfD%CS#ebGi!=iQ}Ys(*0y<1RI9Z;vNv+D9f%!? zoy6KX?`iEq<*s7w7Ai6M-4Q-rEcCy^_r%)Mz^uJetSTz2iJH(>YX;K+b!ITU5^IK7 z^`SIVtcJX#cc}?3`-#;IEEKCHoJ6Zwdxg?0&ND`|iG|wK;d79Efmm~ic?M>ldy6$c zl;E;YUeW>kB40bF`>_mvGjz~)s#sl2t?H;&?*$KWzDl#0ugh z#GNA6$(YjK%z&Ne$$>(w)5K!`(~=H91Ijb$$p=u>F4oylUM|)-z`15$wC8yYIN!j` z`T`SYT`1N?q4Hv}F5%-}nSaC<)_z%D8n}C~cI!%HxLK^LfE&cR8eqq>*#GpNT?gg$ zdiQiPoff-mvfn7yO-5O?+`zg8o~y*V6?j;z+o-%fA7|acxI4wVD^%Vs);;vRH&osy zMLO;M!~?{G#6t$Txv(i(tIf~_E5v#faes;R81S`Nj|0z%^#t%Vvp*>o+A+f6tYMb& zj9AYasptg*>v;s=4xwaR*c+{v#2UEIUZLl|!cku3V6TbwItTk9w7p5&TVlP2DTM7h03YK zv~@hUGm_a5o?Eqpp1YdW!p+3t?ZK{eLH|=1h}siaBB~NtAgT(e zqi40K8hX|e(+viFZ(s(SLC<l;#uKkZ%@c#YiTMV4F!Vo-g>dmiq5na`k0{>4YCkDnNtce$)+wqhlzav( z64g!5p3oK-wSOq}ib{l1ALEju0><@+wgW_^s6_wM6U-11){--nGfAOnS zt}rlZ`C$^LL!Cm~sR%zq)M=Fd8G4>B#k^6}nQ%Er)LC>n+n|KU*Nl0tsPo`*e(uAF zR-!)Dg$zLdQ$LHk82CukCBSo{2ALtIOs5h0nn zp16Uyk+_MtnYaZ*-zDl+O6-4%{ZHLN<(&q4^7uah#r~%yb+1vX`$XN(fCq>NiHGuW z>S21W5;af^kI?1O(B-jwh9^ShlcJsqrKd$b!?YM6nwxe*RNGiemp$|Doqkc~2dO{$~#s^(#;; z>NmQu|EWKq`G*eY+cZQ2~MBSvW#h?MLo_8_9rKu;G+ zjlxqIP$K$@wMz|Pu*)FrBsLC^y}j5&IoJk}Tm}pid&4m8Mq&@A%f_K{6MFLVyS=H{ zBSYILvC;oZkSI!QMr>|iCJ5UWVZa!%+5hYZ3SFwjuEAi_=~8P@(j)lMFHLs#Kf9jNOoIK-<_A=}iAwfAyMny%=(37fjlrG~`w=4<&qu|63`&0DvmY1x z2}n;d!;@0-n(-{%l-Qo5u>aZ5iIg^r{k#EQN$FCtU!==R80BS&@EMszuSn^Q#-?}4 zepPG^ZNF}m1D`wWH}UxQ5c@5T@;31f@h$3*`F zN?$QBqx{Ulei8duy8IT}ey8mZvHwfkpLrW5Y5#2`M+l2hgiXMREKY?qwvn7>$aLzbNTNC4n3B)$Uz&s~WDmNH>6Q#)* zdPi}lP?}0iBeo^BBT(gfE;|_LFZ8vv6R|U~3&C1)cB6!2uRV7+$gOBkQ>{)VQAJb} zHAF4rrW19TXOB2DDAf})p`0&H1EoetG44{6I5>DZZ41##>_yBnz*`^V(q;f>ab}A% zhswFcJYsKy!EaJp0MAY)+DDv)Mrp$L6{npp`wDU!Y`FFjmr%1C7*Kzr2ECWlAc#Vx=$QkME8nw4REtK*BTp!73Vr}t`DUf zbS`n&|D2no>}ig23zWBsbE}bZ)4d(ayTrMJE_c%Q8Kt|4dkl&mG#Ii0vAuyw^neVB z8OeEwco<{8BhD&Ht0BF_DLo?2qmZ5#=P`i&&v{&&C+PVk@sz<36s_K~XNYHs=L|}j z-Fbn^7Y&Bsf$M;mF*Luf z4}2re2UMrAl)fUqHW>J-+WD5s?+k{P zQTl=S5m4g%8)^R%=Rd$N;`~J0&jv#$Qu-CPKg1bG^gC_!wEdU(a~n=DLV%No-{>^l<|d zw>1OC6BCGSh>64`iIiiosS>H+D3ggP9PG_-uxS$6HZKkR*i=u1-+xB<_rC}~K}U9? zZD#_1|J&6p1)DFC-KfONb^m=h&&y_lWY8C6)9vYH>5Sjkp3L@j_^H*qRxY&uuQ5n< zd&&56FB1=v4Rt}#o$%fIWM{>PuC>~E%#F6R`H6HU<+-hyWo~1pYYp_H_J`H&a-&U^ zZR6V9n4el4@AT79uBvmRb7%NV5`HFA-s*MsdZ{kAI_T^7lFPK&cB2amo4j<|>+H#< z;gm+QbS9q3W_&kTpJ;@*ubZhU`E6#JHdsp^)eO^4Y#QpT7{0xVE;zs9JH&@rq zYMI^YR`JgY{E-vm(efjt)HEYFUA(c@hq|1)m|dLC`cvap6g}Y z7;@Ca{q8`!4;|-5XPDI_{ft&@_9NmW*tSkDjmdf)sFQU!F?=e^uflVy@#1|6Kjl{D z>(PywdeoySJsYbgs99?tF_S*CF>W>X!#A@op6f=ngq=)f))k|<)rIR{)E%EgN)_*LfL?^k8jn!s; z55RR)Q+56Nj=5UrV%@IE9g{tG4k}`KkaVlE>3Gslr*+x|FY4bbYV_|>@8~h-{e-ho z9V}gc$lPfgu?cb(SNn-XXW+JasZ0`2swdv>)>KvIQ_PL0P>^YT*7O04t#bhLPhjjB zZglK-I`xcWbm|dEJ*x^M$AWC8rvi^OU$?vAj4v#X1)XspM{qihNv9W2`)*g(o#l1K zgACI3_uwqKu?q7P4zS$l%*t2|e%y2_p@*L`2E{Tx<-?~krWGaQQ2Z%HQ1Q7=6On`5%`$l=(y`pEZq87t2U;a-DqP~%I`vHbWL+BtF&@lD;8W-m&JY1r4MOS z&=Ft1mhp%xaMzhCTA>gAYNy4G&Wp{SH66((xox-@n*B_VpTcaq^uri?uN!Tsu0{E@ zl($ySYSS?VM(72^=oCV>q9@? zi5qC|(XYbWiDi>@`nIiH?8Orv)}#-4u3X*OG4t?>UU#FlGh3VMO_h(FiN}Pvlz!mr zod|Q^tsZ+99`^ircX!}cRm?F>A+n*)hHEkI#ZVUL4E6f7Ce6{+H>;OQ#d&(iJqp*+ zZC-o{u9oTXrG8g#_1HpRmpOJXr^?OMZA}G><6VBrWE*z9zKrAD@eGE>?S%_3U%G{8 zHm&-8P6jEr8I8<$S$@uM$ZPYC(iw7x#+@6a5?xC$xZ9uT)LU1y2A)HjgC#C{OfrN2 z7{6oL!2LGv|B2L5H6?CzpBZsffLq_Xc!&JVq0seRF-bd(dk8}pH2X`;b(T#n!@L&- zdY7^G&K_N2xNcBn+PStI&Q2|^R_vr(l?pslv-yJ7&hXW)Zxz9HcyqkJA2*I(eRs<1 z?=d^6C7#u_>cnlG?lyWIx=2^Q(e6g))Hk=)&T6i0bDL&2w$)Fst*vpZTg)up*w2lQ zmoeJ2=wIke4K+3OuK88nGOM*^R%Kg#OS7@Qk2%<}pd6>ct<{fTZ(SeT*xC(l_43og z7QSwss~#!tAh{t9=w8&m{l65Af8 z_f0=0WWs9PIv+hRxyOu8JH4kji)a%e=XL`$*=N6uy!^ksHGS;O!a`_@GuB^f9z_JJ4KIz+0w=O&`zr)Q! z7`Tw~n>_X&q^q4<>tdtx%E8#{Hof!HTX0iVx;3?KZLGDj*<4}6cfzXC%epJ2vg;rx6Z+LP2)g^)s+#4938TXdASId03w`sRkNAcF&@Rb>~t z(W-iu zVhAym7)F#4*JJrNi&e~8?d7EYn?Gb%KRG;5vQpKkJi$b%FR9~HLvW9g}6Kh4*Y28sNwp#xna`FD6=op(r{lE|hls<(5#lez zUq!x1EBsC5@9EN)Z1NR>P4drl-Pa=DXi2_RbRCec`;H3V6F(@7J^w+JI*!8siu{;X z_(|k{s5{!^JyUNQ2Lgme*n~p_iB3dk;wS}l16{P2Kv$w0aSU;+!i->Ab|;RljPy$61GXG_~SYWUON?2E_ z&_0JyZ>R)@rG;e_UN3=iDhyB8jgY{|_7b`X+}K_UjFP}juq8tEn~Bj9xP`rpA%2xs zsF1*|6o%8nvDB+1s-)|4Y%-1*PgE1ke}UUrnn2VLwM3ml-!{fR>xtVX5NDGF(a_$~ zyx1LhmHXLpB5|h#Cb8w@_LhOWBycx{_oRjQQsF*giUg*zNfXhm=vK+n1H^;GLl|}u z^`=Q+x(e0!84{SuCbNi#iP^*)Vy;3J=1Jg@bm>tE%x9Cw(!vECXrTm{{{l~>>z-uY zQ;KfiOJH%jZixhzva@Bxa$*IsQlUnylE7*T*QABae}T0USjReM=D>QEo+UOA-tuo` zX_MlZph8F3LOZt-+lcMN^TZ3p4q_*~#s;(g)+;zNS@FM!;K*$8}0F#iRR|BUdD z#Gi;i6Q2;D5}y&D6DLRRQRJK|5=xdbulnd%u&U%5M`3OUy?iJXiZzIJ1+4R4x`@K!4hn;z5EZKgxLK@HV7ORAfHJX$ zQejw!UaqIGT;Wg82q@er)=1Xf&_Q7og*Q>?t?k4V`o;T2aP-qfs3hSnJsB5P1{tm(iDSSwwCQTD- zI)yWcnZzvOVPZBhhnP#uBOW0hCFT>45etZg#N)&h3NJIO5l@M=NQGKj%uTsOtfgt; zG76UyE5ur8IO;)>^UFvE?(wdg57P1MwWOQK3d`5^Hn1#QbM%6>A%A zw%!(NJ7)e>+QR&2G5=Z2f7VX1cA*J#oV8o5Jse@L;65omfYJuf+Nb@Hesk%9dZSjqU ztn*&;TMGZBIKlJs@1gKNv3>ylD;DyfnR4d86R|f<=|_V3&s!B+upGA70j;x{|GYkJ zhr*!Z#BW&Y42AQ>J__h1_R&ChvAY1rh~1S?ltm8y_ z^J$-mx>LkHiFNdz9-J)uR0>a1_zI^(p^w;S0It|)vdLMBlV-4#Mf6tquL94q`-*)I z>&{j9^O8;Bd5V+fumsn)bHwfkTq1UV;3BaHu|sEi*kwS4*w+K2#4ZO$ianfqBNQh;!O{)HjS6r7**8Jq z7O`(;9r7Pu#jhJfA@iTN@$6fnFkb91P$l+QHmOwjz7(NwoZ{p)ELB5cg4j`(Zd07X zCAMoQtnDC-LE&Dp>w$@4-wrg09cRmg!k^Yg3hz+(3x_>s-zD}W)=gHNvXZ5{iF*|O z+ujF-2gRNOG>bizO_~(`K=)JlfWn{FhoCS+>}jlLafcauSLWM^=C_F~t0);>aGz*+F49g?kjIdQJ90;Z?Es122pHBAdLV@b^)Y!dDdj488`1H^hFO zb-z~lFa0KkZz=p_irvB{2NnK1v{86SaXcK% zM5!ZC_(JTzu=H1jukbet|E};Y`wFjVi~SGi{a5UN0zZiTHSjO7zoEjn3V(BbN8$I1 zUf%otn?mM4e>?q%uF=s?toxtBt8)Tcax4m6meAViDozkMN}Ns{S$A-JzFXQkc`B z<#p&KUz}Xl98K;hd`1cjmE6ayvVgxF-T;tVd8Q%d0wg+IH)pinMO8SAcB zcw5^UPT>fJ=laeKP6=yt~R4dNpi#fMZI6>iM5~mgl_2SgAE~fCJqH{ZiafKItodzg0 zi_-|)EzTXlBylFP<(+H^mogqQnYc^g?RV!MC`=LOUe?{G@F#03g-soV_d{W(I1d2R z#CecS9%2)3UZztxL*WlJ3kq|@d6;#xJ1EShaGt_hxyOhr9%J3 zkpG6+Mjt-1H9wm!tbe zmi`QdR&hQ7J{RXx3O`f$v&8)89Hj6YHfbXcDf|$01igGE&R>APiNpNoe9=Mg@6`KJ z;Vm{N$?5^uT*%OD|j_ZgC%$kkS{@#|E`l@4uIFZR^hkIr7*99a1csG z5-ea{p~APNn8Hv8VF^m*5-bIVNpJ|83{`li-(VSq*DL%r8jjKp5*)$0kqR#s1aG8p zl)_iI872HPiqXJi3El$WpQVfeswDU;AS}TO0Izwg!nb-Xg_R2b-6JUBU%ZTC-FSuX zZcz$vQ~0CTpp=kcEr6f;>ewWv@b|;*6vh?aO-8T*CH&Oa$htce{(|30;iL}2yHJ`T z!MlNG3El%tk>I@mUhF=F7omewDQr@BAuo78N)Ji!0oFaJ@Ly~ih0_)Oi_Jvo5eb@& zFh_z9v&n3QuQ!*%c?#e7M^VC0ee+rOn4$+WY;Ylkk1PCvjML(0vZsJ&B)AAzDZ$0S zG6^mL@P11bz85T~aD^hv>v|PR_^EF->((gzO}UoBbqe3k^(f&w^I2e{1UInBa|-|c zHc_~lLT{k0DB-Ge8|$_!d?Q|PpPrPn0*3hQ1~`1AQXg}+w#3U8uxK!R@p?@I7(HhD+k>%B)|N|D9c4gLnD4Mk=CJ$%=Tt0`r_WoL3boENG0!>ce@FIzC16A_=#D z-nh7XdERZ|VJ*#TTblQ@G;e5WZfGfd@Uq z+O|H1HZ5+T#qE|iG#A!X#QW=rB|~HJDlAq!Iw9;19D-6qwVP|om0>rhJhwKvP9Lix zHLj_QxcS#pcza{St!gyFDz|u8>#T(>O)FZOm+O43cSn1z&Py3la`vB+M`7?r<>eF2pt{THyg|BPB9By5%~<$Gjg~Zn?G`RE{yb6ad<=6tZ>fI zf8MJWWlWg)%CM_-?zF?pAJqBkJ=d))ggt9hbFp+76VfcVcl01< zDR;Qe(giwjL2e=%sdEPo0x>PU|KsMH$B0`TGQBtLz+ZQCL2|08t}Mn&z2MFK!Cg9yufUVru|ssL$hb9wc283 z%&i=pns=a?UI()e?s)j%j-6SqajJ-zhM-i1fjJ=?lz z58kh^As&u4Bt9#LcZcg@b(3mijfsl-cy+5aG_uBj(+#!Fc zzJ5V|b+isU@aD2etiC4FH{q5?>Kn(_L@V9cIG2?CN-80vU5~&OhAB^3*CsNE0RZGa^ z=k~#hSBA$U_3=mo8Kgsu>|Nu|=^c0b^qx4VVN!jhxEA9!giWM3fh%jjdqUPLYHij# z?!c@+xH%fHB4IyB<@Xl1xc3R}=-%hLmA$WW6TQdiRQvrRIh3c9gNp!y2!hw`Jq3Gu zZhPP`fdqjA2_Q3|y! zmSms5jr*=c-*;v9ZReuz5+-XbE)V3r0`^FA7)^?Ce2Ekba40T!_&94PFrY(+vmM&)+JeP)+HG)o;|k-W9KvK6lu_pPBy+ZG1@RW64x-C zQ!uWqhiTk5rZDW{xPx%NL61A-4EL#>2>948-kY1Czj8fHv99t)KRY+e zJ-fp+)P4<9@;p=W!qv!Q#*3^E5&lgvxen!geT*bG!jF(8=W614R{w6rf}=6!Vjf0* zz_={jYA^C~>kHf3mY9jKUd?#-Gk<+E1}q;*rz&quB$#yy%_A~bP5~Y?->Cra9WqDb zrWZ6J7nLV>X)|o&WmxH{2d45}LkIWYkA&Q|U?T>!FY^ZEk)X`Wc4b`bg0fhRx*2E^ zUQQ7nVkXJw>yw}37VG=zA+vdUMtm1?CBEU#JqZ^sLhJ8`d3XA)Lc8Li)TL!jY*4 zmSZ{|*8>q!j)@qdw)^KG=(J8lbjx-!r=v@c{@(b72+zm{#f8zjIxq9%kyCl zsXaq(j6{=5_Ip#7f3uo6as&=bk;>|*fqzJu zfN!h($&mVjN;ju0BQfMd17?RtR8}X~-QH+y#wmBIImR_*9v`wh^>r)9bV@yq*gp;K zjZXsejj#;Gsj2-qgKqY|zTj}f+8)M>&TM~IEOzh+Ecf$d;HTFxu_5NNn?v}Pa2FtI-8fZTj`FzHnk%Z%}ORUA)@PZy1xVB%N=fvodVZ19nlcy;46rf3e($Q>IspyL!fqrFde zbTL}OgN*|ZO|4ArQZsOT)6cSlJ7?)+n+@K1wl}sp)wiF0OuXsxpSL`3DQ1u>99cI$ z97j+sDsD7S=3s4};`;qjWRhvz^WZu9MT@|SQ{|ovvs5!qtIX#&GAbF`LCk=l%_hHqZ)TKm( z*NQrGsz=H*ZfF+A8~^a?{fFjkfC-qDqgH1O-F3heZ|J-tU6oK?V-4P42QT)X5%9B* z&NT_ftJ9r~Lm^ItIu9Mx^ZX3w)BnoTFXOoIdlj z+&=R&Y&-wuoxW}S1Ts-(Jiik4sS!6L#d!!YmiBE_`@zpn?5f8>e@Nhel zyIB-{^uon79XT^xc-5K$Rv?^B@jv-Ot&5f(o`R-k@GjS6zI#mT9jBD`cSF0Q5?Xi9tO>ea1_ zHqlGF`1#Nr0=s@Y12d7!)Qr!^v7UDp@zGe(bu4DVtI^$yPZE5n;}l}P(lfS+QE%V9 zbKx7C(|$Va$m9E4H_gYPgkS8Kl&*!DD{}53*ip*sv!c={?7;WKz@pL!pK^*ylN-z< zzAdy?Poi*9n1S?%OiDJNq=kn~0x~4R^7%4F4 zLyUwgEfcnf3=cO}Mf>N*!((+TmTKA6G8|#3_n^`S%)&6XbQ9qi^l&d{JaoI7V{W5# zRvyKJKPzaTTs{__?e|bQ#z`$l{Ak9^>@yzgXWO|N__r{`4Qt!-1WbnmwauJfu3IIS-nZk~(-AA7dBzEQb;Qtwv(HI|t(JZ=~7FrmE^rkLf~ z%u!2QmoI_C9@@Q)!R*+hGp5ac%Y3~I_U^a_oBK-!_0h%QM0KRrOoo5Mi@OsxKv?lw z8V~Z7G-aDBXl$t|CjT_UygBB;-MqA=`9Xi~LZ-d9xyta}V?Iu|rMFk%p zxd-mYVBY0u;gQ8l+HVuH-hCg_P|Abj7z_Jy5{_haAU&H6!ryRhxH=k--iEJtvvVh5 z>`~y)zQ2=G;dS_S*Cz0!F~=NQweR4bClBt}eP|!@$u4}oYhE_XX6DHAuLb%!ILZVE zETY%TU~f!})gA~mTaRvcB9j+3&No-1o8}$dvrs?pvyW@`+{zpz`d{&|7aBs$QH3FX z8V)Q9#c?J!pDrUa4pKOvvw@jXK54pwq2{s<%ja(M_IlAC}>hoejNp z*o2=vG>XGGa}k8Qw9Fvt_i$U%T-w~A*U8)+=GtN#?n(4~SZc!h@+ECE7adygAiS^p zKQrug>V7o#a`tY?xb@Pwp$<||8;#=v%pEwwyHPh)t|vvvy^)Uyp~MPf3CAHb4&lTzADG`N-e>K*9^ohZS8_i@}MqADZR;unqomuk_N1PdTAMxNF6u9=-;Jy$0s;$J}?fG~t^omVpV5g2`I>(=Mp(Iuf8oQl1g$|0#m2OduC?dPWU^-pav_W@7s zY2EUav0?lAcfK4FzH>;lu4+JKtoF$0v3S4uq>6Dd+_g7U)P`@1#VhWJ#1r^6Y`OLG zTKquQUc0PhY-6;hsv=S!tE_gcamz3Gen?fgAv`vmh*X#>l@JbXZRIz%!Pgn*K6U9h$I@^L0&Iz_PFqbh>J#{vWs2QcRv?)x{*7>wYZme z>l@?sF>|@~ literal 0 HcmV?d00001 diff --git a/server/src/main/java/org/elasticsearch/upgrades/SystemIndexMigrationInfo.java b/server/src/main/java/org/elasticsearch/upgrades/SystemIndexMigrationInfo.java index f8adead63ea32..a0e9461558b6b 100644 --- a/server/src/main/java/org/elasticsearch/upgrades/SystemIndexMigrationInfo.java +++ b/server/src/main/java/org/elasticsearch/upgrades/SystemIndexMigrationInfo.java @@ -240,7 +240,7 @@ private static Settings copySettingsForNewIndex(Settings currentIndexSettings, I /** * Convenience factory method holding the logic for creating instances from a Feature object. * @param feature The feature that - * @param metadata The current metadata, as index migration depends on the current state of the clsuter. + * @param metadata The current metadata, as index migration depends on the current state of the cluster. * @param indexScopedSettings This is necessary to make adjustments to the indices settings for unmanaged indices. * @return A {@link Stream} of {@link SystemIndexMigrationInfo}s that represent all the indices the given feature currently owns. */ From 1b5d75f537b383962f51750911fa68f716b7fb28 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Fri, 27 Sep 2024 04:21:46 +1000 Subject: [PATCH 29/43] Mute org.elasticsearch.xpack.transform.integration.TransformIT testStopWaitForCheckpoint #106113 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index df82cda4277b0..e6ea16be8c631 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -305,6 +305,9 @@ tests: - class: org.elasticsearch.xpack.esql.qa.mixed.MixedClusterEsqlSpecIT method: test {date_nanos.Date_nanos to date nanos, index version SYNC} issue: https://github.com/elastic/elasticsearch/issues/113632 +- class: org.elasticsearch.xpack.transform.integration.TransformIT + method: testStopWaitForCheckpoint + issue: https://github.com/elastic/elasticsearch/issues/106113 # Examples: # From 888188695a3f26a5ce3d590cbac94e61d83ec374 Mon Sep 17 00:00:00 2001 From: Jake Landis Date: Thu, 26 Sep 2024 14:52:05 -0500 Subject: [PATCH 30/43] Bump compatible rest api version to 9/8 (#113151) This commit bumps the REST API version from 8 to 9. This effectively removes all support for REST API compatibility with version 7 (though earlier commits already chipped away at some v7 support). This also enables REST API compatibility support for version 8, providing support for v8 compatibility headers, i.e. "application/vnd.elasticsearch+json;compatible-with=8" and no-op support (no errors) to accept v9 compatibility headers i.e. "application/vnd.elasticsearch+json;compatible-with=9". see additional context in the GH PR #113151 --- .../transform/header/InjectHeaderTests.java | 4 +- .../elasticsearch/core/RestApiVersion.java | 11 +-- .../xcontent/MediaTypeRegistry.java | 2 +- .../xcontent/ParsedMediaTypeTests.java | 32 +++---- .../RestMultiSearchTemplateActionTests.java | 67 --------------- .../RestSearchTemplateActionTests.java | 56 ------------- .../PercolateQueryBuilderTests.java | 30 ------- .../reindex/RestDeleteByQueryActionTests.java | 53 ------------ .../reindex/RestUpdateByQueryActionTests.java | 52 ------------ rest-api-spec/build.gradle | 1 + .../TransportNodesCapabilitiesAction.java | 9 +- .../org/elasticsearch/rest/RestHandler.java | 57 +++++++------ .../indices/RestPutIndexTemplateAction.java | 5 +- .../action/search/RestKnnSearchAction.java | 13 ++- .../MovAvgPipelineAggregationBuilder.java | 2 + .../rollover/RolloverRequestTests.java | 71 ---------------- .../search/MultiSearchRequestTests.java | 46 ---------- .../AbstractHttpServerTransportTests.java | 6 +- .../RestCompatibleVersionHelperTests.java | 32 +++---- .../indices/RestCreateIndexActionTests.java | 69 --------------- .../indices/RestGetIndicesActionTests.java | 64 -------------- .../RestPutIndexTemplateActionTests.java | 76 ----------------- .../indices/RestValidateQueryActionTests.java | 37 --------- .../document/RestDeleteActionTests.java | 49 ----------- .../action/document/RestGetActionTests.java | 51 ------------ .../document/RestGetSourceActionTests.java | 40 --------- .../action/document/RestIndexActionTests.java | 37 --------- .../document/RestMultiGetActionTests.java | 77 ----------------- .../RestMultiTermVectorsActionTests.java | 83 ------------------- .../document/RestTermVectorsActionTests.java | 63 -------------- .../document/RestUpdateActionTests.java | 18 ---- .../action/search/RestCountActionTests.java | 65 --------------- .../action/search/RestExplainActionTests.java | 51 ------------ .../search/RestMultiSearchActionTests.java | 65 --------------- .../action/search/RestSearchActionTests.java | 53 ++++-------- .../search/SearchModuleTests.java | 62 -------------- .../org/elasticsearch/test/ESTestCase.java | 9 ++ .../test/rest/ESRestTestCase.java | 9 +- .../rest/action/RestFreezeIndexAction.java | 9 +- .../rest/action/RestGraphActionTests.java | 61 -------------- .../RestDeleteTrainedModelAction.java | 15 +++- .../inference/RestGetTrainedModelsAction.java | 19 ++++- .../RestGetTrainedModelsStatsAction.java | 23 +++-- ...RestInferTrainedModelDeploymentAction.java | 25 +++++- .../inference/RestPutTrainedModelAction.java | 15 +++- .../xpack/ml/rest/job/RestPostDataAction.java | 8 +- .../xpack/security/audit/AuditIT.java | 17 +--- .../rest/RestFindStructureAction.java | 11 ++- ...MLModelDeploymentFullClusterRestartIT.java | 20 ++++- .../oldrepos/OldRepositoryAccessIT.java | 18 ++-- .../upgrades/MLModelDeploymentsUpgradeIT.java | 20 ++++- .../MlAssignmentPlannerUpgradeIT.java | 20 ++++- 52 files changed, 293 insertions(+), 1515 deletions(-) delete mode 100644 modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateActionTests.java delete mode 100644 modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/RestSearchTemplateActionTests.java delete mode 100644 modules/reindex/src/test/java/org/elasticsearch/reindex/RestDeleteByQueryActionTests.java delete mode 100644 modules/reindex/src/test/java/org/elasticsearch/reindex/RestUpdateByQueryActionTests.java delete mode 100644 server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestGetIndicesActionTests.java delete mode 100644 server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateActionTests.java delete mode 100644 server/src/test/java/org/elasticsearch/rest/action/document/RestDeleteActionTests.java delete mode 100644 server/src/test/java/org/elasticsearch/rest/action/document/RestGetActionTests.java delete mode 100644 server/src/test/java/org/elasticsearch/rest/action/document/RestMultiGetActionTests.java delete mode 100644 server/src/test/java/org/elasticsearch/rest/action/document/RestMultiTermVectorsActionTests.java delete mode 100644 server/src/test/java/org/elasticsearch/rest/action/document/RestTermVectorsActionTests.java delete mode 100644 server/src/test/java/org/elasticsearch/rest/action/search/RestCountActionTests.java delete mode 100644 server/src/test/java/org/elasticsearch/rest/action/search/RestExplainActionTests.java delete mode 100644 server/src/test/java/org/elasticsearch/rest/action/search/RestMultiSearchActionTests.java delete mode 100644 x-pack/plugin/graph/src/test/java/org/elasticsearch/xpack/graph/rest/action/RestGraphActionTests.java diff --git a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/header/InjectHeaderTests.java b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/header/InjectHeaderTests.java index 3ba9ab0f697e0..0ef7cc7108bce 100644 --- a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/header/InjectHeaderTests.java +++ b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/header/InjectHeaderTests.java @@ -26,9 +26,9 @@ public class InjectHeaderTests extends InjectFeatureTests { private static final Map headers = Map.of( "Content-Type", - "application/vnd.elasticsearch+json;compatible-with=7", + "application/vnd.elasticsearch+json;compatible-with=8", "Accept", - "application/vnd.elasticsearch+json;compatible-with=7" + "application/vnd.elasticsearch+json;compatible-with=8" ); /** diff --git a/libs/core/src/main/java/org/elasticsearch/core/RestApiVersion.java b/libs/core/src/main/java/org/elasticsearch/core/RestApiVersion.java index 9ff31a191ce37..387d05db84441 100644 --- a/libs/core/src/main/java/org/elasticsearch/core/RestApiVersion.java +++ b/libs/core/src/main/java/org/elasticsearch/core/RestApiVersion.java @@ -22,14 +22,13 @@ public enum RestApiVersion { V_8(8), + @UpdateForV9 // remove all references to V_7 then delete this annotation V_7(7); public final byte major; - @UpdateForV9 - // We need to bump current and previous to V_9 and V_8, respectively - private static final RestApiVersion CURRENT = V_8; - private static final RestApiVersion PREVIOUS = V_7; + private static final RestApiVersion CURRENT = V_9; + private static final RestApiVersion PREVIOUS = V_8; RestApiVersion(int major) { this.major = (byte) major; @@ -67,8 +66,6 @@ public static Predicate onOrAfter(RestApiVersion restApiVersion) }; } - @UpdateForV9 - // Right now we return api version 8 for major version 9 until we bump the api version above public static RestApiVersion forMajor(int major) { switch (major) { case 7 -> { @@ -78,7 +75,7 @@ public static RestApiVersion forMajor(int major) { return V_8; } case 9 -> { - return V_8; + return V_9; } default -> throw new IllegalArgumentException("Unknown REST API version " + major); } diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/MediaTypeRegistry.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/MediaTypeRegistry.java index 88724062bb452..ae2c80a136437 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/xcontent/MediaTypeRegistry.java +++ b/libs/x-content/src/main/java/org/elasticsearch/xcontent/MediaTypeRegistry.java @@ -27,7 +27,7 @@ * A MediaType can have only one query parameter representation. * For example "json" (case insensitive) maps back to a JSON media type. * - * Additionally, a http header may optionally have parameters. For example "application/vnd.elasticsearch+json; compatible-with=7". + * Additionally, a http header may optionally have parameters. For example "application/vnd.elasticsearch+json; compatible-with=8". * This class also allows to define a regular expression for valid values of charset. */ public class MediaTypeRegistry { diff --git a/libs/x-content/src/test/java/org/elasticsearch/xcontent/ParsedMediaTypeTests.java b/libs/x-content/src/test/java/org/elasticsearch/xcontent/ParsedMediaTypeTests.java index 9fafd7c7e7150..8cecd3d25201f 100644 --- a/libs/x-content/src/test/java/org/elasticsearch/xcontent/ParsedMediaTypeTests.java +++ b/libs/x-content/src/test/java/org/elasticsearch/xcontent/ParsedMediaTypeTests.java @@ -30,19 +30,19 @@ public void testCanonicalParsing() { assertThat(ParsedMediaType.parseMediaType("application/cbor").toMediaType(mediaTypeRegistry), equalTo(XContentType.CBOR)); assertThat( - ParsedMediaType.parseMediaType("application/vnd.elasticsearch+json;compatible-with=7").toMediaType(mediaTypeRegistry), + ParsedMediaType.parseMediaType("application/vnd.elasticsearch+json;compatible-with=8").toMediaType(mediaTypeRegistry), equalTo(XContentType.VND_JSON) ); assertThat( - ParsedMediaType.parseMediaType("application/vnd.elasticsearch+yaml;compatible-with=7").toMediaType(mediaTypeRegistry), + ParsedMediaType.parseMediaType("application/vnd.elasticsearch+yaml;compatible-with=8").toMediaType(mediaTypeRegistry), equalTo(XContentType.VND_YAML) ); assertThat( - ParsedMediaType.parseMediaType("application/vnd.elasticsearch+smile;compatible-with=7").toMediaType(mediaTypeRegistry), + ParsedMediaType.parseMediaType("application/vnd.elasticsearch+smile;compatible-with=8").toMediaType(mediaTypeRegistry), equalTo(XContentType.VND_SMILE) ); assertThat( - ParsedMediaType.parseMediaType("application/vnd.elasticsearch+cbor;compatible-with=7").toMediaType(mediaTypeRegistry), + ParsedMediaType.parseMediaType("application/vnd.elasticsearch+cbor;compatible-with=8").toMediaType(mediaTypeRegistry), equalTo(XContentType.VND_CBOR) ); } @@ -179,19 +179,19 @@ public void testParseMediaTypeFromXContentType() { ); assertThat( - ParsedMediaType.parseMediaType(XContentType.VND_JSON, Map.of("compatible-with", "7")).toMediaType(mediaTypeRegistry), + ParsedMediaType.parseMediaType(XContentType.VND_JSON, Map.of("compatible-with", "8")).toMediaType(mediaTypeRegistry), equalTo(XContentType.VND_JSON) ); assertThat( - ParsedMediaType.parseMediaType(XContentType.VND_YAML, Map.of("compatible-with", "7")).toMediaType(mediaTypeRegistry), + ParsedMediaType.parseMediaType(XContentType.VND_YAML, Map.of("compatible-with", "8")).toMediaType(mediaTypeRegistry), equalTo(XContentType.VND_YAML) ); assertThat( - ParsedMediaType.parseMediaType(XContentType.VND_SMILE, Map.of("compatible-with", "7")).toMediaType(mediaTypeRegistry), + ParsedMediaType.parseMediaType(XContentType.VND_SMILE, Map.of("compatible-with", "8")).toMediaType(mediaTypeRegistry), equalTo(XContentType.VND_SMILE) ); assertThat( - ParsedMediaType.parseMediaType(XContentType.VND_CBOR, Map.of("compatible-with", "7")).toMediaType(mediaTypeRegistry), + ParsedMediaType.parseMediaType(XContentType.VND_CBOR, Map.of("compatible-with", "8")).toMediaType(mediaTypeRegistry), equalTo(XContentType.VND_CBOR) ); } @@ -215,20 +215,20 @@ public void testResponseContentTypeHeader() { ); assertThat( - ParsedMediaType.parseMediaType(XContentType.VND_JSON, Map.of("compatible-with", "7")).responseContentTypeHeader(), - equalTo("application/vnd.elasticsearch+json;compatible-with=7") + ParsedMediaType.parseMediaType(XContentType.VND_JSON, Map.of("compatible-with", "8")).responseContentTypeHeader(), + equalTo("application/vnd.elasticsearch+json;compatible-with=8") ); assertThat( - ParsedMediaType.parseMediaType(XContentType.VND_YAML, Map.of("compatible-with", "7")).responseContentTypeHeader(), - equalTo("application/vnd.elasticsearch+yaml;compatible-with=7") + ParsedMediaType.parseMediaType(XContentType.VND_YAML, Map.of("compatible-with", "8")).responseContentTypeHeader(), + equalTo("application/vnd.elasticsearch+yaml;compatible-with=8") ); assertThat( - ParsedMediaType.parseMediaType(XContentType.VND_SMILE, Map.of("compatible-with", "7")).responseContentTypeHeader(), - equalTo("application/vnd.elasticsearch+smile;compatible-with=7") + ParsedMediaType.parseMediaType(XContentType.VND_SMILE, Map.of("compatible-with", "8")).responseContentTypeHeader(), + equalTo("application/vnd.elasticsearch+smile;compatible-with=8") ); assertThat( - ParsedMediaType.parseMediaType(XContentType.VND_CBOR, Map.of("compatible-with", "7")).responseContentTypeHeader(), - equalTo("application/vnd.elasticsearch+cbor;compatible-with=7") + ParsedMediaType.parseMediaType(XContentType.VND_CBOR, Map.of("compatible-with", "8")).responseContentTypeHeader(), + equalTo("application/vnd.elasticsearch+cbor;compatible-with=8") ); assertThat( diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateActionTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateActionTests.java deleted file mode 100644 index 3613d7390fda2..0000000000000 --- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateActionTests.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.script.mustache; - -import org.elasticsearch.common.bytes.BytesArray; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.core.RestApiVersion; -import org.elasticsearch.rest.RestRequest; -import org.elasticsearch.test.rest.FakeRestRequest; -import org.elasticsearch.test.rest.RestActionTestCase; -import org.elasticsearch.xcontent.XContentType; -import org.junit.Before; -import org.mockito.Mockito; - -import java.nio.charset.StandardCharsets; -import java.util.Collections; -import java.util.List; -import java.util.Map; - -public final class RestMultiSearchTemplateActionTests extends RestActionTestCase { - final List contentTypeHeader = Collections.singletonList(compatibleMediaType(XContentType.VND_JSON, RestApiVersion.V_7)); - - @Before - public void setUpAction() { - controller().registerHandler(new RestMultiSearchTemplateAction(Settings.EMPTY)); - // todo how to workaround this? we get AssertionError without this - verifyingClient.setExecuteVerifier((actionType, request) -> Mockito.mock(MultiSearchTemplateResponse.class)); - verifyingClient.setExecuteLocallyVerifier((actionType, request) -> Mockito.mock(MultiSearchTemplateResponse.class)); - } - - public void testTypeInPath() { - String content = """ - { "index": "some_index" } - {"source": {"query" : {"match_all" :{}}}} - """; - BytesArray bytesContent = new BytesArray(content.getBytes(StandardCharsets.UTF_8)); - - RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withHeaders( - Map.of("Content-Type", contentTypeHeader, "Accept", contentTypeHeader) - ).withMethod(RestRequest.Method.GET).withPath("/some_index/some_type/_msearch/template").withContent(bytesContent, null).build(); - - dispatchRequest(request); - assertCriticalWarnings(RestMultiSearchTemplateAction.TYPES_DEPRECATION_MESSAGE); - } - - public void testTypeInBody() { - String content = """ - { "index": "some_index", "type": "some_type" }\s - {"source": {"query" : {"match_all" :{}}}}\s - """; - BytesArray bytesContent = new BytesArray(content.getBytes(StandardCharsets.UTF_8)); - - RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withHeaders( - Map.of("Content-Type", contentTypeHeader, "Accept", contentTypeHeader) - ).withPath("/some_index/_msearch/template").withContent(bytesContent, null).build(); - - dispatchRequest(request); - assertCriticalWarnings(RestMultiSearchTemplateAction.TYPES_DEPRECATION_MESSAGE); - } -} diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/RestSearchTemplateActionTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/RestSearchTemplateActionTests.java deleted file mode 100644 index 0216e750c55e0..0000000000000 --- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/RestSearchTemplateActionTests.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.script.mustache; - -import org.elasticsearch.core.RestApiVersion; -import org.elasticsearch.rest.RestRequest; -import org.elasticsearch.rest.action.search.RestSearchAction; -import org.elasticsearch.test.rest.FakeRestRequest; -import org.elasticsearch.test.rest.RestActionTestCase; -import org.junit.Before; - -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static org.mockito.Mockito.mock; - -public final class RestSearchTemplateActionTests extends RestActionTestCase { - final List contentTypeHeader = Collections.singletonList(randomCompatibleMediaType(RestApiVersion.V_7)); - - @Before - public void setUpAction() { - controller().registerHandler(new RestSearchTemplateAction(nf -> false)); - verifyingClient.setExecuteVerifier((actionType, request) -> mock(SearchTemplateResponse.class)); - verifyingClient.setExecuteLocallyVerifier((actionType, request) -> mock(SearchTemplateResponse.class)); - } - - public void testTypeInPath() { - RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withHeaders( - Map.of("Content-Type", contentTypeHeader, "Accept", contentTypeHeader) - ).withMethod(RestRequest.Method.GET).withPath("/some_index/some_type/_search/template").build(); - - dispatchRequest(request); - assertCriticalWarnings(RestSearchAction.TYPES_DEPRECATION_MESSAGE); - } - - public void testTypeParameter() { - Map params = new HashMap<>(); - params.put("type", "some_type"); - - RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withHeaders( - Map.of("Content-Type", contentTypeHeader, "Accept", contentTypeHeader) - ).withMethod(RestRequest.Method.GET).withPath("/some_index/_search/template").withParams(params).build(); - - dispatchRequest(request); - assertCriticalWarnings(RestSearchAction.TYPES_DEPRECATION_MESSAGE); - } -} diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryBuilderTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryBuilderTests.java index d6908f58a901a..88b773d413fab 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryBuilderTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryBuilderTests.java @@ -21,7 +21,6 @@ import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.lucene.uid.Versions; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.index.get.GetResult; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.query.QueryBuilder; @@ -32,9 +31,7 @@ import org.elasticsearch.test.AbstractQueryTestCase; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; -import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentType; -import org.elasticsearch.xcontent.json.JsonXContent; import org.hamcrest.Matchers; import java.io.IOException; @@ -379,31 +376,4 @@ public void testDisallowExpensiveQueries() { ElasticsearchException e = expectThrows(ElasticsearchException.class, () -> queryBuilder.toQuery(searchExecutionContext)); assertEquals("[percolate] queries cannot be executed when 'search.allow_expensive_queries' is set to false.", e.getMessage()); } - - public void testFromJsonWithDocumentType() throws IOException { - SearchExecutionContext searchExecutionContext = createSearchExecutionContext(); - String queryAsString = Strings.format(""" - {"percolate" : { "document": {}, "document_type":"%s", "field":"%s"}} - """, docType, queryField); - XContentParser parser = createParserWithCompatibilityFor(JsonXContent.jsonXContent, queryAsString, RestApiVersion.V_7); - QueryBuilder queryBuilder = parseQuery(parser); - queryBuilder.toQuery(searchExecutionContext); - assertCriticalWarnings(PercolateQueryBuilder.DOCUMENT_TYPE_DEPRECATION_MESSAGE); - } - - public void testFromJsonWithType() throws IOException { - indexedDocumentIndex = randomAlphaOfLength(4); - indexedDocumentId = randomAlphaOfLength(4); - indexedDocumentVersion = Versions.MATCH_ANY; - documentSource = Collections.singletonList(randomSource(new HashSet<>())); - SearchExecutionContext searchExecutionContext = createSearchExecutionContext(); - - String queryAsString = Strings.format(""" - {"percolate" : { "index": "%s", "type": "_doc", "id": "%s", "field":"%s"}} - """, indexedDocumentIndex, indexedDocumentId, queryField); - XContentParser parser = createParserWithCompatibilityFor(JsonXContent.jsonXContent, queryAsString, RestApiVersion.V_7); - QueryBuilder queryBuilder = parseQuery(parser); - rewriteAndFetch(queryBuilder, searchExecutionContext).toQuery(searchExecutionContext); - assertCriticalWarnings(PercolateQueryBuilder.TYPE_DEPRECATION_MESSAGE); - } } diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/RestDeleteByQueryActionTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/RestDeleteByQueryActionTests.java deleted file mode 100644 index 505b12833fb5e..0000000000000 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/RestDeleteByQueryActionTests.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.reindex; - -import org.elasticsearch.core.RestApiVersion; -import org.elasticsearch.index.reindex.BulkByScrollResponse; -import org.elasticsearch.rest.RestRequest; -import org.elasticsearch.rest.action.search.RestSearchAction; -import org.elasticsearch.test.rest.FakeRestRequest; -import org.elasticsearch.test.rest.RestActionTestCase; -import org.elasticsearch.xcontent.XContentType; -import org.junit.Before; - -import java.io.IOException; -import java.util.Collections; -import java.util.List; -import java.util.Map; - -import static org.mockito.Mockito.mock; - -public final class RestDeleteByQueryActionTests extends RestActionTestCase { - - final List contentTypeHeader = Collections.singletonList(compatibleMediaType(XContentType.VND_JSON, RestApiVersion.V_7)); - - @Before - public void setUpAction() { - controller().registerHandler(new RestDeleteByQueryAction(nf -> false)); - verifyingClient.setExecuteVerifier((actionType, request) -> mock(BulkByScrollResponse.class)); - verifyingClient.setExecuteLocallyVerifier((actionType, request) -> mock(BulkByScrollResponse.class)); - } - - public void testTypeInPath() throws IOException { - RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withHeaders( - Map.of("Content-Type", contentTypeHeader, "Accept", contentTypeHeader) - ).withMethod(RestRequest.Method.POST).withPath("/some_index/some_type/_delete_by_query").build(); - - // checks the type in the URL is propagated correctly to the request object - // only works after the request is dispatched, so its params are filled from url. - dispatchRequest(request); - - // RestDeleteByQueryAction itself doesn't check for a deprecated type usage - // checking here for a deprecation from its internal search request - assertCriticalWarnings(RestSearchAction.TYPES_DEPRECATION_MESSAGE); - } - -} diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/RestUpdateByQueryActionTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/RestUpdateByQueryActionTests.java deleted file mode 100644 index 0d35b30c86a5a..0000000000000 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/RestUpdateByQueryActionTests.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.reindex; - -import org.elasticsearch.core.RestApiVersion; -import org.elasticsearch.index.reindex.BulkByScrollResponse; -import org.elasticsearch.rest.RestRequest; -import org.elasticsearch.rest.action.search.RestSearchAction; -import org.elasticsearch.test.rest.FakeRestRequest; -import org.elasticsearch.test.rest.RestActionTestCase; -import org.elasticsearch.xcontent.XContentType; -import org.junit.Before; - -import java.io.IOException; -import java.util.Collections; -import java.util.List; -import java.util.Map; - -import static org.mockito.Mockito.mock; - -public final class RestUpdateByQueryActionTests extends RestActionTestCase { - - final List contentTypeHeader = Collections.singletonList(compatibleMediaType(XContentType.VND_JSON, RestApiVersion.V_7)); - - @Before - public void setUpAction() { - controller().registerHandler(new RestUpdateByQueryAction(nf -> false)); - verifyingClient.setExecuteVerifier((actionType, request) -> mock(BulkByScrollResponse.class)); - verifyingClient.setExecuteLocallyVerifier((actionType, request) -> mock(BulkByScrollResponse.class)); - } - - public void testTypeInPath() throws IOException { - RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withHeaders( - Map.of("Content-Type", contentTypeHeader, "Accept", contentTypeHeader) - ).withMethod(RestRequest.Method.POST).withPath("/some_index/some_type/_update_by_query").build(); - - // checks the type in the URL is propagated correctly to the request object - // only works after the request is dispatched, so its params are filled from url. - dispatchRequest(request); - - // RestUpdateByQueryAction itself doesn't check for a deprecated type usage - // checking here for a deprecation from its internal search request - assertCriticalWarnings(RestSearchAction.TYPES_DEPRECATION_MESSAGE); - } -} diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index a742e83255bbb..ed1cf905f7e9d 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -57,4 +57,5 @@ tasks.named("precommit").configure { tasks.named("yamlRestCompatTestTransform").configure({task -> task.skipTest("indices.sort/10_basic/Index Sort", "warning does not exist for compatibility") task.skipTest("search/330_fetch_fields/Test search rewrite", "warning does not exist for compatibility") + task.skipTestsByFilePattern("indices.create/synthetic_source*.yml", "@UpdateForV9 -> tests do not pass after bumping API version to 9 [ES-9597]") }) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/capabilities/TransportNodesCapabilitiesAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/capabilities/TransportNodesCapabilitiesAction.java index 8df34d882941a..fb7539ea218d1 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/capabilities/TransportNodesCapabilitiesAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/capabilities/TransportNodesCapabilitiesAction.java @@ -19,6 +19,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.core.RestApiVersion; +import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.features.FeatureService; import org.elasticsearch.injection.guice.Inject; import org.elasticsearch.rest.RestController; @@ -151,6 +152,10 @@ public NodeCapabilitiesRequest( this.restApiVersion = restApiVersion; } + @UpdateForV9 // 8.x blows up in a mixed cluster when trying to read RestApiVersion.forMajor(9) + // ./gradlew ":qa:mixed-cluster:v8.16.0#mixedClusterTest" + // -Dtests.class="org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT" + // -Dtests.method="test {p0=capabilities/10_basic/Capabilities API}" @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); @@ -159,7 +164,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(path); out.writeCollection(parameters, StreamOutput::writeString); out.writeCollection(capabilities, StreamOutput::writeString); - out.writeVInt(restApiVersion.major); + // Fixme: lies! all lies! + out.writeVInt(8); + // out.writeVInt(restApiVersion.major); } } } diff --git a/server/src/main/java/org/elasticsearch/rest/RestHandler.java b/server/src/main/java/org/elasticsearch/rest/RestHandler.java index b7dbe09db4a39..ede295fee9f4d 100644 --- a/server/src/main/java/org/elasticsearch/rest/RestHandler.java +++ b/server/src/main/java/org/elasticsearch/rest/RestHandler.java @@ -195,8 +195,9 @@ private RouteBuilder(Method method, String path) { } /** - * Marks that the route being built has been deprecated (for some reason -- the deprecationMessage), and notes the major - * version in which that deprecation occurred. + * Marks that the route being built has been deprecated (for some reason -- the deprecationMessage) for removal. Notes the last + * major version in which the path is fully supported without compatibility headers. If this path is being replaced by another + * then use {@link #replaces(Method, String, RestApiVersion)} instead. *

* For example: * 

 {@code
@@ -205,55 +206,57 @@ private RouteBuilder(Method method, String path) {
              *  .build()}
* * @param deprecationMessage the user-visible explanation of this deprecation - * @param deprecatedInVersion the major version in which the deprecation occurred + * @param lastFullySupportedVersion the last {@link RestApiVersion} (i.e. 7) for which this route is fully supported. + * The next major version (i.e. 8) will require compatibility header(s). (;compatible-with=7) + * The next major version (i.e. 9) will have no support whatsoever for this route. * @return a reference to this object. */ - public RouteBuilder deprecated(String deprecationMessage, RestApiVersion deprecatedInVersion) { + public RouteBuilder deprecated(String deprecationMessage, RestApiVersion lastFullySupportedVersion) { assert this.replacedRoute == null; - this.restApiVersion = Objects.requireNonNull(deprecatedInVersion); + this.restApiVersion = Objects.requireNonNull(lastFullySupportedVersion); this.deprecationMessage = Objects.requireNonNull(deprecationMessage); return this; } /** - * Marks that the route being built has been deprecated (for some reason -- the deprecationMessage), and notes the major - * version in which that deprecation occurred. + * Marks that the route being built replaces another route, and notes the last major version in which the path is fully + * supported without compatibility headers. *

* For example: * 

 {@code
-             * Route.builder(GET, "_upgrade")
-             *  .deprecated("The _upgrade API is no longer useful and will be removed.", RestApiVersion.V_7)
-             *  .build()}
+ * Route.builder(GET, "/_security/user/") + * .replaces(GET, "/_xpack/security/user/", RestApiVersion.V_7).build()} * - * @param deprecationMessage the user-visible explanation of this deprecation - * @param deprecationLevel the level at which to log the deprecation - * @param deprecatedInVersion the major version in which the deprecation occurred + * @param method the method being replaced + * @param path the path being replaced + * @param lastFullySupportedVersion the last {@link RestApiVersion} (i.e. 7) for which this route is fully supported. + * The next major version (i.e. 8) will require compatibility header(s). (;compatible-with=7) + * The next major version (i.e. 9) will have no support whatsoever for this route. * @return a reference to this object. */ - public RouteBuilder deprecated(String deprecationMessage, Level deprecationLevel, RestApiVersion deprecatedInVersion) { - assert this.replacedRoute == null; - this.restApiVersion = Objects.requireNonNull(deprecatedInVersion); - this.deprecationMessage = Objects.requireNonNull(deprecationMessage); - this.deprecationLevel = deprecationLevel; + public RouteBuilder replaces(Method method, String path, RestApiVersion lastFullySupportedVersion) { + assert this.deprecationMessage == null; + this.replacedRoute = new Route(method, path, lastFullySupportedVersion, null, null, null); return this; } /** - * Marks that the route being built replaces another route, and notes the major version in which that replacement occurred. + * Marks that the route being built has been deprecated (for some reason -- the deprecationMessage), but will not be removed. *

* For example: * 

 {@code
-             * Route.builder(GET, "/_security/user/")
-             *   .replaces(GET, "/_xpack/security/user/", RestApiVersion.V_7).build()}
+ * Route.builder(GET, "_upgrade") + * .deprecated("The _upgrade API is no longer useful but will not be removed.") + * .build()} * - * @param method the method being replaced - * @param path the path being replaced - * @param replacedInVersion the major version in which the replacement occurred + * @param deprecationMessage the user-visible explanation of this deprecation * @return a reference to this object. */ - public RouteBuilder replaces(Method method, String path, RestApiVersion replacedInVersion) { - assert this.deprecationMessage == null; - this.replacedRoute = new Route(method, path, replacedInVersion, null, null, null); + public RouteBuilder deprecateAndKeep(String deprecationMessage) { + assert this.replacedRoute == null; + this.restApiVersion = RestApiVersion.current(); + this.deprecationMessage = Objects.requireNonNull(deprecationMessage); + this.deprecationLevel = Level.WARN; return this; } diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateAction.java index 362713a8f48cb..f70d9351e69c9 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateAction.java @@ -9,7 +9,6 @@ package org.elasticsearch.rest.action.admin.indices; -import org.apache.logging.log4j.Level; import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.common.Strings; @@ -41,8 +40,8 @@ public class RestPutIndexTemplateAction extends BaseRestHandler { @Override public List routes() { return List.of( - Route.builder(POST, "/_template/{name}").deprecated(DEPRECATION_WARNING, Level.WARN, DEPRECATION_VERSION).build(), - Route.builder(PUT, "/_template/{name}").deprecated(DEPRECATION_WARNING, Level.WARN, DEPRECATION_VERSION).build() + Route.builder(POST, "/_template/{name}").deprecateAndKeep(DEPRECATION_WARNING).build(), + Route.builder(PUT, "/_template/{name}").deprecateAndKeep(DEPRECATION_WARNING).build() ); } diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/RestKnnSearchAction.java b/server/src/main/java/org/elasticsearch/rest/action/search/RestKnnSearchAction.java index dd868b8321f1d..9b9be199eedae 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/search/RestKnnSearchAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/search/RestKnnSearchAction.java @@ -11,7 +11,7 @@ import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.client.internal.node.NodeClient; -import org.elasticsearch.core.RestApiVersion; +import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestCancellableNodeClient; @@ -34,11 +34,18 @@ public class RestKnnSearchAction extends BaseRestHandler { public RestKnnSearchAction() {} + @UpdateForV9 // these routes were ".deprecated" in RestApiVersion.V_8 which will require use of REST API compatibility headers to access + // this API in v9. It is unclear if this was intentional for v9, and the code has been updated to ".deprecateAndKeep" which will + // continue to emit deprecations warnings but will not require any special headers to access the API in v9. + // Please review and update the code and tests as needed. The original code remains commented out below for reference. @Override public List routes() { + return List.of( - Route.builder(GET, "{index}/_knn_search").deprecated(DEPRECATION_MESSAGE, RestApiVersion.V_8).build(), - Route.builder(POST, "{index}/_knn_search").deprecated(DEPRECATION_MESSAGE, RestApiVersion.V_8).build() + // Route.builder(GET, "{index}/_knn_search").deprecated(DEPRECATION_MESSAGE, RestApiVersion.V_8).build(), + // Route.builder(POST, "{index}/_knn_search").deprecated(DEPRECATION_MESSAGE, RestApiVersion.V_8).build() + Route.builder(GET, "{index}/_knn_search").deprecateAndKeep(DEPRECATION_MESSAGE).build(), + Route.builder(POST, "{index}/_knn_search").deprecateAndKeep(DEPRECATION_MESSAGE).build() ); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MovAvgPipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MovAvgPipelineAggregationBuilder.java index d163cafaffe2e..8326342df09f2 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MovAvgPipelineAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MovAvgPipelineAggregationBuilder.java @@ -16,6 +16,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.core.RestApiVersion; +import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.index.query.CommonTermsQueryBuilder; import org.elasticsearch.xcontent.ContextParser; import org.elasticsearch.xcontent.ParseField; @@ -32,6 +33,7 @@ * * @deprecated Only for 7.x rest compat */ +@UpdateForV9 // remove this since it's only for 7.x compat and 7.x compat will be removed in 9.0 @Deprecated public class MovAvgPipelineAggregationBuilder extends AbstractPipelineAggregationBuilder { private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(CommonTermsQueryBuilder.class); diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java index 1f0e3c52d55bb..a7fa81eb24a57 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java @@ -21,7 +21,6 @@ import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.indices.IndicesModule; @@ -30,9 +29,7 @@ import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.xcontent.XContentParseException; -import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentType; -import org.elasticsearch.xcontent.json.JsonXContent; import org.junit.Before; import java.io.IOException; @@ -276,72 +273,4 @@ public void testValidation() { ); } } - - public void testParsingWithType() throws Exception { - final XContentBuilder builder = XContentFactory.jsonBuilder() - .startObject() - .startObject("conditions") - .field("max_age", "10d") - .field("max_docs", 100) - .endObject() - .startObject("mappings") - .startObject("type1") - .startObject("properties") - .startObject("field1") - .field("type", "string") - .field("index", "not_analyzed") - .endObject() - .endObject() - .endObject() - .endObject() - .startObject("settings") - .field("number_of_shards", 10) - .endObject() - .startObject("aliases") - .startObject("alias1") - .endObject() - .endObject() - .endObject(); - - try ( - XContentParser parser = createParserWithCompatibilityFor( - JsonXContent.jsonXContent, - BytesReference.bytes(builder).utf8ToString(), - RestApiVersion.V_7 - ) - ) { - final RolloverRequest request = new RolloverRequest(randomAlphaOfLength(10), randomAlphaOfLength(10)); - request.fromXContent(true, parser); - Map> conditions = request.getConditions().getConditions(); - assertThat(conditions.size(), equalTo(2)); - assertThat(request.getCreateIndexRequest().mappings(), equalTo(""" - {"_doc":{"properties":{"field1":{"index":"not_analyzed","type":"string"}}}}""")); - } - } - - public void testTypedRequestWithoutIncludeTypeName() throws IOException { - final XContentBuilder builder = XContentFactory.jsonBuilder() - .startObject() - .startObject("mappings") - .startObject("_doc") - .startObject("properties") - .startObject("field1") - .field("type", "string") - .field("index", "not_analyzed") - .endObject() - .endObject() - .endObject() - .endObject() - .endObject(); - try ( - XContentParser parser = createParserWithCompatibilityFor( - JsonXContent.jsonXContent, - BytesReference.bytes(builder).utf8ToString(), - RestApiVersion.V_7 - ) - ) { - final RolloverRequest request = new RolloverRequest(randomAlphaOfLength(10), randomAlphaOfLength(10)); - expectThrows(IllegalArgumentException.class, () -> request.fromXContent(false, parser)); - } - } } diff --git a/server/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java b/server/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java index 8211fc8dfa4c6..f2bc561792991 100644 --- a/server/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java @@ -471,52 +471,6 @@ public void testWritingExpandWildcards() throws IOException { ); } - public void testEmptyFirstLine1() throws Exception { - MultiSearchRequest request = parseMultiSearchRequestFromString(""" - - - { "query": {"match_all": {}}} - {} - { "query": {"match_all": {}}} - - { "query": {"match_all": {}}} - {} - { "query": {"match_all": {}}} - """, RestApiVersion.V_7); - assertThat(request.requests().size(), equalTo(4)); - for (SearchRequest searchRequest : request.requests()) { - assertThat(searchRequest.indices().length, equalTo(0)); - assertThat(searchRequest.source().query(), instanceOf(MatchAllQueryBuilder.class)); - } - assertCriticalWarnings( - "support for empty first line before any action metadata in msearch API is deprecated and will be removed " - + "in the next major version" - ); - } - - public void testEmptyFirstLine2() throws Exception { - MultiSearchRequest request = parseMultiSearchRequestFromString(""" - - {} - { "query": {"match_all": {}}} - - { "query": {"match_all": {}}} - {} - { "query": {"match_all": {}}} - - { "query": {"match_all": {}}} - """, RestApiVersion.V_7); - assertThat(request.requests().size(), equalTo(4)); - for (SearchRequest searchRequest : request.requests()) { - assertThat(searchRequest.indices().length, equalTo(0)); - assertThat(searchRequest.source().query(), instanceOf(MatchAllQueryBuilder.class)); - } - assertCriticalWarnings( - "support for empty first line before any action metadata in msearch API is deprecated and will be removed " - + "in the next major version" - ); - } - public void testTaskDescription() { MultiSearchRequest request = new MultiSearchRequest(); request.add(new SearchRequest().preference("abc")); diff --git a/server/src/test/java/org/elasticsearch/http/AbstractHttpServerTransportTests.java b/server/src/test/java/org/elasticsearch/http/AbstractHttpServerTransportTests.java index 981eae9d60694..77133516f37d5 100644 --- a/server/src/test/java/org/elasticsearch/http/AbstractHttpServerTransportTests.java +++ b/server/src/test/java/org/elasticsearch/http/AbstractHttpServerTransportTests.java @@ -420,15 +420,15 @@ protected void populatePerRequestThreadContext(RestRequest restRequest, ThreadCo } public void testHandlingCompatibleVersionParsingErrors() { - // a compatible version exception (v7 on accept and v8 on content-type) should be handled gracefully + // a compatible version exception (v8 on accept and v9 on content-type) should be handled gracefully final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); try ( AbstractHttpServerTransport transport = failureAssertingtHttpServerTransport(clusterSettings, Set.of("Accept", "Content-Type")) ) { Map> headers = new HashMap<>(); - headers.put("Accept", Collections.singletonList("aaa/bbb;compatible-with=7")); - headers.put("Content-Type", Collections.singletonList("aaa/bbb;compatible-with=8")); + headers.put("Accept", Collections.singletonList("aaa/bbb;compatible-with=8")); + headers.put("Content-Type", Collections.singletonList("aaa/bbb;compatible-with=9")); FakeRestRequest.FakeHttpRequest fakeHttpRequest = new FakeRestRequest.FakeHttpRequest( RestRequest.Method.GET, diff --git a/server/src/test/java/org/elasticsearch/rest/RestCompatibleVersionHelperTests.java b/server/src/test/java/org/elasticsearch/rest/RestCompatibleVersionHelperTests.java index 104d578ef969b..040ab9fd5c2e9 100644 --- a/server/src/test/java/org/elasticsearch/rest/RestCompatibleVersionHelperTests.java +++ b/server/src/test/java/org/elasticsearch/rest/RestCompatibleVersionHelperTests.java @@ -10,7 +10,6 @@ import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.core.RestApiVersion; -import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xcontent.ParsedMediaType; import org.hamcrest.CustomTypeSafeMatcher; @@ -165,8 +164,6 @@ public void testAcceptAndContentTypeCombinations() { assertThat(requestWith(acceptHeader(null), contentTypeHeader("application/json"), bodyPresent()), not(isCompatible())); } - @UpdateForV9 - @AwaitsFix(bugUrl = "this can be re-enabled once our rest api version is bumped to V_9") public void testObsoleteVersion() { ElasticsearchStatusException e = expectThrows( ElasticsearchStatusException.class, @@ -213,14 +210,11 @@ public void testObsoleteVersion() { assertThat( e.getMessage(), equalTo( - "Content-Type version must be either version " - + CURRENT_VERSION - + " or " - + PREVIOUS_VERSION - + ", but found " - + OBSOLETE_VERSION - + ". " - + "Content-Type=" + "A compatible version is required on both Content-Type and Accept headers if either one has requested a " + + "compatible version and the compatible versions must match. " + + "Accept=" + + acceptHeader(PREVIOUS_VERSION) + + ", Content-Type=" + contentTypeHeader(OBSOLETE_VERSION) ) ); @@ -242,8 +236,8 @@ public void testMediaTypeCombinations() { assertThat( requestWith( - acceptHeader("application/vnd.elasticsearch+json;compatible-with=7"), - contentTypeHeader("application/vnd.elasticsearch+cbor;compatible-with=7"), + acceptHeader("application/vnd.elasticsearch+json;compatible-with=8"), + contentTypeHeader("application/vnd.elasticsearch+cbor;compatible-with=8"), bodyPresent() ), isCompatible() @@ -253,8 +247,8 @@ public void testMediaTypeCombinations() { expectThrows( ElasticsearchStatusException.class, () -> requestWith( - acceptHeader("application/vnd.elasticsearch+json;compatible-with=7"), - contentTypeHeader("application/vnd.elasticsearch+cbor;compatible-with=8"), + acceptHeader("application/vnd.elasticsearch+json;compatible-with=8"), + contentTypeHeader("application/vnd.elasticsearch+cbor;compatible-with=9"), bodyPresent() ) ); @@ -273,20 +267,20 @@ public void testTextMediaTypes() { // versioned assertThat( requestWith( - acceptHeader("text/vnd.elasticsearch+tab-separated-values;compatible-with=7"), - contentTypeHeader(7), + acceptHeader("text/vnd.elasticsearch+tab-separated-values;compatible-with=8"), + contentTypeHeader(8), bodyNotPresent() ), isCompatible() ); assertThat( - requestWith(acceptHeader("text/vnd.elasticsearch+plain;compatible-with=7"), contentTypeHeader(7), bodyNotPresent()), + requestWith(acceptHeader("text/vnd.elasticsearch+plain;compatible-with=8"), contentTypeHeader(8), bodyNotPresent()), isCompatible() ); assertThat( - requestWith(acceptHeader("text/vnd.elasticsearch+csv;compatible-with=7"), contentTypeHeader(7), bodyNotPresent()), + requestWith(acceptHeader("text/vnd.elasticsearch+csv;compatible-with=8"), contentTypeHeader(8), bodyNotPresent()), isCompatible() ); } diff --git a/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexActionTests.java index 601905635ff5e..2682a8c778168 100644 --- a/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexActionTests.java +++ b/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexActionTests.java @@ -9,29 +9,15 @@ package org.elasticsearch.rest.action.admin.indices; -import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; -import org.elasticsearch.client.internal.node.NodeClient; -import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.core.RestApiVersion; -import org.elasticsearch.rest.RestRequest; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.rest.FakeRestRequest; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; -import org.elasticsearch.xcontent.XContentType; import java.io.IOException; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; import java.util.Map; -import static org.elasticsearch.rest.BaseRestHandler.INCLUDE_TYPE_NAME_PARAMETER; -import static org.hamcrest.Matchers.equalTo; -import static org.mockito.Mockito.mock; - public class RestCreateIndexActionTests extends ESTestCase { public void testPrepareTypelessRequest() throws IOException { @@ -99,59 +85,4 @@ public void testMalformedMappings() throws IOException { Map source = RestCreateIndexAction.prepareMappings(contentAsMap); assertEquals(contentAsMap, source); } - - public void testIncludeTypeName() throws IOException { - RestCreateIndexAction action = new RestCreateIndexAction(); - List compatibleMediaType = Collections.singletonList(randomCompatibleMediaType(RestApiVersion.V_7)); - - Map params = new HashMap<>(); - params.put(INCLUDE_TYPE_NAME_PARAMETER, randomFrom("true", "false")); - RestRequest deprecatedRequest = new FakeRestRequest.Builder(xContentRegistry()).withHeaders(Map.of("Accept", compatibleMediaType)) - .withMethod(RestRequest.Method.PUT) - .withPath("/some_index") - .withParams(params) - .build(); - - action.prepareRequest(deprecatedRequest, mock(NodeClient.class)); - assertCriticalWarnings(RestCreateIndexAction.TYPES_DEPRECATION_MESSAGE); - - RestRequest validRequest = new FakeRestRequest.Builder(xContentRegistry()).withMethod(RestRequest.Method.PUT) - .withPath("/some_index") - .build(); - action.prepareRequest(validRequest, mock(NodeClient.class)); - } - - public void testTypeInMapping() throws IOException { - RestCreateIndexAction action = new RestCreateIndexAction(); - - List contentTypeHeader = Collections.singletonList(compatibleMediaType(XContentType.VND_JSON, RestApiVersion.V_7)); - - String content = """ - { - "mappings": { - "some_type": { - "properties": { - "field1": { - "type": "text" - } - } - } - } - }"""; - - Map params = new HashMap<>(); - params.put(RestCreateIndexAction.INCLUDE_TYPE_NAME_PARAMETER, "true"); - RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withMethod(RestRequest.Method.PUT) - .withHeaders(Map.of("Content-Type", contentTypeHeader, "Accept", contentTypeHeader)) - .withPath("/some_index") - .withParams(params) - .withContent(new BytesArray(content), null) - .build(); - - CreateIndexRequest createIndexRequest = RestCreateIndexAction.prepareRequestV7(request); - // some_type is replaced with _doc - assertThat(createIndexRequest.mappings(), equalTo(""" - {"_doc":{"properties":{"field1":{"type":"text"}}}}""")); - assertCriticalWarnings(RestCreateIndexAction.TYPES_DEPRECATION_MESSAGE); - } } diff --git a/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestGetIndicesActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestGetIndicesActionTests.java deleted file mode 100644 index 33e7b7fa21382..0000000000000 --- a/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestGetIndicesActionTests.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.rest.action.admin.indices; - -import org.elasticsearch.client.internal.node.NodeClient; -import org.elasticsearch.core.RestApiVersion; -import org.elasticsearch.rest.RestRequest; -import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.rest.FakeRestRequest; - -import java.io.IOException; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static org.elasticsearch.rest.BaseRestHandler.INCLUDE_TYPE_NAME_PARAMETER; -import static org.mockito.Mockito.mock; - -public final class RestGetIndicesActionTests extends ESTestCase { - final List contentTypeHeader = Collections.singletonList(randomCompatibleMediaType(RestApiVersion.V_7)); - - /** - * Test that setting the "include_type_name" parameter raises a warning for the GET request - */ - public void testIncludeTypeNamesWarning() throws IOException { - Map params = new HashMap<>(); - params.put(INCLUDE_TYPE_NAME_PARAMETER, randomFrom("true", "false")); - RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withHeaders( - Map.of("Content-Type", contentTypeHeader, "Accept", contentTypeHeader) - ).withMethod(RestRequest.Method.GET).withPath("/some_index").withParams(params).build(); - - RestGetIndicesAction handler = new RestGetIndicesAction(); - handler.prepareRequest(request, mock(NodeClient.class)); - assertCriticalWarnings(RestGetIndicesAction.TYPES_DEPRECATION_MESSAGE); - - // the same request without the parameter should pass without warning - request = new FakeRestRequest.Builder(xContentRegistry()).withHeaders( - Map.of("Content-Type", contentTypeHeader, "Accept", contentTypeHeader) - ).withMethod(RestRequest.Method.GET).withPath("/some_index").build(); - handler.prepareRequest(request, mock(NodeClient.class)); - } - - /** - * Test that setting the "include_type_name" parameter doesn't raises a warning if the HEAD method is used (indices.exists) - */ - public void testIncludeTypeNamesWarningExists() throws IOException { - Map params = new HashMap<>(); - params.put(INCLUDE_TYPE_NAME_PARAMETER, randomFrom("true", "false")); - RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withHeaders( - Map.of("Content-Type", contentTypeHeader, "Accept", contentTypeHeader) - ).withMethod(RestRequest.Method.HEAD).withPath("/some_index").withParams(params).build(); - - RestGetIndicesAction handler = new RestGetIndicesAction(); - handler.prepareRequest(request, mock(NodeClient.class)); - } -} diff --git a/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateActionTests.java deleted file mode 100644 index 5728e902aff6b..0000000000000 --- a/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateActionTests.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.rest.action.admin.indices; - -import org.elasticsearch.client.internal.node.NodeClient; -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.core.RestApiVersion; -import org.elasticsearch.rest.RestRequest; -import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.rest.FakeRestRequest; -import org.elasticsearch.xcontent.XContentBuilder; -import org.elasticsearch.xcontent.XContentFactory; -import org.elasticsearch.xcontent.XContentType; -import org.junit.Before; - -import java.io.IOException; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static org.elasticsearch.rest.BaseRestHandler.INCLUDE_TYPE_NAME_PARAMETER; -import static org.mockito.Mockito.mock; - -public final class RestPutIndexTemplateActionTests extends ESTestCase { - final List contentTypeHeader = Collections.singletonList(compatibleMediaType(XContentType.VND_JSON, RestApiVersion.V_7)); - - private RestPutIndexTemplateAction action; - - @Before - public void setUpAction() { - action = new RestPutIndexTemplateAction(); - } - - public void testIncludeTypeName() throws IOException { - XContentBuilder typedContent = XContentFactory.jsonBuilder() - .startObject() - .startObject("mappings") - .startObject("my_doc") - .startObject("properties") - .startObject("field1") - .field("type", "keyword") - .endObject() - .startObject("field2") - .field("type", "text") - .endObject() - .endObject() - .endObject() - .endObject() - .startObject("aliases") - .startObject("read_alias") - .endObject() - .endObject() - .endObject(); - - Map params = new HashMap<>(); - params.put(INCLUDE_TYPE_NAME_PARAMETER, "true"); - RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withHeaders( - Map.of("Content-Type", contentTypeHeader, "Accept", contentTypeHeader) - ) - .withMethod(RestRequest.Method.PUT) - .withParams(params) - .withPath("/_template/_some_template") - .withContent(BytesReference.bytes(typedContent), null) - .build(); - action.prepareRequest(request, mock(NodeClient.class)); - assertCriticalWarnings(RestPutIndexTemplateAction.TYPES_DEPRECATION_MESSAGE); - } -} diff --git a/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestValidateQueryActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestValidateQueryActionTests.java index 2d719c1ed537d..16e651a12c4d6 100644 --- a/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestValidateQueryActionTests.java +++ b/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestValidateQueryActionTests.java @@ -19,10 +19,7 @@ import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.EsExecutors; -import org.elasticsearch.common.util.concurrent.ThreadContext; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; -import org.elasticsearch.rest.RestChannel; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.search.AbstractSearchTestCase; @@ -41,7 +38,6 @@ import java.util.Collections; import java.util.HashMap; -import java.util.List; import java.util.Map; import static java.util.Collections.emptyMap; @@ -159,37 +155,4 @@ private RestRequest createRestRequest(String content) { .withContent(new BytesArray(content), XContentType.JSON) .build(); } - - public void testTypeInPath() { - List compatibleMediaType = Collections.singletonList(randomCompatibleMediaType(RestApiVersion.V_7)); - - RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withHeaders(Map.of("Accept", compatibleMediaType)) - .withMethod(RestRequest.Method.GET) - .withPath("/some_index/some_type/_validate/query") - .build(); - - performRequest(request); - assertCriticalWarnings(RestValidateQueryAction.TYPES_DEPRECATION_MESSAGE); - } - - public void testTypeParameter() { - List compatibleMediaType = Collections.singletonList(randomCompatibleMediaType(RestApiVersion.V_7)); - - Map params = new HashMap<>(); - params.put("type", "some_type"); - RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withHeaders(Map.of("Accept", compatibleMediaType)) - .withMethod(RestRequest.Method.GET) - .withPath("_validate/query") - .withParams(params) - .build(); - - performRequest(request); - assertCriticalWarnings(RestValidateQueryAction.TYPES_DEPRECATION_MESSAGE); - } - - private void performRequest(RestRequest request) { - RestChannel channel = new FakeRestChannel(request, false, 1); - ThreadContext threadContext = new ThreadContext(Settings.EMPTY); - controller.dispatchRequest(request, channel, threadContext); - } } diff --git a/server/src/test/java/org/elasticsearch/rest/action/document/RestDeleteActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/document/RestDeleteActionTests.java deleted file mode 100644 index d9141002eb32c..0000000000000 --- a/server/src/test/java/org/elasticsearch/rest/action/document/RestDeleteActionTests.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.rest.action.document; - -import org.elasticsearch.action.delete.DeleteResponse; -import org.elasticsearch.core.RestApiVersion; -import org.elasticsearch.rest.RestRequest; -import org.elasticsearch.test.rest.FakeRestRequest; -import org.elasticsearch.test.rest.RestActionTestCase; -import org.junit.Before; -import org.mockito.Mockito; - -import java.util.Collections; -import java.util.List; -import java.util.Map; - -public final class RestDeleteActionTests extends RestActionTestCase { - - final List contentTypeHeader = Collections.singletonList(randomCompatibleMediaType(RestApiVersion.V_7)); - - @Before - public void setUpAction() { - controller().registerHandler(new RestDeleteAction()); - verifyingClient.setExecuteVerifier((actionType, request) -> Mockito.mock(DeleteResponse.class)); - verifyingClient.setExecuteLocallyVerifier((actionType, request) -> Mockito.mock(DeleteResponse.class)); - } - - public void testTypeInPath() { - RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withHeaders(Map.of("Accept", contentTypeHeader)) - .withMethod(RestRequest.Method.DELETE) - .withPath("/some_index/some_type/some_id") - .build(); - dispatchRequest(request); - assertCriticalWarnings(RestDeleteAction.TYPES_DEPRECATION_MESSAGE); - - RestRequest validRequest = new FakeRestRequest.Builder(xContentRegistry()).withHeaders(Map.of("Accept", contentTypeHeader)) - .withMethod(RestRequest.Method.DELETE) - .withPath("/some_index/_doc/some_id") - .build(); - dispatchRequest(validRequest); - } -} diff --git a/server/src/test/java/org/elasticsearch/rest/action/document/RestGetActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/document/RestGetActionTests.java deleted file mode 100644 index 0a9abacd82635..0000000000000 --- a/server/src/test/java/org/elasticsearch/rest/action/document/RestGetActionTests.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.rest.action.document; - -import org.elasticsearch.action.get.GetRequest; -import org.elasticsearch.action.get.GetResponse; -import org.elasticsearch.core.RestApiVersion; -import org.elasticsearch.rest.RestRequest; -import org.elasticsearch.test.rest.FakeRestRequest; -import org.elasticsearch.test.rest.RestActionTestCase; -import org.junit.Before; -import org.mockito.Mockito; - -import java.util.Collections; -import java.util.List; -import java.util.Map; - -import static org.hamcrest.Matchers.instanceOf; - -public final class RestGetActionTests extends RestActionTestCase { - final List contentTypeHeader = Collections.singletonList(randomCompatibleMediaType(RestApiVersion.V_7)); - - @Before - public void setUpAction() { - controller().registerHandler(new RestGetAction()); - verifyingClient.setExecuteVerifier((actionType, request) -> { - assertThat(request, instanceOf(GetRequest.class)); - return Mockito.mock(GetResponse.class); - }); - } - - public void testTypeInPath() { - testTypeInPath(RestRequest.Method.GET); - testTypeInPath(RestRequest.Method.HEAD); - } - - private void testTypeInPath(RestRequest.Method method) { - FakeRestRequest.Builder deprecatedRequest = new FakeRestRequest.Builder(xContentRegistry()).withHeaders( - Map.of("Content-Type", contentTypeHeader, "Accept", contentTypeHeader) - ).withPath("/some_index/some_type/some_id"); - dispatchRequest(deprecatedRequest.withMethod(method).build()); - assertCriticalWarnings(RestGetAction.TYPES_DEPRECATION_MESSAGE); - } -} diff --git a/server/src/test/java/org/elasticsearch/rest/action/document/RestGetSourceActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/document/RestGetSourceActionTests.java index 17840990d5b10..7cec10299280e 100644 --- a/server/src/test/java/org/elasticsearch/rest/action/document/RestGetSourceActionTests.java +++ b/server/src/test/java/org/elasticsearch/rest/action/document/RestGetSourceActionTests.java @@ -14,7 +14,6 @@ import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.index.get.GetResult; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.RestResponse; @@ -26,12 +25,6 @@ import org.junit.Before; import org.mockito.Mockito; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - import static java.util.Collections.emptyMap; import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO; import static org.elasticsearch.rest.RestStatus.OK; @@ -43,7 +36,6 @@ public final class RestGetSourceActionTests extends RestActionTestCase { private static RestRequest request = new FakeRestRequest(); private static FakeRestChannel channel = new FakeRestChannel(request, true, 0); private static RestGetSourceResponseListener listener = new RestGetSourceResponseListener(channel, request); - private final List compatibleMediaType = Collections.singletonList(randomCompatibleMediaType(RestApiVersion.V_7)); @Before public void setUpAction() { @@ -89,36 +81,4 @@ public void testRestGetSourceActionWithMissingDocumentSource() { assertThat(exception.getMessage(), equalTo("Source not found [index1]/[1]")); } - - /** - * test deprecation is logged if type is used in path - */ - public void testTypeInPath() { - for (RestRequest.Method method : Arrays.asList(RestRequest.Method.GET, RestRequest.Method.HEAD)) { - RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withHeaders(Map.of("Accept", compatibleMediaType)) - .withMethod(method) - .withPath("/some_index/some_type/id/_source") - .build(); - dispatchRequest(request); - assertCriticalWarnings(RestGetSourceAction.TYPES_DEPRECATION_MESSAGE); - } - } - - /** - * test deprecation is logged if type is used as parameter - */ - public void testTypeParameter() { - Map params = new HashMap<>(); - params.put("type", "some_type"); - for (RestRequest.Method method : Arrays.asList(RestRequest.Method.GET, RestRequest.Method.HEAD)) { - RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withHeaders(Map.of("Accept", compatibleMediaType)) - .withMethod(method) - .withPath("/some_index/_source/id") - .withParams(params) - .build(); - dispatchRequest(request); - assertCriticalWarnings(RestGetSourceAction.TYPES_DEPRECATION_MESSAGE); - } - } - } diff --git a/server/src/test/java/org/elasticsearch/rest/action/document/RestIndexActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/document/RestIndexActionTests.java index b7f0fa3c1c707..1aa53382666ef 100644 --- a/server/src/test/java/org/elasticsearch/rest/action/document/RestIndexActionTests.java +++ b/server/src/test/java/org/elasticsearch/rest/action/document/RestIndexActionTests.java @@ -19,7 +19,6 @@ import org.elasticsearch.cluster.node.DiscoveryNodeUtils; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.common.bytes.BytesArray; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.document.RestIndexAction.AutoIdHandler; @@ -29,18 +28,12 @@ import org.elasticsearch.xcontent.XContentType; import org.junit.Before; -import java.util.Collections; -import java.util.List; -import java.util.Map; import java.util.concurrent.atomic.AtomicReference; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; public final class RestIndexActionTests extends RestActionTestCase { - - final List contentTypeHeader = Collections.singletonList(randomCompatibleMediaType(RestApiVersion.V_7)); - private final AtomicReference clusterStateSupplier = new AtomicReference<>(); @Before @@ -85,34 +78,4 @@ private void checkAutoIdOpType(Version minClusterVersion, DocWriteRequest.OpType dispatchRequest(autoIdRequest); assertThat(executeCalled.get(), equalTo(true)); } - - public void testTypeInPath() { - // using CompatibleRestIndexAction - RestRequest deprecatedRequest = new FakeRestRequest.Builder(xContentRegistry()).withMethod(RestRequest.Method.PUT) - .withHeaders(Map.of("Content-Type", contentTypeHeader, "Accept", contentTypeHeader)) - .withPath("/some_index/some_type/some_id") - .build(); - dispatchRequest(deprecatedRequest); - assertCriticalWarnings(RestIndexAction.TYPES_DEPRECATION_MESSAGE); - } - - public void testCreateWithTypeInPath() { - // using CompatibleCreateHandler - RestRequest deprecatedRequest = new FakeRestRequest.Builder(xContentRegistry()).withMethod(RestRequest.Method.PUT) - .withHeaders(Map.of("Content-Type", contentTypeHeader, "Accept", contentTypeHeader)) - .withPath("/some_index/some_type/some_id/_create") - .build(); - dispatchRequest(deprecatedRequest); - assertCriticalWarnings(RestIndexAction.TYPES_DEPRECATION_MESSAGE); - } - - public void testAutoIdWithType() { - // using CompatibleAutoIdHandler - RestRequest deprecatedRequest = new FakeRestRequest.Builder(xContentRegistry()).withMethod(RestRequest.Method.POST) - .withHeaders(Map.of("Content-Type", contentTypeHeader, "Accept", contentTypeHeader)) - .withPath("/some_index/some_type/") - .build(); - dispatchRequest(deprecatedRequest); - assertCriticalWarnings(RestIndexAction.TYPES_DEPRECATION_MESSAGE); - } } diff --git a/server/src/test/java/org/elasticsearch/rest/action/document/RestMultiGetActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/document/RestMultiGetActionTests.java deleted file mode 100644 index ed793f3127a93..0000000000000 --- a/server/src/test/java/org/elasticsearch/rest/action/document/RestMultiGetActionTests.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.rest.action.document; - -import org.elasticsearch.action.get.MultiGetRequest; -import org.elasticsearch.action.get.MultiGetResponse; -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.core.RestApiVersion; -import org.elasticsearch.rest.RestRequest; -import org.elasticsearch.test.rest.FakeRestRequest; -import org.elasticsearch.test.rest.RestActionTestCase; -import org.elasticsearch.xcontent.XContentBuilder; -import org.elasticsearch.xcontent.XContentFactory; -import org.elasticsearch.xcontent.XContentType; -import org.junit.Before; -import org.mockito.Mockito; - -import java.util.Collections; -import java.util.List; -import java.util.Map; - -import static org.hamcrest.Matchers.instanceOf; - -public final class RestMultiGetActionTests extends RestActionTestCase { - XContentType VND_TYPE = randomVendorType(); - List contentTypeHeader = Collections.singletonList(compatibleMediaType(VND_TYPE, RestApiVersion.V_7)); - - @Before - public void setUpAction() { - controller().registerHandler(new RestMultiGetAction(Settings.EMPTY)); - verifyingClient.setExecuteVerifier((actionType, request) -> { - assertThat(request, instanceOf(MultiGetRequest.class)); - return Mockito.mock(MultiGetResponse.class); - }); - } - - public void testTypeInPath() { - RestRequest deprecatedRequest = new FakeRestRequest.Builder(xContentRegistry()).withHeaders( - Map.of("Content-Type", contentTypeHeader, "Accept", contentTypeHeader) - ).withMethod(RestRequest.Method.GET).withPath("some_index/some_type/_mget").build(); - dispatchRequest(deprecatedRequest); - assertCriticalWarnings(RestMultiGetAction.TYPES_DEPRECATION_MESSAGE); - } - - public void testTypeInBody() throws Exception { - XContentBuilder content = XContentFactory.contentBuilder(VND_TYPE) - .startObject() - .startArray("docs") - .startObject() - .field("_index", "some_index") - .field("_type", "_doc") - .field("_id", "2") - .endObject() - .startObject() - .field("_index", "test") - .field("_id", "2") - .endObject() - .endArray() - .endObject(); - - RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withPath("_mget") - .withHeaders(Map.of("Content-Type", contentTypeHeader, "Accept", contentTypeHeader)) - .withContent(BytesReference.bytes(content), null) - .build(); - dispatchRequest(request); - assertCriticalWarnings(RestMultiGetAction.TYPES_DEPRECATION_MESSAGE); - } - -} diff --git a/server/src/test/java/org/elasticsearch/rest/action/document/RestMultiTermVectorsActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/document/RestMultiTermVectorsActionTests.java deleted file mode 100644 index 0e247d70b2ba3..0000000000000 --- a/server/src/test/java/org/elasticsearch/rest/action/document/RestMultiTermVectorsActionTests.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.rest.action.document; - -import org.elasticsearch.action.termvectors.MultiTermVectorsResponse; -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.core.RestApiVersion; -import org.elasticsearch.rest.RestRequest; -import org.elasticsearch.test.rest.FakeRestRequest; -import org.elasticsearch.test.rest.RestActionTestCase; -import org.elasticsearch.xcontent.XContentBuilder; -import org.elasticsearch.xcontent.XContentFactory; -import org.elasticsearch.xcontent.XContentType; -import org.junit.Before; -import org.mockito.Mockito; - -import java.io.IOException; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -public final class RestMultiTermVectorsActionTests extends RestActionTestCase { - final List contentTypeHeader = Collections.singletonList(compatibleMediaType(XContentType.VND_JSON, RestApiVersion.V_7)); - - @Before - public void setUpAction() { - controller().registerHandler(new RestMultiTermVectorsAction()); - verifyingClient.setExecuteVerifier((actionType, request) -> Mockito.mock(MultiTermVectorsResponse.class)); - verifyingClient.setExecuteLocallyVerifier((actionType, request) -> Mockito.mock(MultiTermVectorsResponse.class)); - } - - public void testTypeInPath() { - RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withHeaders( - Map.of("Content-Type", contentTypeHeader, "Accept", contentTypeHeader) - ).withMethod(RestRequest.Method.POST).withPath("/some_index/some_type/_mtermvectors").build(); - - dispatchRequest(request); - assertCriticalWarnings(RestMultiTermVectorsAction.TYPES_DEPRECATION_MESSAGE); - } - - public void testTypeParameter() { - Map params = new HashMap<>(); - params.put("type", "some_type"); - - RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withHeaders( - Map.of("Content-Type", contentTypeHeader, "Accept", contentTypeHeader) - ).withPath("/some_index/_mtermvectors").withParams(params).build(); - - dispatchRequest(request); - assertCriticalWarnings(RestMultiTermVectorsAction.TYPES_DEPRECATION_MESSAGE); - } - - public void testTypeInBody() throws IOException { - XContentBuilder content = XContentFactory.jsonBuilder() - .startObject() - .startArray("docs") - .startObject() - .field("_type", "some_type") - .field("_id", 1) - .endObject() - .endArray() - .endObject(); - - RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withHeaders( - Map.of("Content-Type", contentTypeHeader, "Accept", contentTypeHeader) - ) - .withMethod(RestRequest.Method.POST) - .withPath("/some_index/_mtermvectors") - .withContent(BytesReference.bytes(content), null) - .build(); - - dispatchRequest(request); - assertCriticalWarnings(RestTermVectorsAction.TYPES_DEPRECATION_MESSAGE); - } -} diff --git a/server/src/test/java/org/elasticsearch/rest/action/document/RestTermVectorsActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/document/RestTermVectorsActionTests.java deleted file mode 100644 index a69c167c12729..0000000000000 --- a/server/src/test/java/org/elasticsearch/rest/action/document/RestTermVectorsActionTests.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.rest.action.document; - -import org.elasticsearch.action.termvectors.TermVectorsResponse; -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.core.RestApiVersion; -import org.elasticsearch.rest.RestRequest; -import org.elasticsearch.test.rest.FakeRestRequest; -import org.elasticsearch.test.rest.RestActionTestCase; -import org.elasticsearch.xcontent.XContentBuilder; -import org.elasticsearch.xcontent.XContentFactory; -import org.elasticsearch.xcontent.XContentType; -import org.junit.Before; -import org.mockito.Mockito; - -import java.io.IOException; -import java.util.Collections; -import java.util.List; -import java.util.Map; - -public final class RestTermVectorsActionTests extends RestActionTestCase { - final List contentTypeHeader = Collections.singletonList(compatibleMediaType(XContentType.VND_JSON, RestApiVersion.V_7)); - - @Before - public void setUpAction() { - controller().registerHandler(new RestTermVectorsAction()); - // todo how to workaround this? we get AssertionError without this - verifyingClient.setExecuteVerifier((actionType, request) -> Mockito.mock(TermVectorsResponse.class)); - verifyingClient.setExecuteLocallyVerifier((actionType, request) -> Mockito.mock(TermVectorsResponse.class)); - } - - public void testTypeInPath() { - RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withHeaders( - Map.of("Content-Type", contentTypeHeader, "Accept", contentTypeHeader) - ).withMethod(RestRequest.Method.POST).withPath("/some_index/some_type/some_id/_termvectors").build(); - - dispatchRequest(request); - assertCriticalWarnings(RestTermVectorsAction.TYPES_DEPRECATION_MESSAGE); - } - - public void testTypeInBody() throws IOException { - XContentBuilder content = XContentFactory.jsonBuilder().startObject().field("_type", "some_type").field("_id", 1).endObject(); - - RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withHeaders( - Map.of("Content-Type", contentTypeHeader, "Accept", contentTypeHeader) - ) - .withMethod(RestRequest.Method.GET) - .withPath("/some_index/_termvectors/some_id") - .withContent(BytesReference.bytes(content), null) - .build(); - - dispatchRequest(request); - assertCriticalWarnings(RestTermVectorsAction.TYPES_DEPRECATION_MESSAGE); - } -} diff --git a/server/src/test/java/org/elasticsearch/rest/action/document/RestUpdateActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/document/RestUpdateActionTests.java index def6e8eb0375d..c68867649e25a 100644 --- a/server/src/test/java/org/elasticsearch/rest/action/document/RestUpdateActionTests.java +++ b/server/src/test/java/org/elasticsearch/rest/action/document/RestUpdateActionTests.java @@ -13,7 +13,6 @@ import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.common.bytes.BytesArray; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.index.VersionType; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.test.rest.FakeRestRequest; @@ -22,17 +21,13 @@ import org.junit.Before; import org.mockito.Mockito; -import java.util.Collections; import java.util.HashMap; -import java.util.List; import java.util.Map; import static org.hamcrest.CoreMatchers.containsString; import static org.mockito.Mockito.mock; public final class RestUpdateActionTests extends RestActionTestCase { - final List contentTypeHeader = Collections.singletonList(randomCompatibleMediaType(RestApiVersion.V_7)); - private RestUpdateAction action; @Before @@ -76,17 +71,4 @@ public void testUpdateDocVersion() { ) ); } - - public void testTypeInPath() { - RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withHeaders( - Map.of("Content-Type", contentTypeHeader, "Accept", contentTypeHeader) - ).withMethod(RestRequest.Method.POST).withPath("/some_index/some_type/some_id/_update").build(); - dispatchRequest(request); - assertCriticalWarnings(RestUpdateAction.TYPES_DEPRECATION_MESSAGE); - - RestRequest validRequest = new FakeRestRequest.Builder(xContentRegistry()).withHeaders( - Map.of("Content-Type", contentTypeHeader, "Accept", contentTypeHeader) - ).withMethod(RestRequest.Method.DELETE).withPath("/some_index/_update/some_id").build(); - dispatchRequest(validRequest); - } } diff --git a/server/src/test/java/org/elasticsearch/rest/action/search/RestCountActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/search/RestCountActionTests.java deleted file mode 100644 index e72511989f083..0000000000000 --- a/server/src/test/java/org/elasticsearch/rest/action/search/RestCountActionTests.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.rest.action.search; - -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.core.RestApiVersion; -import org.elasticsearch.rest.RestRequest; -import org.elasticsearch.rest.RestRequest.Method; -import org.elasticsearch.test.rest.FakeRestRequest; -import org.elasticsearch.test.rest.RestActionTestCase; -import org.junit.Before; -import org.mockito.Mockito; - -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static org.hamcrest.Matchers.instanceOf; - -public final class RestCountActionTests extends RestActionTestCase { - - final List contentTypeHeader = Collections.singletonList(randomCompatibleMediaType(RestApiVersion.V_7)); - - @Before - public void setUpAction() { - controller().registerHandler(new RestCountAction()); - verifyingClient.setExecuteVerifier((actionType, request) -> { - assertThat(request, instanceOf(SearchRequest.class)); - return Mockito.mock(SearchResponse.class); - }); - } - - public void testTypeInPath() { - RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withHeaders(Map.of("Accept", contentTypeHeader)) - .withMethod(Method.POST) - .withPath("/some_index/some_type/_count") - .build(); - - dispatchRequest(request); - assertCriticalWarnings(RestCountAction.TYPES_DEPRECATION_MESSAGE); - } - - public void testTypeParameter() { - Map params = new HashMap<>(); - params.put("type", "some_type"); - - RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withHeaders(Map.of("Accept", contentTypeHeader)) - .withMethod(Method.GET) - .withPath("/some_index/_count") - .withParams(params) - .build(); - - dispatchRequest(request); - assertCriticalWarnings(RestCountAction.TYPES_DEPRECATION_MESSAGE); - } -} diff --git a/server/src/test/java/org/elasticsearch/rest/action/search/RestExplainActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/search/RestExplainActionTests.java deleted file mode 100644 index ddbe7243d304b..0000000000000 --- a/server/src/test/java/org/elasticsearch/rest/action/search/RestExplainActionTests.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.rest.action.search; - -import org.elasticsearch.action.explain.ExplainResponse; -import org.elasticsearch.core.RestApiVersion; -import org.elasticsearch.rest.RestRequest; -import org.elasticsearch.test.rest.FakeRestRequest; -import org.elasticsearch.test.rest.RestActionTestCase; -import org.elasticsearch.xcontent.XContentType; -import org.junit.Before; -import org.mockito.Mockito; - -import java.util.Collections; -import java.util.List; -import java.util.Map; - -public final class RestExplainActionTests extends RestActionTestCase { - final List contentTypeHeader = Collections.singletonList(compatibleMediaType(XContentType.VND_JSON, RestApiVersion.V_7)); - - @Before - public void setUpAction() { - RestExplainAction action = new RestExplainAction(); - controller().registerHandler(action); - verifyingClient.setExecuteVerifier((actionType, request) -> Mockito.mock(ExplainResponse.class)); - verifyingClient.setExecuteLocallyVerifier((actionType, request) -> Mockito.mock(ExplainResponse.class)); - } - - public void testTypeInPath() { - RestRequest deprecatedRequest = new FakeRestRequest.Builder(xContentRegistry()).withHeaders(Map.of("Accept", contentTypeHeader)) - .withMethod(RestRequest.Method.GET) - .withPath("/some_index/some_type/some_id/_explain") - .build(); - dispatchRequest(deprecatedRequest); - assertCriticalWarnings(RestExplainAction.TYPES_DEPRECATION_MESSAGE); - - RestRequest validRequest = new FakeRestRequest.Builder(xContentRegistry()).withHeaders(Map.of("Accept", contentTypeHeader)) - .withMethod(RestRequest.Method.GET) - .withPath("/some_index/_explain/some_id") - .build(); - dispatchRequest(validRequest); - } - -} diff --git a/server/src/test/java/org/elasticsearch/rest/action/search/RestMultiSearchActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/search/RestMultiSearchActionTests.java deleted file mode 100644 index b77817e4c0258..0000000000000 --- a/server/src/test/java/org/elasticsearch/rest/action/search/RestMultiSearchActionTests.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.rest.action.search; - -import org.elasticsearch.action.search.MultiSearchResponse; -import org.elasticsearch.common.bytes.BytesArray; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.core.RestApiVersion; -import org.elasticsearch.rest.RestRequest; -import org.elasticsearch.test.rest.FakeRestRequest; -import org.elasticsearch.test.rest.RestActionTestCase; -import org.elasticsearch.usage.UsageService; -import org.elasticsearch.xcontent.XContentType; -import org.junit.Before; - -import java.nio.charset.StandardCharsets; -import java.util.Collections; -import java.util.List; -import java.util.Map; - -import static org.mockito.Mockito.mock; - -public final class RestMultiSearchActionTests extends RestActionTestCase { - final List contentTypeHeader = Collections.singletonList(compatibleMediaType(XContentType.VND_JSON, RestApiVersion.V_7)); - - @Before - public void setUpAction() { - RestMultiSearchAction action = new RestMultiSearchAction(Settings.EMPTY, new UsageService().getSearchUsageHolder(), nf -> false); - controller().registerHandler(action); - verifyingClient.setExecuteVerifier((actionType, request) -> mock(MultiSearchResponse.class)); - verifyingClient.setExecuteLocallyVerifier((actionType, request) -> mock(MultiSearchResponse.class)); - } - - public void testTypeInPath() { - String content = "{ \"index\": \"some_index\" } \n {} \n"; - BytesArray bytesContent = new BytesArray(content.getBytes(StandardCharsets.UTF_8)); - - RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withHeaders( - Map.of("Content-Type", contentTypeHeader, "Accept", contentTypeHeader) - ).withMethod(RestRequest.Method.GET).withPath("/some_index/some_type/_msearch").withContent(bytesContent, null).build(); - - dispatchRequest(request); - assertCriticalWarnings(RestMultiSearchAction.TYPES_DEPRECATION_MESSAGE); - } - - public void testTypeInBody() { - String content = "{ \"index\": \"some_index\", \"type\": \"some_type\" } \n {} \n"; - BytesArray bytesContent = new BytesArray(content.getBytes(StandardCharsets.UTF_8)); - - RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withHeaders( - Map.of("Content-Type", contentTypeHeader, "Accept", contentTypeHeader) - ).withMethod(RestRequest.Method.POST).withPath("/some_index/_msearch").withContent(bytesContent, null).build(); - - dispatchRequest(request); - assertCriticalWarnings(RestMultiSearchAction.TYPES_DEPRECATION_MESSAGE); - } - -} diff --git a/server/src/test/java/org/elasticsearch/rest/action/search/RestSearchActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/search/RestSearchActionTests.java index e207d150ac6cd..24f59a8c3abe7 100644 --- a/server/src/test/java/org/elasticsearch/rest/action/search/RestSearchActionTests.java +++ b/server/src/test/java/org/elasticsearch/rest/action/search/RestSearchActionTests.java @@ -11,7 +11,6 @@ import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.suggest.SuggestBuilder; @@ -23,7 +22,6 @@ import org.elasticsearch.usage.UsageService; import org.junit.Before; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -31,8 +29,6 @@ import static org.mockito.Mockito.mock; public final class RestSearchActionTests extends RestActionTestCase { - final List contentTypeHeader = Collections.singletonList(randomCompatibleMediaType(RestApiVersion.V_7)); - private RestSearchAction action; @Before @@ -43,27 +39,6 @@ public void setUpAction() { verifyingClient.setExecuteLocallyVerifier((actionType, request) -> mock(SearchResponse.class)); } - public void testTypeInPath() { - RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withHeaders( - Map.of("Content-Type", contentTypeHeader, "Accept", contentTypeHeader) - ).withMethod(RestRequest.Method.GET).withPath("/some_index/some_type/_search").build(); - - dispatchRequest(request); - assertCriticalWarnings(RestSearchAction.TYPES_DEPRECATION_MESSAGE); - } - - public void testTypeParameter() { - Map params = new HashMap<>(); - params.put("type", "some_type"); - - RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withHeaders( - Map.of("Content-Type", contentTypeHeader, "Accept", contentTypeHeader) - ).withMethod(RestRequest.Method.GET).withPath("/some_index/_search").withParams(params).build(); - - dispatchRequest(request); - assertCriticalWarnings(RestSearchAction.TYPES_DEPRECATION_MESSAGE); - } - /** * The "enable_fields_emulation" flag on search requests is a no-op but should not raise an error */ @@ -71,9 +46,10 @@ public void testEnableFieldsEmulationNoErrors() throws Exception { Map params = new HashMap<>(); params.put("enable_fields_emulation", "true"); - RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withHeaders( - Map.of("Content-Type", contentTypeHeader, "Accept", contentTypeHeader) - ).withMethod(RestRequest.Method.GET).withPath("/some_index/_search").withParams(params).build(); + RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withMethod(RestRequest.Method.GET) + .withPath("/some_index/_search") + .withParams(params) + .build(); action.handleRequest(request, new FakeRestChannel(request, false, 1), verifyingClient); } @@ -83,9 +59,10 @@ public void testValidateSearchRequest() { Map params = new HashMap<>(); params.put("rest_total_hits_as_int", "true"); - RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withHeaders( - Map.of("Content-Type", contentTypeHeader, "Accept", contentTypeHeader) - ).withMethod(RestRequest.Method.GET).withPath("/some_index/_search").withParams(params).build(); + RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withMethod(RestRequest.Method.GET) + .withPath("/some_index/_search") + .withParams(params) + .build(); SearchRequest searchRequest = new SearchRequest(); searchRequest.source(new SearchSourceBuilder().trackTotalHitsUpTo(100)); @@ -100,9 +77,10 @@ public void testValidateSearchRequest() { Map params = new HashMap<>(); params.put("search_type", randomFrom(SearchType.values()).name()); - RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withHeaders( - Map.of("Content-Type", contentTypeHeader, "Accept", contentTypeHeader) - ).withMethod(RestRequest.Method.GET).withPath("/some_index/_search").withParams(params).build(); + RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withMethod(RestRequest.Method.GET) + .withPath("/some_index/_search") + .withParams(params) + .build(); SearchRequest searchRequest = new SearchRequest(); KnnSearchBuilder knnSearch = new KnnSearchBuilder("vector", new float[] { 1, 1, 1 }, 10, 100, null); @@ -126,9 +104,10 @@ public void testIllegalSearchType() { Map params = new HashMap<>(); params.put("search_type", "some_search_type"); - RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withHeaders( - Map.of("Content-Type", contentTypeHeader, "Accept", contentTypeHeader) - ).withMethod(RestRequest.Method.GET).withPath("/some_index/_search").withParams(params).build(); + RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withMethod(RestRequest.Method.GET) + .withPath("/some_index/_search") + .withParams(params) + .build(); Exception ex = expectThrows(IllegalArgumentException.class, () -> action.prepareRequest(request, verifyingClient)); assertEquals("No search type for [some_search_type]", ex.getMessage()); diff --git a/server/src/test/java/org/elasticsearch/search/SearchModuleTests.java b/server/src/test/java/org/elasticsearch/search/SearchModuleTests.java index 0a3c2c939b456..9109cd6b89bed 100644 --- a/server/src/test/java/org/elasticsearch/search/SearchModuleTests.java +++ b/server/src/test/java/org/elasticsearch/search/SearchModuleTests.java @@ -18,12 +18,10 @@ import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.index.query.AbstractQueryBuilder; -import org.elasticsearch.index.query.CommonTermsQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryRewriteContext; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.query.TermQueryBuilder; -import org.elasticsearch.index.query.TypeQueryV7Builder; import org.elasticsearch.index.query.functionscore.GaussDecayFunctionBuilder; import org.elasticsearch.plugins.SearchPlugin; import org.elasticsearch.search.aggregations.AggregationBuilder; @@ -33,7 +31,6 @@ import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.terms.heuristic.ChiSquare; import org.elasticsearch.search.aggregations.pipeline.AbstractPipelineAggregationBuilder; -import org.elasticsearch.search.aggregations.pipeline.MovAvgPipelineAggregationBuilder; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; import org.elasticsearch.search.aggregations.support.AggregationContext; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; @@ -60,7 +57,6 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xcontent.ConstructingObjectParser; import org.elasticsearch.xcontent.NamedXContentRegistry; -import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentParser; @@ -301,7 +297,6 @@ public void testRegisteredQueries() { List allSupportedQueries = new ArrayList<>(); Collections.addAll(allSupportedQueries, NON_DEPRECATED_QUERIES); Collections.addAll(allSupportedQueries, DEPRECATED_QUERIES); - Collections.addAll(allSupportedQueries, REST_COMPATIBLE_QUERIES); SearchModule module = new SearchModule(Settings.EMPTY, emptyList()); @@ -471,11 +466,6 @@ public CheckedBiConsumer getReque // add here deprecated queries to make sure we log a deprecation warnings when they are used private static final String[] DEPRECATED_QUERIES = new String[] { "field_masking_span", "geo_polygon" }; - private static final String[] REST_COMPATIBLE_QUERIES = new String[] { - TypeQueryV7Builder.NAME_V7.getPreferredName(), - CommonTermsQueryBuilder.NAME_V7.getPreferredName() }; - private static final String[] REST_COMPATIBLE_AGGREGATIONS = new String[] { - MovAvgPipelineAggregationBuilder.NAME_V7.getPreferredName() }; /** * Dummy test {@link AggregationBuilder} used to test registering aggregation builders. @@ -692,58 +682,6 @@ public String getWriteableName() { } } - static class CompatQueryBuilder extends DummyQueryBuilder { - public static final String NAME = "compat_name"; - public static final ParseField NAME_OLD = new ParseField(NAME).forRestApiVersion( - RestApiVersion.equalTo(RestApiVersion.minimumSupported()) - ); - - @Override - public String getWriteableName() { - return NAME; - } - } - - public void testRegisterRestApiCompatibleQuery() { - SearchPlugin registerCompatQuery = new SearchPlugin() { - @Override - public List> getQueries() { - return singletonList( - new QuerySpec<>( - CompatQueryBuilder.NAME_OLD, - (streamInput) -> new CompatQueryBuilder(), - CompatQueryBuilder::fromXContent - ) - ); - } - }; - - final SearchModule searchModule = new SearchModule(Settings.EMPTY, singletonList(registerCompatQuery)); - - // all entries can be used for current and previous versions except for compatible entry - assertThat(searchModule.getNamedXContents().stream().filter(e -> - // filter out compatible entry - e.name.match(CompatQueryBuilder.NAME_OLD.getPreferredName(), LoggingDeprecationHandler.INSTANCE) == false) - .filter(e -> RestApiVersion.minimumSupported().matches(e.restApiCompatibility)) - .filter(e -> RestApiVersion.current().matches(e.restApiCompatibility)) - .collect(toSet()), - // -1 because of the registered in the test - hasSize(searchModule.getNamedXContents().size() - REST_COMPATIBLE_QUERIES.length - REST_COMPATIBLE_AGGREGATIONS.length - 1) - ); - - final List compatEntry = searchModule.getNamedXContents() - .stream() - .filter( - e -> e.categoryClass.equals(QueryBuilder.class) - && RestApiVersion.minimumSupported().matches(e.name.getForRestApiVersion()) // v7 compatbile - && RestApiVersion.current().matches(e.name.getForRestApiVersion()) == false - ) // but not v8 compatible - .collect(toList()); - assertThat(compatEntry, hasSize(REST_COMPATIBLE_QUERIES.length + 1));// +1 because of registered in the test - assertTrue(RestApiVersion.minimumSupported().matches(compatEntry.get(0).restApiCompatibility)); - assertFalse(RestApiVersion.current().matches(compatEntry.get(0).restApiCompatibility)); - } - public void testDefaultMaxNestedDepth() { new SearchModule(Settings.EMPTY, emptyList()); assertEquals( diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java index e6fc32a8ebe1b..8854a209da571 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java @@ -1658,6 +1658,15 @@ public String randomCompatibleMediaType(RestApiVersion version) { } public String compatibleMediaType(XContentType type, RestApiVersion version) { + if (type.canonical().equals(type)) { + throw new IllegalArgumentException( + "Compatible header is only supported for vendor content types." + + " You requested " + + type.name() + + "but likely want VND_" + + type.name() + ); + } return type.toParsedMediaType() .responseContentTypeHeader(Map.of(MediaType.COMPATIBLE_WITH_PARAMETER_NAME, String.valueOf(version.major))); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index 6ed0a1dfe0229..c8542011bcfd8 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -1179,7 +1179,14 @@ protected static void wipeDataStreams() throws IOException { // We hit a version of ES that doesn't serialize DeleteDataStreamAction.Request#wildcardExpressionsOriginallySpecified field // or that doesn't support data streams so it's safe to ignore int statusCode = ee.getResponse().getStatusLine().getStatusCode(); - if (statusCode < 404 || statusCode > 405) { + if (statusCode == 400) { + // the test cluster likely does not include the data streams module so we can ignore this error code + // additionally there is an implementation gotcha that cause response code to be 400 or 405 dependent on if + // "_data_stream/*" matches a registered index pattern such as {a}/{b} but not for the HTTP verb. + // Prior to v9 POST {index}/{type} was registered as a compatible index pattern so the request would partially match + // and return a 405, but without that pattern registered at all the return value is a 400. + return; + } else if (statusCode < 404 || statusCode > 405) { throw ee; } } diff --git a/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/rest/action/RestFreezeIndexAction.java b/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/rest/action/RestFreezeIndexAction.java index 0daf2d8a1ebf5..f3e57d61d1b36 100644 --- a/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/rest/action/RestFreezeIndexAction.java +++ b/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/rest/action/RestFreezeIndexAction.java @@ -13,6 +13,7 @@ import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.common.Strings; import org.elasticsearch.core.RestApiVersion; +import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.protocol.xpack.frozen.FreezeRequest; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; @@ -37,11 +38,17 @@ public final class RestFreezeIndexAction extends BaseRestHandler { private static final String UNFREEZE_DEPRECATED = "Frozen indices are deprecated because they provide no benefit given improvements " + "in heap memory utilization. They will be removed in a future release."; + @UpdateForV9 + // these routes were ".deprecated" in RestApiVersion.V_8 which will require use of REST API compatibility headers to access + // this API in v9. It is unclear if this was intentional for v9, and the code has been updated to ".deprecateAndKeep" which will + // continue to emit deprecations warnings but will not require any special headers to access the API in v9. + // Please review and update the code and tests as needed. The original code remains commented out below for reference. @Override public List routes() { return List.of( Route.builder(POST, "/{index}/_freeze").deprecated(FREEZE_REMOVED, RestApiVersion.V_7).build(), - Route.builder(POST, "/{index}/_unfreeze").deprecated(UNFREEZE_DEPRECATED, RestApiVersion.V_8).build() + // Route.builder(POST, "/{index}/_unfreeze").deprecated(UNFREEZE_DEPRECATED, RestApiVersion.V_8).build() + Route.builder(POST, "/{index}/_unfreeze").deprecateAndKeep(UNFREEZE_DEPRECATED).build() ); } diff --git a/x-pack/plugin/graph/src/test/java/org/elasticsearch/xpack/graph/rest/action/RestGraphActionTests.java b/x-pack/plugin/graph/src/test/java/org/elasticsearch/xpack/graph/rest/action/RestGraphActionTests.java deleted file mode 100644 index 4961efd7253ec..0000000000000 --- a/x-pack/plugin/graph/src/test/java/org/elasticsearch/xpack/graph/rest/action/RestGraphActionTests.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.graph.rest.action; - -import org.elasticsearch.common.bytes.BytesArray; -import org.elasticsearch.core.RestApiVersion; -import org.elasticsearch.core.Tuple; -import org.elasticsearch.protocol.xpack.graph.GraphExploreRequest; -import org.elasticsearch.protocol.xpack.graph.GraphExploreResponse; -import org.elasticsearch.rest.RestRequest; -import org.elasticsearch.test.rest.FakeRestRequest; -import org.elasticsearch.test.rest.RestActionTestCase; -import org.elasticsearch.xcontent.XContentType; -import org.junit.Before; -import org.mockito.Mockito; - -import java.util.Collections; -import java.util.List; -import java.util.Map; - -import static org.hamcrest.Matchers.instanceOf; - -public final class RestGraphActionTests extends RestActionTestCase { - private final List compatibleMediaType = Collections.singletonList(randomCompatibleMediaType(RestApiVersion.V_7)); - - @Before - public void setUpAction() { - controller().registerHandler(new RestGraphAction()); - verifyingClient.setExecuteVerifier((actionType, request) -> { - assertThat(request, instanceOf(GraphExploreRequest.class)); - return Mockito.mock(GraphExploreResponse.class); - }); - } - - public void testTypeInPath() { - for (Tuple methodAndPath : List.of( - Tuple.tuple(RestRequest.Method.GET, "/some_index/some_type/_graph/explore"), - Tuple.tuple(RestRequest.Method.POST, "/some_index/some_type/_graph/explore"), - Tuple.tuple(RestRequest.Method.GET, "/some_index/some_type/_xpack/graph/_explore"), - Tuple.tuple(RestRequest.Method.POST, "/some_index/some_type/_xpack/graph/_explore") - )) { - - RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withHeaders( - Map.of( - "Accept", - compatibleMediaType, - "Content-Type", - Collections.singletonList(compatibleMediaType(XContentType.VND_JSON, RestApiVersion.V_7)) - ) - ).withMethod(methodAndPath.v1()).withPath(methodAndPath.v2()).withContent(new BytesArray("{}"), null).build(); - - dispatchRequest(request); - assertCriticalWarnings(RestGraphAction.TYPES_DEPRECATION_MESSAGE); - } - } -} diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestDeleteTrainedModelAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestDeleteTrainedModelAction.java index e010bd67dff75..ad3e752a3ea3e 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestDeleteTrainedModelAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestDeleteTrainedModelAction.java @@ -8,8 +8,8 @@ import org.elasticsearch.action.support.master.AcknowledgedRequest; import org.elasticsearch.client.internal.node.NodeClient; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.Scope; @@ -28,11 +28,20 @@ @ServerlessScope(Scope.PUBLIC) public class RestDeleteTrainedModelAction extends BaseRestHandler { + @UpdateForV9 + // one or more routes use ".replaces" with RestApiVersion.V_8 which will require use of REST API compatibility headers to access + // that route in v9. It is unclear if this was intentional for v9, and the code has been updated to ".deprecateAndKeep" which will + // continue to emit deprecations warnings but will not require any special headers to access the API in v9. + // Please review and update the code and tests as needed. The original code remains commented out below for reference. @Override public List routes() { return List.of( - Route.builder(DELETE, BASE_PATH + "trained_models/{" + TrainedModelConfig.MODEL_ID + "}") - .replaces(DELETE, BASE_PATH + "inference/{" + TrainedModelConfig.MODEL_ID + "}", RestApiVersion.V_8) + // Route.builder(DELETE, BASE_PATH + "trained_models/{" + TrainedModelConfig.MODEL_ID + "}") + // .replaces(DELETE, BASE_PATH + "inference/{" + TrainedModelConfig.MODEL_ID + "}", RestApiVersion.V_8) + // .build() + new Route(DELETE, BASE_PATH + "trained_models/{" + TrainedModelConfig.MODEL_ID + "}"), + Route.builder(DELETE, BASE_PATH + "inference/{" + TrainedModelConfig.MODEL_ID + "}") + .deprecateAndKeep("Use the trained_models API instead.") .build() ); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestGetTrainedModelsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestGetTrainedModelsAction.java index ae7b26ebad0e4..dfbe375f0d1fc 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestGetTrainedModelsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestGetTrainedModelsAction.java @@ -11,7 +11,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.logging.DeprecationCategory; import org.elasticsearch.common.logging.DeprecationLogger; -import org.elasticsearch.core.RestApiVersion; +import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestChannel; import org.elasticsearch.rest.RestRequest; @@ -49,13 +49,24 @@ public class RestGetTrainedModelsAction extends BaseRestHandler { private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(RestGetTrainedModelsAction.class); private static final String INCLUDE_MODEL_DEFINITION = "include_model_definition"; + @UpdateForV9 + // one or more routes use ".replaces" with RestApiVersion.V_8 which will require use of REST API compatibility headers to access + // that route in v9. It is unclear if this was intentional for v9, and the code has been updated to ".deprecateAndKeep" which will + // continue to emit deprecations warnings but will not require any special headers to access the API in v9. + // Please review and update the code and tests as needed. The original code remains commented out below for reference. @Override public List routes() { return List.of( - Route.builder(GET, BASE_PATH + "trained_models/{" + TrainedModelConfig.MODEL_ID + "}") - .replaces(GET, BASE_PATH + "inference/{" + TrainedModelConfig.MODEL_ID + "}", RestApiVersion.V_8) + // Route.builder(GET, BASE_PATH + "trained_models/{" + TrainedModelConfig.MODEL_ID + "}") + // .replaces(GET, BASE_PATH + "inference/{" + TrainedModelConfig.MODEL_ID + "}", RestApiVersion.V_8) + // .build(), + // Route.builder(GET, BASE_PATH + "trained_models").replaces(GET, BASE_PATH + "inference", RestApiVersion.V_8).build() + new Route(GET, BASE_PATH + "trained_models/{" + TrainedModelConfig.MODEL_ID + "}"), + Route.builder(GET, BASE_PATH + "inference/{" + TrainedModelConfig.MODEL_ID + "}") + .deprecateAndKeep("Use the trained_models API instead.") .build(), - Route.builder(GET, BASE_PATH + "trained_models").replaces(GET, BASE_PATH + "inference", RestApiVersion.V_8).build() + new Route(GET, BASE_PATH + "trained_models"), + Route.builder(GET, BASE_PATH + "inference").deprecateAndKeep("Use the trained_models API instead.").build() ); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestGetTrainedModelsStatsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestGetTrainedModelsStatsAction.java index 9c44728fb75e2..3c192d80f7485 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestGetTrainedModelsStatsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestGetTrainedModelsStatsAction.java @@ -9,7 +9,7 @@ import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.common.Strings; -import org.elasticsearch.core.RestApiVersion; +import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.Scope; @@ -30,15 +30,26 @@ @ServerlessScope(Scope.PUBLIC) public class RestGetTrainedModelsStatsAction extends BaseRestHandler { + @UpdateForV9 + // one or more routes use ".replaces" with RestApiVersion.V_8 which will require use of REST API compatibility headers to access + // that route in v9. It is unclear if this was intentional for v9, and the code has been updated to ".deprecateAndKeep" which will + // continue to emit deprecations warnings but will not require any special headers to access the API in v9. + // Please review and update the code and tests as needed. The original code remains commented out below for reference. @Override public List routes() { return List.of( - Route.builder(GET, BASE_PATH + "trained_models/{" + TrainedModelConfig.MODEL_ID + "}/_stats") - .replaces(GET, BASE_PATH + "inference/{" + TrainedModelConfig.MODEL_ID + "}/_stats", RestApiVersion.V_8) + // Route.builder(GET, BASE_PATH + "trained_models/{" + TrainedModelConfig.MODEL_ID + "}/_stats") + // .replaces(GET, BASE_PATH + "inference/{" + TrainedModelConfig.MODEL_ID + "}/_stats", RestApiVersion.V_8) + // .build(), + // Route.builder(GET, BASE_PATH + "trained_models/_stats") + // .replaces(GET, BASE_PATH + "inference/_stats", RestApiVersion.V_8) + // .build() + new Route(GET, BASE_PATH + "trained_models/{" + TrainedModelConfig.MODEL_ID + "}/_stats"), + Route.builder(GET, BASE_PATH + "inference/{" + TrainedModelConfig.MODEL_ID + "}/_stats") + .deprecateAndKeep("Use the trained_models API instead.") .build(), - Route.builder(GET, BASE_PATH + "trained_models/_stats") - .replaces(GET, BASE_PATH + "inference/_stats", RestApiVersion.V_8) - .build() + new Route(GET, BASE_PATH + "trained_models/_stats"), + Route.builder(GET, BASE_PATH + "inference/_stats").deprecateAndKeep("Use the trained_models API instead.").build() ); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestInferTrainedModelDeploymentAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestInferTrainedModelDeploymentAction.java index 61f319a2157c4..7327f7426e00c 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestInferTrainedModelDeploymentAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestInferTrainedModelDeploymentAction.java @@ -10,8 +10,8 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.common.ValidationException; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestCancellableNodeClient; @@ -37,11 +37,29 @@ public String getName() { return "xpack_ml_infer_trained_models_deployment_action"; } + @UpdateForV9 // these routes were ".deprecated" in RestApiVersion.V_8 which will require use of REST API compatibility headers to access + // this API in v9. It is unclear if this was intentional for v9, and the code has been updated to ".deprecateAndKeep" which will + // continue to emit deprecations warnings but will not require any special headers to access the API in v9. + // Please review and update the code and tests as needed. The original code remains commented out below for reference. @Override public List routes() { return Collections.singletonList( + // Route.builder(POST, PATH) + // .deprecated( + // "[" + // + POST.name() + // + " " + // + PATH + // + "] is deprecated! Use [" + // + POST.name() + // + " " + // + RestInferTrainedModelAction.PATH + // + "] instead.", + // RestApiVersion.V_8 + // ) + // .build() Route.builder(POST, PATH) - .deprecated( + .deprecateAndKeep( "[" + POST.name() + " " @@ -50,8 +68,7 @@ public List routes() { + POST.name() + " " + RestInferTrainedModelAction.PATH - + "] instead.", - RestApiVersion.V_8 + + "] instead." ) .build() ); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestPutTrainedModelAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestPutTrainedModelAction.java index e57d5912752d2..13d46b8878679 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestPutTrainedModelAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestPutTrainedModelAction.java @@ -7,7 +7,7 @@ package org.elasticsearch.xpack.ml.rest.inference; import org.elasticsearch.client.internal.node.NodeClient; -import org.elasticsearch.core.RestApiVersion; +import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.Scope; @@ -27,11 +27,20 @@ @ServerlessScope(Scope.PUBLIC) public class RestPutTrainedModelAction extends BaseRestHandler { + @UpdateForV9 + // one or more routes use ".replaces" with RestApiVersion.V_8 which will require use of REST API compatibility headers to access + // that route in v9. It is unclear if this was intentional for v9, and the code has been updated to ".deprecateAndKeep" which will + // continue to emit deprecations warnings but will not require any special headers to access the API in v9. + // Please review and update the code and tests as needed. The original code remains commented out below for reference. @Override public List routes() { return List.of( - Route.builder(PUT, BASE_PATH + "trained_models/{" + TrainedModelConfig.MODEL_ID + "}") - .replaces(PUT, BASE_PATH + "inference/{" + TrainedModelConfig.MODEL_ID + "}", RestApiVersion.V_8) + // Route.builder(PUT, BASE_PATH + "trained_models/{" + TrainedModelConfig.MODEL_ID + "}") + // .replaces(PUT, BASE_PATH + "inference/{" + TrainedModelConfig.MODEL_ID + "}", RestApiVersion.V_8) + // .build() + new Route(PUT, BASE_PATH + "trained_models/{" + TrainedModelConfig.MODEL_ID + "}"), + Route.builder(PUT, BASE_PATH + "inference/{" + TrainedModelConfig.MODEL_ID + "}") + .deprecateAndKeep("Use the trained_models API instead.") .build() ); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestPostDataAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestPostDataAction.java index 3d7b2d392836a..41462b016a60a 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestPostDataAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestPostDataAction.java @@ -8,6 +8,7 @@ import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.core.RestApiVersion; +import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.RestStatus; @@ -27,12 +28,17 @@ public class RestPostDataAction extends BaseRestHandler { private static final String DEFAULT_RESET_START = ""; private static final String DEFAULT_RESET_END = ""; + @UpdateForV9 // these routes were ".deprecated" in RestApiVersion.V_8 which will require use of REST API compatibility headers to access + // this API in v9. It is unclear if this was intentional for v9, and the code has been updated to ".deprecateAndKeep" which will + // continue to emit deprecations warnings but will not require any special headers to access the API in v9. + // Please review and update the code and tests as needed. The original code remains commented out below for reference. @Override public List routes() { final String msg = "Posting data directly to anomaly detection jobs is deprecated, " + "in a future major version it will be compulsory to use a datafeed"; return List.of( - Route.builder(POST, BASE_PATH + "anomaly_detectors/{" + Job.ID + "}/_data").deprecated(msg, RestApiVersion.V_8).build(), + // Route.builder(POST, BASE_PATH + "anomaly_detectors/{" + Job.ID + "}/_data").deprecated(msg, RestApiVersion.V_8).build(), + Route.builder(POST, BASE_PATH + "anomaly_detectors/{" + Job.ID + "}/_data").deprecateAndKeep(msg).build(), Route.builder(POST, PRE_V7_BASE_PATH + "anomaly_detectors/{" + Job.ID + "}/_data").deprecated(msg, RestApiVersion.V_7).build() ); } diff --git a/x-pack/plugin/security/qa/audit/src/javaRestTest/java/org/elasticsearch/xpack/security/audit/AuditIT.java b/x-pack/plugin/security/qa/audit/src/javaRestTest/java/org/elasticsearch/xpack/security/audit/AuditIT.java index e6af9c634e72f..2c329db5e3b50 100644 --- a/x-pack/plugin/security/qa/audit/src/javaRestTest/java/org/elasticsearch/xpack/security/audit/AuditIT.java +++ b/x-pack/plugin/security/qa/audit/src/javaRestTest/java/org/elasticsearch/xpack/security/audit/AuditIT.java @@ -8,9 +8,7 @@ package org.elasticsearch.xpack.security.audit; import org.elasticsearch.client.Request; -import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.Response; -import org.elasticsearch.client.WarningsHandler; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.settings.SecureString; @@ -81,14 +79,14 @@ protected String getTestRestCluster() { } public void testAuditAuthenticationSuccess() throws Exception { - final Request request = new Request("GET", randomFrom("/_security/_authenticate", "/_xpack/security/_authenticate")); + final Request request = new Request("GET", "/_security/_authenticate"); executeAndVerifyAudit(request, AuditLevel.AUTHENTICATION_SUCCESS, event -> { assertThat(event, hasEntry(LoggingAuditTrail.AUTHENTICATION_TYPE_FIELD_NAME, "REALM")); }); } public void testAuditAuthenticationFailure() throws Exception { - final Request request = new Request("GET", randomFrom("/_security/_authenticate", "/_xpack/security/_authenticate")); + final Request request = new Request("GET", "/_security/_authenticate"); String basicAuth = basicAuthHeaderValue(API_USER, new SecureString(new char[0])); request.setOptions(request.getOptions().toBuilder().addHeader("Authorization", basicAuth).addParameter("ignore", "401")); executeAndVerifyAudit(request, AuditLevel.AUTHENTICATION_FAILED, event -> {}); @@ -96,7 +94,7 @@ public void testAuditAuthenticationFailure() throws Exception { public void testFilteringOfRequestBodies() throws Exception { final String username = randomAlphaOfLength(4) + randomIntBetween(100, 999); - final Request request = new Request(randomFrom("PUT", "POST"), randomFrom("/_security/user/", "/_xpack/security/user/") + username); + final Request request = new Request(randomFrom("PUT", "POST"), "/_security/user/" + username); final String password = randomAlphaOfLength(4) + randomIntBetween(10, 99) + randomAlphaOfLength(4); request.setJsonEntity("{ \"password\":\"" + password + "\", \"roles\":[\"superuser\"] }"); executeAndVerifyAudit(request, AuditLevel.AUTHENTICATION_SUCCESS, event -> { @@ -141,15 +139,6 @@ private void executeAndVerifyAudit(Request request, AuditLevel eventType, Checke } private static Response executeRequest(Request request) throws IOException { - if (request.getEndpoint().startsWith("/_xpack/security/")) { - final RequestOptions options = request.getOptions() - .toBuilder() - .addHeader("Content-Type", "application/json; compatible-with=7") - .addHeader("Accept", "application/json; compatible-with=7") - .setWarningsHandler(WarningsHandler.PERMISSIVE) - .build(); - request.setOptions(options); - } return client().performRequest(request); } diff --git a/x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/rest/RestFindStructureAction.java b/x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/rest/RestFindStructureAction.java index 65325f2268ed2..03b18744eba2a 100644 --- a/x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/rest/RestFindStructureAction.java +++ b/x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/rest/RestFindStructureAction.java @@ -8,7 +8,7 @@ import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.client.internal.node.NodeClient; -import org.elasticsearch.core.RestApiVersion; +import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.Scope; @@ -27,10 +27,17 @@ @ServerlessScope(Scope.INTERNAL) public class RestFindStructureAction extends BaseRestHandler { + @UpdateForV9 + // one or more routes use ".replaces" with RestApiVersion.V_8 which will require use of REST API compatibility headers to access + // that route in v9. It is unclear if this was intentional for v9, and the code has been updated to ".deprecateAndKeep" which will + // continue to emit deprecations warnings but will not require any special headers to access the API in v9. + // Please review and update the code and tests as needed. The original code remains commented out below for reference. @Override public List routes() { return List.of( - Route.builder(POST, BASE_PATH + "find_structure").replaces(POST, "/_ml/find_file_structure", RestApiVersion.V_8).build() + // Route.builder(POST, BASE_PATH + "find_structure").replaces(POST, "/_ml/find_file_structure", RestApiVersion.V_8).build() + new Route(POST, BASE_PATH + "find_structure"), + Route.builder(POST, "/_ml/find_file_structure").deprecateAndKeep("Use the _text_structure API instead.").build() ); } diff --git a/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/MLModelDeploymentFullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/MLModelDeploymentFullClusterRestartIT.java index 484e2ed3ac9c3..abc09a6563ee9 100644 --- a/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/MLModelDeploymentFullClusterRestartIT.java +++ b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/MLModelDeploymentFullClusterRestartIT.java @@ -16,10 +16,12 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.core.Strings; import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.test.rest.RestTestLegacyFeatures; import org.elasticsearch.upgrades.FullClusterRestartUpgradeStatus; +import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.core.ml.inference.assignment.AllocationStatus; import org.junit.Before; @@ -195,14 +197,30 @@ private Response startDeployment(String modelId) throws IOException { } private Response startDeployment(String modelId, String waitForState) throws IOException { + String inferenceThreadParamName = "threads_per_allocation"; + String modelThreadParamName = "number_of_allocations"; + String compatibleHeader = null; + if (isRunningAgainstOldCluster()) { + compatibleHeader = compatibleMediaType(XContentType.VND_JSON, RestApiVersion.V_8); + inferenceThreadParamName = "inference_threads"; + modelThreadParamName = "model_threads"; + } + Request request = new Request( "POST", "/_ml/trained_models/" + modelId + "/deployment/_start?timeout=40s&wait_for=" + waitForState - + "&inference_threads=1&model_threads=1" + + "&" + + inferenceThreadParamName + + "=1&" + + modelThreadParamName + + "=1" ); + if (compatibleHeader != null) { + request.setOptions(request.getOptions().toBuilder().addHeader("Accept", compatibleHeader).build()); + } request.setOptions(request.getOptions().toBuilder().setWarningsHandler(PERMISSIVE).build()); var response = client().performRequest(request); assertOK(response); diff --git a/x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/OldRepositoryAccessIT.java b/x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/OldRepositoryAccessIT.java index c8c72855eaf7a..d61c143098fcb 100644 --- a/x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/OldRepositoryAccessIT.java +++ b/x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/OldRepositoryAccessIT.java @@ -370,14 +370,10 @@ private void assertDocs( Version oldVersion, int numberOfShards ) throws IOException { - RequestOptions v7RequestOptions = RequestOptions.DEFAULT.toBuilder() - .addHeader("Content-Type", "application/vnd.elasticsearch+json;compatible-with=7") - .addHeader("Accept", "application/vnd.elasticsearch+json;compatible-with=7") - .build(); - RequestOptions randomRequestOptions = randomBoolean() ? RequestOptions.DEFAULT : v7RequestOptions; + RequestOptions requestOptions = RequestOptions.DEFAULT; // run a search against the index - SearchResponse searchResponse = search(index, null, randomRequestOptions); + SearchResponse searchResponse = search(index, null, requestOptions); try { logger.info(searchResponse); // check hit count @@ -404,7 +400,7 @@ private void assertDocs( SearchSourceBuilder.searchSource() .query(QueryBuilders.matchQuery("val", num)) .runtimeMappings(Map.of("val", Map.of("type", "long"))), - randomRequestOptions + requestOptions ); try { logger.info(searchResponse); @@ -422,7 +418,7 @@ private void assertDocs( SearchSourceBuilder.searchSource() .query(QueryBuilders.matchAllQuery()) .sort(SortBuilders.fieldSort("val").order(SortOrder.DESC)), - randomRequestOptions + requestOptions ); try { logger.info(searchResponse); @@ -439,7 +435,7 @@ private void assertDocs( searchResponse = search( index, SearchSourceBuilder.searchSource().query(QueryBuilders.matchQuery("test", "test" + num)), - randomRequestOptions + requestOptions ); try { logger.info(searchResponse); @@ -456,7 +452,7 @@ private void assertDocs( searchResponse = search( index, SearchSourceBuilder.searchSource().query(QueryBuilders.termQuery("_type", randomType)), - randomRequestOptions + requestOptions ); try { logger.info(searchResponse); @@ -482,7 +478,7 @@ private void assertDocs( searchResponse = search( index, SearchSourceBuilder.searchSource().query(QueryBuilders.rangeQuery("create_date").from("2020-02-01")), - randomRequestOptions + requestOptions ); try { logger.info(searchResponse); diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MLModelDeploymentsUpgradeIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MLModelDeploymentsUpgradeIT.java index d935672e0a243..553d5e7425de7 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MLModelDeploymentsUpgradeIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MLModelDeploymentsUpgradeIT.java @@ -12,8 +12,10 @@ import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.core.Strings; import org.elasticsearch.core.UpdateForV9; +import org.elasticsearch.xcontent.XContentType; import org.junit.After; import org.junit.Before; import org.junit.BeforeClass; @@ -242,14 +244,30 @@ private Response startDeployment(String modelId) throws IOException { } private Response startDeployment(String modelId, String waitForState) throws IOException { + String inferenceThreadParamName = "threads_per_allocation"; + String modelThreadParamName = "number_of_allocations"; + String compatibleHeader = null; + if (CLUSTER_TYPE.equals(ClusterType.OLD) || CLUSTER_TYPE.equals(ClusterType.MIXED)) { + compatibleHeader = compatibleMediaType(XContentType.VND_JSON, RestApiVersion.V_8); + inferenceThreadParamName = "inference_threads"; + modelThreadParamName = "model_threads"; + } + Request request = new Request( "POST", "/_ml/trained_models/" + modelId + "/deployment/_start?timeout=40s&wait_for=" + waitForState - + "&inference_threads=1&model_threads=1" + + "&" + + inferenceThreadParamName + + "=1&" + + modelThreadParamName + + "=1" ); + if (compatibleHeader != null) { + request.setOptions(request.getOptions().toBuilder().addHeader("Accept", compatibleHeader).build()); + } request.setOptions(request.getOptions().toBuilder().setWarningsHandler(PERMISSIVE).build()); var response = client().performRequest(request); assertOK(response); diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlAssignmentPlannerUpgradeIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlAssignmentPlannerUpgradeIT.java index 6d34ef5887629..88118f6c2727a 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlAssignmentPlannerUpgradeIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlAssignmentPlannerUpgradeIT.java @@ -12,11 +12,13 @@ import org.elasticsearch.client.Response; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.core.Strings; import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.logging.LogManager; import org.elasticsearch.logging.Logger; import org.elasticsearch.test.rest.RestTestLegacyFeatures; +import org.elasticsearch.xcontent.XContentType; import java.io.IOException; import java.util.ArrayList; @@ -278,14 +280,30 @@ private Response startDeployment(String modelId) throws IOException { } private Response startDeployment(String modelId, String waitForState) throws IOException { + String inferenceThreadParamName = "threads_per_allocation"; + String modelThreadParamName = "number_of_allocations"; + String compatibleHeader = null; + if (CLUSTER_TYPE.equals(ClusterType.OLD) || CLUSTER_TYPE.equals(ClusterType.MIXED)) { + compatibleHeader = compatibleMediaType(XContentType.VND_JSON, RestApiVersion.V_8); + inferenceThreadParamName = "inference_threads"; + modelThreadParamName = "model_threads"; + } + Request request = new Request( "POST", "/_ml/trained_models/" + modelId + "/deployment/_start?timeout=40s&wait_for=" + waitForState - + "&inference_threads=1&model_threads=1" + + "&" + + inferenceThreadParamName + + "=1&" + + modelThreadParamName + + "=1" ); + if (compatibleHeader != null) { + request.setOptions(request.getOptions().toBuilder().addHeader("Accept", compatibleHeader).build()); + } request.setOptions(request.getOptions().toBuilder().setWarningsHandler(PERMISSIVE).build()); var response = client().performRequest(request); assertOK(response); From fba8045d99b537217446a77628c4fc7b2cc8336b Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Thu, 26 Sep 2024 15:57:50 -0400 Subject: [PATCH 31/43] [docker] Rename -ubi8 suffix to -ubi (#112884) Co-authored-by: Elastic Machine --- .../main/java/org/elasticsearch/gradle/internal/DockerBase.java | 2 +- .../java/org/elasticsearch/packaging/util/docker/DockerRun.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java index aa9aaa3064024..ed1689cfb0eb9 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java @@ -16,7 +16,7 @@ public enum DockerBase { DEFAULT("ubuntu:20.04", "", "apt-get"), // "latest" here is intentional, since the image name specifies "8" - UBI("docker.elastic.co/ubi8/ubi-minimal:latest", "-ubi8", "microdnf"), + UBI("docker.elastic.co/ubi8/ubi-minimal:latest", "-ubi", "microdnf"), // The Iron Bank base image is UBI (albeit hardened), but we are required to parameterize the Docker build IRON_BANK("${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG}", "-ironbank", "yum"), diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/DockerRun.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/DockerRun.java index 2b3eb7ff7a617..e562e7591564e 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/DockerRun.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/DockerRun.java @@ -163,7 +163,7 @@ String build() { public static String getImageName(Distribution distribution) { String suffix = switch (distribution.packaging) { case DOCKER -> ""; - case DOCKER_UBI -> "-ubi8"; + case DOCKER_UBI -> "-ubi"; case DOCKER_IRON_BANK -> "-ironbank"; case DOCKER_CLOUD -> "-cloud"; case DOCKER_CLOUD_ESS -> "-cloud-ess"; From 55cab5fa98496081966be8232905f132e48ec44b Mon Sep 17 00:00:00 2001 From: Oleksandr Kolomiiets Date: Thu, 26 Sep 2024 13:01:21 -0700 Subject: [PATCH 32/43] Fix ignore_above handling in synthetic source when index level setting is used (#113570) --- docs/changelog/113570.yaml | 7 +++++++ .../test/search/540_ignore_above_synthetic_source.yml | 3 +-- .../org/elasticsearch/index/mapper/KeywordFieldMapper.java | 2 +- x-pack/plugin/build.gradle | 1 + .../test/wildcard/30_ignore_above_synthetic_source.yml | 2 +- .../xpack/wildcard/mapper/WildcardFieldMapper.java | 2 +- 6 files changed, 12 insertions(+), 5 deletions(-) create mode 100644 docs/changelog/113570.yaml diff --git a/docs/changelog/113570.yaml b/docs/changelog/113570.yaml new file mode 100644 index 0000000000000..8cfad9195c5cd --- /dev/null +++ b/docs/changelog/113570.yaml @@ -0,0 +1,7 @@ +pr: 113570 +summary: Fix `ignore_above` handling in synthetic source when index level setting + is used +area: Logs +type: bug +issues: + - 113538 diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/540_ignore_above_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/540_ignore_above_synthetic_source.yml index 11259d3e1bfd1..435cda637cca6 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/540_ignore_above_synthetic_source.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/540_ignore_above_synthetic_source.yml @@ -80,8 +80,7 @@ ignore_above mapping level setting on arrays: match_all: {} - length: { hits.hits: 1 } - #TODO: synthetic source field reconstruction bug (TBD: add link to the issue here) - #- match: { hits.hits.0._source.keyword: ["foo bar", "the quick brown fox"] } + - match: { hits.hits.0._source.keyword: ["foo bar", "the quick brown fox"] } - match: { hits.hits.0._source.flattened.value: [ "jumps over", "the quick brown fox" ] } - match: { hits.hits.0.fields.keyword.0: "foo bar" } - match: { hits.hits.0.fields.flattened.0.value: "jumps over" } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 46b1dbdce4c4b..529ff19bfffd7 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -1093,7 +1093,7 @@ protected BytesRef preserve(BytesRef value) { }); } - if (fieldType().ignoreAbove != ignoreAboveDefault) { + if (fieldType().ignoreAbove != Integer.MAX_VALUE) { layers.add(new CompositeSyntheticFieldLoader.StoredFieldLayer(originalName()) { @Override protected void writeValue(Object value, XContentBuilder b) throws IOException { diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index ebc79ca6ce44a..eb0796672a174 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -82,5 +82,6 @@ tasks.named("precommit").configure { tasks.named("yamlRestCompatTestTransform").configure({ task -> task.skipTest("security/10_forbidden/Test bulk response with invalid credentials", "warning does not exist for compatibility") + task.skipTest("wildcard/30_ignore_above_synthetic_source/wildcard field type ignore_above", "Temporary until backported") }) diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/wildcard/30_ignore_above_synthetic_source.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/wildcard/30_ignore_above_synthetic_source.yml index f5c9f3d92369a..2e3ba773fb0f2 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/wildcard/30_ignore_above_synthetic_source.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/wildcard/30_ignore_above_synthetic_source.yml @@ -49,7 +49,7 @@ wildcard field type ignore_above: - length: { hits.hits: 1 } - match: { hits.hits.0._source.a_wildcard: "foo bar" } - match: { hits.hits.0._source.b_wildcard: "the quick brown" } - - match: { hits.hits.0._source.c_wildcard: ["bar", "foo"] } + - match: { hits.hits.0._source.c_wildcard: ["bar", "foo", "jumps over the lazy dog"] } - match: { hits.hits.0._source.d_wildcard: ["bar", "foo", "the quick"] } - match: { hits.hits.0.fields.a_wildcard.0: "foo bar" } - match: { hits.hits.0.fields.b_wildcard.0: "the quick brown" } diff --git a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java index 1e97e64371586..7784e7ffdda12 100644 --- a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java +++ b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java @@ -994,7 +994,7 @@ public FieldMapper.Builder getMergeBuilder() { protected SyntheticSourceSupport syntheticSourceSupport() { var layers = new ArrayList(); layers.add(new WildcardSyntheticFieldLoader()); - if (ignoreAbove != ignoreAboveDefault) { + if (ignoreAbove != Integer.MAX_VALUE) { layers.add(new CompositeSyntheticFieldLoader.StoredFieldLayer(originalName()) { @Override protected void writeValue(Object value, XContentBuilder b) throws IOException { From de4883fd2972c4f4382948311398f0b3a15908cf Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Fri, 27 Sep 2024 07:14:25 +1000 Subject: [PATCH 33/43] Mute org.elasticsearch.smoketest.MlWithSecurityIT test {yaml=ml/3rd_party_deployment/Test start and stop multiple deployments} #101458 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index e6ea16be8c631..44edbdc9f764a 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -308,6 +308,9 @@ tests: - class: org.elasticsearch.xpack.transform.integration.TransformIT method: testStopWaitForCheckpoint issue: https://github.com/elastic/elasticsearch/issues/106113 +- class: org.elasticsearch.smoketest.MlWithSecurityIT + method: test {yaml=ml/3rd_party_deployment/Test start and stop multiple deployments} + issue: https://github.com/elastic/elasticsearch/issues/101458 # Examples: # From c52427568f75f936ef12f44b5c7be706db2ec24b Mon Sep 17 00:00:00 2001 From: Mark Vieira Date: Thu, 26 Sep 2024 14:20:37 -0700 Subject: [PATCH 34/43] Update contributing doc for new Java 21 requirement (#113642) Update references in our contributing doc mention Java 17 to Java 21. Closes https://github.com/elastic/elasticsearch/issues/113347 --- CONTRIBUTING.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 930de97a3c213..9480b76da20e6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -111,9 +111,9 @@ Contributing to the Elasticsearch codebase **Repository:** [https://github.com/elastic/elasticsearch](https://github.com/elastic/elasticsearch) -JDK 17 is required to build Elasticsearch. You must have a JDK 17 installation +JDK 21 is required to build Elasticsearch. You must have a JDK 21 installation with the environment variable `JAVA_HOME` referencing the path to Java home for -your JDK 17 installation. +your JDK 21 installation. Elasticsearch uses the Gradle wrapper for its build. You can execute Gradle using the wrapper via the `gradlew` script on Unix systems or `gradlew.bat` @@ -152,9 +152,9 @@ The definition of this Elasticsearch cluster can be found [here](build-tools-int ### Importing the project into IntelliJ IDEA The minimum IntelliJ IDEA version required to import the Elasticsearch project is 2020.1. -Elasticsearch builds using Java 17. When importing into IntelliJ you will need +Elasticsearch builds using Java 21. When importing into IntelliJ you will need to define an appropriate SDK. The convention is that **this SDK should be named -"17"** so that the project import will detect it automatically. For more details +"21"** so that the project import will detect it automatically. For more details on defining an SDK in IntelliJ please refer to [their documentation](https://www.jetbrains.com/help/idea/sdk.html#define-sdk). SDK definitions are global, so you can add the JDK from any project, or after project import. Importing with a missing JDK will still work, IntelliJ will From 00ad045c688f2b64612734e775227eddfb004843 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Thu, 26 Sep 2024 17:26:12 -0400 Subject: [PATCH 35/43] [CI] Remove debian-10 from CI since it is EOL (#113644) --- .buildkite/pipelines/periodic-packaging.template.yml | 1 - .buildkite/pipelines/periodic-packaging.yml | 1 - .buildkite/pipelines/periodic-platform-support.yml | 1 - .buildkite/pipelines/pull-request/packaging-tests-unix.yml | 3 --- 4 files changed, 6 deletions(-) diff --git a/.buildkite/pipelines/periodic-packaging.template.yml b/.buildkite/pipelines/periodic-packaging.template.yml index 64c5fa5060e6c..7d2b4df893266 100644 --- a/.buildkite/pipelines/periodic-packaging.template.yml +++ b/.buildkite/pipelines/periodic-packaging.template.yml @@ -8,7 +8,6 @@ steps: setup: image: - centos-7 - - debian-10 - debian-11 - opensuse-leap-15 - oraclelinux-7 diff --git a/.buildkite/pipelines/periodic-packaging.yml b/.buildkite/pipelines/periodic-packaging.yml index 1c33337dc821f..425a550b9d5e1 100644 --- a/.buildkite/pipelines/periodic-packaging.yml +++ b/.buildkite/pipelines/periodic-packaging.yml @@ -9,7 +9,6 @@ steps: setup: image: - centos-7 - - debian-10 - debian-11 - opensuse-leap-15 - oraclelinux-7 diff --git a/.buildkite/pipelines/periodic-platform-support.yml b/.buildkite/pipelines/periodic-platform-support.yml index 867ebe41ed6af..2e6f789f907fa 100644 --- a/.buildkite/pipelines/periodic-platform-support.yml +++ b/.buildkite/pipelines/periodic-platform-support.yml @@ -8,7 +8,6 @@ steps: setup: image: - centos-7 - - debian-10 - debian-11 - opensuse-leap-15 - oraclelinux-7 diff --git a/.buildkite/pipelines/pull-request/packaging-tests-unix.yml b/.buildkite/pipelines/pull-request/packaging-tests-unix.yml index d5c937aa4b5a2..963596220442a 100644 --- a/.buildkite/pipelines/pull-request/packaging-tests-unix.yml +++ b/.buildkite/pipelines/pull-request/packaging-tests-unix.yml @@ -11,7 +11,6 @@ steps: setup: image: - centos-7 - - debian-10 - debian-11 - opensuse-leap-15 - oraclelinux-7 @@ -40,7 +39,6 @@ steps: setup: image: - centos-7 - - debian-10 - debian-11 - opensuse-leap-15 - oraclelinux-7 @@ -69,7 +67,6 @@ steps: setup: image: - centos-7 - - debian-10 - debian-11 - opensuse-leap-15 - oraclelinux-7 From cf7a89e5e9ec0c4865d4819aa708c6879b06557d Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Fri, 27 Sep 2024 08:21:57 +1000 Subject: [PATCH 36/43] Mute org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT test {p0=search/540_ignore_above_synthetic_source/ignore_above mapping level setting on arrays} #113648 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 44edbdc9f764a..7f95a88219aa6 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -311,6 +311,9 @@ tests: - class: org.elasticsearch.smoketest.MlWithSecurityIT method: test {yaml=ml/3rd_party_deployment/Test start and stop multiple deployments} issue: https://github.com/elastic/elasticsearch/issues/101458 +- class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT + method: test {p0=search/540_ignore_above_synthetic_source/ignore_above mapping level setting on arrays} + issue: https://github.com/elastic/elasticsearch/issues/113648 # Examples: # From beb37e5b08e208c9314180b378868f008b0e4d73 Mon Sep 17 00:00:00 2001 From: Stanislav Malyshev Date: Thu, 26 Sep 2024 17:18:02 -0600 Subject: [PATCH 37/43] Make all remote index name parsing go through RemoteClusterAware (#113501) * Make all remote index name parsing go through RemoteClusterAware --- .../action/PainlessExecuteAction.java | 26 +++------ .../reindex/ReindexValidator.java | 17 +----- .../resolve/ResolveClusterActionRequest.java | 11 +--- .../action/search/ShardSearchFailure.java | 9 +-- .../action/search/TransportSearchHelper.java | 12 +--- .../metadata/IndexNameExpressionResolver.java | 5 +- .../index/query/SearchIndexNameMatcher.java | 10 ++-- .../search/profile/SearchProfileResults.java | 12 ++-- .../transport/RemoteClusterAware.java | 57 ++++++++++++++++--- .../transport/RemoteClusterAwareTests.java | 2 +- .../search/SearchResponseUtils.java | 8 +-- .../license/RemoteClusterLicenseChecker.java | 4 +- .../xpack/esql/core/util/StringUtils.java | 10 ++-- .../xpack/esql/plan/TableIdentifier.java | 6 +- .../xpack/ql/plan/TableIdentifier.java | 6 +- .../xpack/ql/util/StringUtils.java | 10 ++-- .../authz/IndicesAndAliasesResolver.java | 8 +-- ...earchRequestCacheDisablingInterceptor.java | 4 +- 18 files changed, 105 insertions(+), 112 deletions(-) diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java index 1950c72c80ec4..4f34cbd3cc475 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java @@ -108,7 +108,6 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -238,29 +237,21 @@ static Tuple parseClusterAliasAndIndex(String indexExpression) { return new Tuple<>(null, null); } String trimmed = indexExpression.trim(); - String sep = String.valueOf(RemoteClusterAware.REMOTE_CLUSTER_INDEX_SEPARATOR); - if (trimmed.startsWith(sep) || trimmed.endsWith(sep)) { - throw new IllegalArgumentException( - "Unable to parse one single valid index name from the provided index: [" + indexExpression + "]" - ); - } - + String[] parts = RemoteClusterAware.splitIndexName(trimmed); // The parser here needs to ensure that the indexExpression is not of the form "remote1:blogs,remote2:blogs" // because (1) only a single index is allowed for Painless Execute and // (2) if this method returns Tuple("remote1", "blogs,remote2:blogs") that will not fail with "index not found". // Instead, it will fail with the inaccurate and confusing error message: // "Cross-cluster calls are not supported in this context but remote indices were requested: [blogs,remote1:blogs]" // which comes later out of the IndexNameExpressionResolver pathway this code uses. - String[] parts = indexExpression.split(sep, 2); - if (parts.length == 1) { - return new Tuple<>(null, parts[0]); - } else if (parts.length == 2 && parts[1].contains(sep) == false) { - return new Tuple<>(parts[0], parts[1]); - } else { + if ((parts[0] != null && parts[1].isEmpty()) + || parts[1].contains(String.valueOf(RemoteClusterAware.REMOTE_CLUSTER_INDEX_SEPARATOR))) { throw new IllegalArgumentException( "Unable to parse one single valid index name from the provided index: [" + indexExpression + "]" ); } + + return new Tuple<>(parts[0], parts[1]); } public String getClusterAlias() { @@ -556,8 +547,8 @@ protected void doExecute(Task task, Request request, ActionListener li // Visible for testing static void removeClusterAliasFromIndexExpression(Request request) { if (request.index() != null) { - String[] split = request.index().split(String.valueOf(RemoteClusterAware.REMOTE_CLUSTER_INDEX_SEPARATOR)); - if (split.length > 1) { + String[] split = RemoteClusterAware.splitIndexName(request.index()); + if (split[0] != null) { /* * if the cluster alias is null and the index field has a clusterAlias (clusterAlias:index notation) * that means this is executing on a remote cluster (it was forwarded by the querying cluster). @@ -565,9 +556,6 @@ static void removeClusterAliasFromIndexExpression(Request request) { * We need to strip off the clusterAlias from the index before executing the script locally, * so it will resolve to a local index */ - assert split.length == 2 - : "If the index contains the REMOTE_CLUSTER_INDEX_SEPARATOR it should have only two parts but it has " - + Arrays.toString(split); request.index(split[1]); } } diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexValidator.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexValidator.java index 4d18f00ab572d..4b960e97ce0e0 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexValidator.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexValidator.java @@ -156,21 +156,10 @@ static void validateAgainstAliases( } private static SearchRequest skipRemoteIndexNames(SearchRequest source) { - return new SearchRequest(source).indices( - Arrays.stream(source.indices()).filter(name -> isRemoteExpression(name) == false).toArray(String[]::new) - ); - } - - private static boolean isRemoteExpression(String expression) { // An index expression that references a remote cluster uses ":" to separate the cluster-alias from the index portion of the // expression, e.g., cluster0:index-name - // in the same time date-math `expression` can also contain ':' symbol inside its name - // to distinguish between those two, given `expression` is pre-evaluated using date-math resolver - // after evaluation date-math `expression` should not contain ':' symbol - // otherwise if `expression` is legit remote name, ':' symbol remains - // NOTE: index expressions can be prefixed with "-", which will not be parsed by resolveDateMathExpression, - // but in this particular case it doesn't seem to be relevant. - return IndexNameExpressionResolver.resolveDateMathExpression(expression) - .contains(String.valueOf(RemoteClusterAware.REMOTE_CLUSTER_INDEX_SEPARATOR)); + return new SearchRequest(source).indices( + Arrays.stream(source.indices()).filter(name -> RemoteClusterAware.isRemoteIndexName(name) == false).toArray(String[]::new) + ); } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveClusterActionRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveClusterActionRequest.java index dbcece1eb4364..9c5b6097b11bd 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveClusterActionRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveClusterActionRequest.java @@ -15,13 +15,12 @@ import org.elasticsearch.action.IndicesRequest; import org.elasticsearch.action.ValidateActions; import org.elasticsearch.action.support.IndicesOptions; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; -import org.elasticsearch.transport.RemoteClusterService; +import org.elasticsearch.transport.RemoteClusterAware; import java.io.IOException; import java.util.Arrays; @@ -166,13 +165,7 @@ public String getDescription() { boolean localIndicesPresent(String[] indices) { for (String index : indices) { - // ensure that `index` is a remote name and not a date math expression which includes ':' symbol - // since date math expression after evaluation should not contain ':' symbol - // NOTE: index expressions can be prefixed with "-" for index exclusion, which will not be parsed by resolveDateMathExpression - String indexExpression = IndexNameExpressionResolver.resolveDateMathExpression( - index.charAt(0) == '-' ? index.substring(1) : index - ); - if (indexExpression.indexOf(RemoteClusterService.REMOTE_CLUSTER_INDEX_SEPARATOR) < 0) { + if (RemoteClusterAware.isRemoteIndexName(index) == false) { return true; } } diff --git a/server/src/main/java/org/elasticsearch/action/search/ShardSearchFailure.java b/server/src/main/java/org/elasticsearch/action/search/ShardSearchFailure.java index 42957e7c932d1..5b2b2dccd6d10 100644 --- a/server/src/main/java/org/elasticsearch/action/search/ShardSearchFailure.java +++ b/server/src/main/java/org/elasticsearch/action/search/ShardSearchFailure.java @@ -141,12 +141,9 @@ public static ShardSearchFailure fromXContent(XContentParser parser) throws IOEx if (SHARD_FIELD.equals(currentFieldName)) { shardId = parser.intValue(); } else if (INDEX_FIELD.equals(currentFieldName)) { - indexName = parser.text(); - int indexOf = indexName.indexOf(RemoteClusterAware.REMOTE_CLUSTER_INDEX_SEPARATOR); - if (indexOf > 0) { - clusterAlias = indexName.substring(0, indexOf); - indexName = indexName.substring(indexOf + 1); - } + String[] split = RemoteClusterAware.splitIndexName(parser.text()); + clusterAlias = split[0]; + indexName = split[1]; } else if (NODE_FIELD.equals(currentFieldName)) { nodeId = parser.text(); } else { diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java index 4ed8feb098ad2..4e3544f0170cb 100644 --- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java +++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java @@ -110,15 +110,9 @@ private static SearchContextIdForNode readSearchContextIdForNodeExcludingContext private static SearchContextIdForNode innerReadSearchContextIdForNode(String contextUUID, StreamInput in) throws IOException { long id = in.readLong(); - String target = in.readString(); - String clusterAlias; - final int index = target.indexOf(RemoteClusterAware.REMOTE_CLUSTER_INDEX_SEPARATOR); - if (index == -1) { - clusterAlias = null; - } else { - clusterAlias = target.substring(0, index); - target = target.substring(index + 1); - } + String[] split = RemoteClusterAware.splitIndexName(in.readString()); + String clusterAlias = split[0]; + String target = split[1]; return new SearchContextIdForNode(clusterAlias, target, new ShardSearchContextId(contextUUID, id)); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java index 6e865db0ebb39..2229166a2d779 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java @@ -37,6 +37,7 @@ import org.elasticsearch.indices.InvalidIndexNameException; import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.indices.SystemIndices.SystemIndexAccessLevel; +import org.elasticsearch.transport.RemoteClusterAware; import java.time.Instant; import java.time.ZoneId; @@ -1753,7 +1754,7 @@ private static void ensureRemoteIndicesRequireIgnoreUnavailable(IndicesOptions o return; } for (String index : indexExpressions) { - if (index.contains(":")) { + if (RemoteClusterAware.isRemoteIndexName(index)) { failOnRemoteIndicesNotIgnoringUnavailable(indexExpressions); } } @@ -1762,7 +1763,7 @@ private static void ensureRemoteIndicesRequireIgnoreUnavailable(IndicesOptions o private static void failOnRemoteIndicesNotIgnoringUnavailable(List indexExpressions) { List crossClusterIndices = new ArrayList<>(); for (String index : indexExpressions) { - if (index.contains(":")) { + if (RemoteClusterAware.isRemoteIndexName(index)) { crossClusterIndices.add(index); } } diff --git a/server/src/main/java/org/elasticsearch/index/query/SearchIndexNameMatcher.java b/server/src/main/java/org/elasticsearch/index/query/SearchIndexNameMatcher.java index 9e34093776fb2..6799895d8e278 100644 --- a/server/src/main/java/org/elasticsearch/index/query/SearchIndexNameMatcher.java +++ b/server/src/main/java/org/elasticsearch/index/query/SearchIndexNameMatcher.java @@ -53,14 +53,12 @@ public SearchIndexNameMatcher( * the separator ':', and must match on both the cluster alias and index name. */ public boolean test(String pattern) { - int separatorIndex = pattern.indexOf(RemoteClusterAware.REMOTE_CLUSTER_INDEX_SEPARATOR); - if (separatorIndex < 0) { + String[] splitIndex = RemoteClusterAware.splitIndexName(pattern); + + if (splitIndex[0] == null) { return clusterAlias == null && matchesIndex(pattern); } else { - String clusterPattern = pattern.substring(0, separatorIndex); - String indexPattern = pattern.substring(separatorIndex + 1); - - return Regex.simpleMatch(clusterPattern, clusterAlias) && matchesIndex(indexPattern); + return Regex.simpleMatch(splitIndex[0], clusterAlias) && matchesIndex(splitIndex[1]); } } diff --git a/server/src/main/java/org/elasticsearch/search/profile/SearchProfileResults.java b/server/src/main/java/org/elasticsearch/search/profile/SearchProfileResults.java index 5c4c7d2ea5574..8227cb5674809 100644 --- a/server/src/main/java/org/elasticsearch/search/profile/SearchProfileResults.java +++ b/server/src/main/java/org/elasticsearch/search/profile/SearchProfileResults.java @@ -16,6 +16,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.core.Nullable; +import org.elasticsearch.transport.RemoteClusterAware; import org.elasticsearch.xcontent.ToXContentFragment; import org.elasticsearch.xcontent.XContentBuilder; @@ -143,15 +144,10 @@ static ShardProfileId parseCompositeProfileShardId(String compositeId) { Matcher m = SHARD_ID_DECOMPOSITION.matcher(compositeId); if (m.find()) { String nodeId = m.group(1); - String indexName = m.group(2); + String[] tokens = RemoteClusterAware.splitIndexName(m.group(2)); + String cluster = tokens[0]; + String indexName = tokens[1]; int shardId = Integer.parseInt(m.group(3)); - String cluster = null; - if (indexName.contains(":")) { - // index names and cluster names cannot contain a ':', so this split should be accurate - String[] tokens = indexName.split(":", 2); - cluster = tokens[0]; - indexName = tokens[1]; - } return new ShardProfileId(nodeId, indexName, shardId, cluster); } else { assert false : "Unable to match input against expected pattern of [nodeId][indexName][shardId]. Input: " + compositeId; diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteClusterAware.java b/server/src/main/java/org/elasticsearch/transport/RemoteClusterAware.java index 76b93a2f802ec..0d6b2cf45138b 100644 --- a/server/src/main/java/org/elasticsearch/transport/RemoteClusterAware.java +++ b/server/src/main/java/org/elasticsearch/transport/RemoteClusterAware.java @@ -53,6 +53,45 @@ protected static Set getEnabledRemoteClusters(final Settings settings) { return RemoteConnectionStrategy.getRemoteClusters(settings); } + /** + * Check whether the index expression represents remote index or not. + * The index name is assumed to be individual index (no commas) but can contain `-`, wildcards, + * datemath, remote cluster name and any other syntax permissible in index expression component. + */ + public static boolean isRemoteIndexName(String indexExpression) { + if (indexExpression.isEmpty() || indexExpression.charAt(0) == '<' || indexExpression.startsWith("-<")) { + // This is date math, but even if it is not, the remote can't start with '<'. + // Thus, whatever it is, this is definitely not a remote index. + return false; + } + // Note remote index name also can not start with ':' + return indexExpression.indexOf(RemoteClusterService.REMOTE_CLUSTER_INDEX_SEPARATOR) > 0; + } + + /** + * Split the index name into remote cluster alias and index name. + * The index expression is assumed to be individual index (no commas) but can contain `-`, wildcards, + * datemath, remote cluster name and any other syntax permissible in index expression component. + * There's no guarantee the components actually represent existing remote cluster or index, only + * rudimentary checks are done on the syntax. + */ + public static String[] splitIndexName(String indexExpression) { + if (indexExpression.isEmpty() || indexExpression.charAt(0) == '<' || indexExpression.startsWith("-<")) { + // This is date math, but even if it is not, the remote can't start with '<'. + // Thus, whatever it is, this is definitely not a remote index. + return new String[] { null, indexExpression }; + } + int i = indexExpression.indexOf(RemoteClusterService.REMOTE_CLUSTER_INDEX_SEPARATOR); + if (i == 0) { + throw new IllegalArgumentException("index name [" + indexExpression + "] is invalid because the remote part is empty"); + } + if (i < 0) { + return new String[] { null, indexExpression }; + } else { + return new String[] { indexExpression.substring(0, i), indexExpression.substring(i + 1) }; + } + } + /** * Groups indices per cluster by splitting remote cluster-alias, index-name pairs on {@link #REMOTE_CLUSTER_INDEX_SEPARATOR}. All * indices per cluster are collected as a list in the returned map keyed by the cluster alias. Local indices are grouped under @@ -77,18 +116,20 @@ protected Map> groupClusterIndices(Set remoteCluste for (String index : requestIndices) { // ensure that `index` is a remote name and not a datemath expression which includes ':' symbol // Remote names can not start with '<' so we are assuming that if the first character is '<' then it is a datemath expression. - boolean isDateMathExpression = (index.charAt(0) == '<' || index.startsWith("-<")); - int i = index.indexOf(RemoteClusterService.REMOTE_CLUSTER_INDEX_SEPARATOR); - if (isDateMathExpression == false && i >= 0) { + String[] split = splitIndexName(index); + if (split[0] != null) { if (isRemoteClusterClientEnabled == false) { assert remoteClusterNames.isEmpty() : remoteClusterNames; throw new IllegalArgumentException("node [" + nodeName + "] does not have the remote cluster client role enabled"); } - int startIdx = index.charAt(0) == '-' ? 1 : 0; - String remoteClusterName = index.substring(startIdx, i); - List clusters = ClusterNameExpressionResolver.resolveClusterNames(remoteClusterNames, remoteClusterName); - String indexName = index.substring(i + 1); - if (startIdx == 1) { + String remoteClusterName = split[0]; + String indexName = split[1]; + boolean isNegative = remoteClusterName.startsWith("-"); + List clusters = ClusterNameExpressionResolver.resolveClusterNames( + remoteClusterNames, + isNegative ? remoteClusterName.substring(1) : remoteClusterName + ); + if (isNegative) { if (indexName.equals("*") == false) { throw new IllegalArgumentException( Strings.format( diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteClusterAwareTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteClusterAwareTests.java index 169f6d8060020..2394e0b07cc57 100644 --- a/server/src/test/java/org/elasticsearch/transport/RemoteClusterAwareTests.java +++ b/server/src/test/java/org/elasticsearch/transport/RemoteClusterAwareTests.java @@ -130,7 +130,7 @@ public void testGroupClusterIndicesFail() { RemoteClusterAwareTest remoteClusterAware = new RemoteClusterAwareTest(); Set remoteClusterNames = Set.of("cluster1", "cluster2", "some-cluster3"); - mustThrowException(new String[] { ":foo" }, NoSuchRemoteClusterException.class, "no such remote cluster"); + mustThrowException(new String[] { ":foo" }, IllegalArgumentException.class, "is invalid because the remote part is empty"); mustThrowException(new String[] { "notacluster:foo" }, NoSuchRemoteClusterException.class, "no such remote cluster"); // Cluster wildcard exclusion requires :* mustThrowException( diff --git a/test/framework/src/main/java/org/elasticsearch/search/SearchResponseUtils.java b/test/framework/src/main/java/org/elasticsearch/search/SearchResponseUtils.java index 86bd0899e862a..60bb858af9452 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/SearchResponseUtils.java +++ b/test/framework/src/main/java/org/elasticsearch/search/SearchResponseUtils.java @@ -851,11 +851,9 @@ public static SearchHit searchHitFromMap(Map values) { String index = get(SearchHit.Fields._INDEX, values, null); String clusterAlias = null; if (index != null) { - int indexOf = index.indexOf(RemoteClusterAware.REMOTE_CLUSTER_INDEX_SEPARATOR); - if (indexOf > 0) { - clusterAlias = index.substring(0, indexOf); - index = index.substring(indexOf + 1); - } + String[] split = RemoteClusterAware.splitIndexName(index); + clusterAlias = split[0]; + index = split[1]; } ShardId shardId = get(SearchHit.Fields._SHARD, values, null); String nodeId = get(SearchHit.Fields._NODE, values, null); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RemoteClusterLicenseChecker.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RemoteClusterLicenseChecker.java index 8db05703a3f0d..01280b1d95f80 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RemoteClusterLicenseChecker.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RemoteClusterLicenseChecker.java @@ -239,7 +239,7 @@ private void remoteClusterLicense(final String clusterAlias, final ActionListene * @return true if the collection of indices contains a remote index, otherwise false */ public static boolean isRemoteIndex(final String index) { - return index.indexOf(RemoteClusterAware.REMOTE_CLUSTER_INDEX_SEPARATOR) != -1; + return RemoteClusterAware.isRemoteIndexName(index); } /** @@ -275,7 +275,7 @@ public static List remoteIndices(final Collection indices) { public static List remoteClusterAliases(final Set remoteClusters, final List indices) { return indices.stream() .filter(RemoteClusterLicenseChecker::isRemoteIndex) - .map(index -> index.substring(0, index.indexOf(RemoteClusterAware.REMOTE_CLUSTER_INDEX_SEPARATOR))) + .map(index -> RemoteClusterAware.splitIndexName(index)[0]) .distinct() .flatMap(clusterExpression -> ClusterNameExpressionResolver.resolveClusterNames(remoteClusters, clusterExpression).stream()) .distinct() diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/StringUtils.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/StringUtils.java index cd0ade2054ce6..1bfd94730c4fc 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/StringUtils.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/StringUtils.java @@ -14,6 +14,7 @@ import org.elasticsearch.common.network.InetAddresses; import org.elasticsearch.core.Tuple; import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.transport.RemoteClusterAware; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; @@ -27,7 +28,6 @@ import java.util.StringJoiner; import static java.util.stream.Collectors.toList; -import static org.elasticsearch.transport.RemoteClusterAware.REMOTE_CLUSTER_INDEX_SEPARATOR; import static org.elasticsearch.transport.RemoteClusterAware.buildRemoteIndexName; import static org.elasticsearch.xpack.esql.core.util.NumericUtils.isUnsignedLong; @@ -378,10 +378,8 @@ public static String ordinal(int i) { } public static Tuple splitQualifiedIndex(String indexName) { - int separatorOffset = indexName.indexOf(REMOTE_CLUSTER_INDEX_SEPARATOR); - return separatorOffset > 0 - ? Tuple.tuple(indexName.substring(0, separatorOffset), indexName.substring(separatorOffset + 1)) - : Tuple.tuple(null, indexName); + String[] split = RemoteClusterAware.splitIndexName(indexName); + return Tuple.tuple(split[0], split[1]); } public static String qualifyAndJoinIndices(String cluster, String[] indices) { @@ -393,7 +391,7 @@ public static String qualifyAndJoinIndices(String cluster, String[] indices) { } public static boolean isQualified(String indexWildcard) { - return indexWildcard.indexOf(REMOTE_CLUSTER_INDEX_SEPARATOR) > 0; + return RemoteClusterAware.isRemoteIndexName(indexWildcard); } public static boolean isInteger(String value) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/TableIdentifier.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/TableIdentifier.java index ceefe4e254557..532d93eec48af 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/TableIdentifier.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/TableIdentifier.java @@ -10,6 +10,8 @@ import java.util.Objects; +import static org.elasticsearch.transport.RemoteClusterAware.REMOTE_CLUSTER_INDEX_SEPARATOR; + public class TableIdentifier { private final Source source; @@ -55,7 +57,7 @@ public Source source() { } public String qualifiedIndex() { - return cluster != null ? cluster + ":" + index : index; + return cluster != null ? cluster + REMOTE_CLUSTER_INDEX_SEPARATOR + index : index; } @Override @@ -63,7 +65,7 @@ public String toString() { StringBuilder builder = new StringBuilder(); if (cluster != null) { builder.append(cluster); - builder.append(":"); + builder.append(REMOTE_CLUSTER_INDEX_SEPARATOR); } builder.append(index); return builder.toString(); diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/plan/TableIdentifier.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/plan/TableIdentifier.java index 188bd4cce9c13..ad3322ce4501d 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/plan/TableIdentifier.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/plan/TableIdentifier.java @@ -10,6 +10,8 @@ import java.util.Objects; +import static org.elasticsearch.transport.RemoteClusterAware.REMOTE_CLUSTER_INDEX_SEPARATOR; + public class TableIdentifier { private final Source source; @@ -55,7 +57,7 @@ public Source source() { } public String qualifiedIndex() { - return cluster != null ? cluster + ":" + index : index; + return cluster != null ? cluster + REMOTE_CLUSTER_INDEX_SEPARATOR + index : index; } @Override @@ -63,7 +65,7 @@ public String toString() { StringBuilder builder = new StringBuilder(); if (cluster != null) { builder.append(cluster); - builder.append(":"); + builder.append(REMOTE_CLUSTER_INDEX_SEPARATOR); } builder.append(index); return builder.toString(); diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/StringUtils.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/StringUtils.java index dad3c8574dc4a..f03e3a111d189 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/StringUtils.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/StringUtils.java @@ -14,6 +14,7 @@ import org.elasticsearch.common.network.InetAddresses; import org.elasticsearch.core.Tuple; import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.transport.RemoteClusterAware; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; @@ -27,7 +28,6 @@ import java.util.StringJoiner; import static java.util.stream.Collectors.toList; -import static org.elasticsearch.transport.RemoteClusterAware.REMOTE_CLUSTER_INDEX_SEPARATOR; import static org.elasticsearch.transport.RemoteClusterAware.buildRemoteIndexName; import static org.elasticsearch.xpack.ql.util.NumericUtils.isUnsignedLong; @@ -375,10 +375,8 @@ public static String ordinal(int i) { } public static Tuple splitQualifiedIndex(String indexName) { - int separatorOffset = indexName.indexOf(REMOTE_CLUSTER_INDEX_SEPARATOR); - return separatorOffset > 0 - ? Tuple.tuple(indexName.substring(0, separatorOffset), indexName.substring(separatorOffset + 1)) - : Tuple.tuple(null, indexName); + String[] split = RemoteClusterAware.splitIndexName(indexName); + return Tuple.tuple(split[0], split[1]); } public static String qualifyAndJoinIndices(String cluster, String[] indices) { @@ -390,6 +388,6 @@ public static String qualifyAndJoinIndices(String cluster, String[] indices) { } public static boolean isQualified(String indexWildcard) { - return indexWildcard.indexOf(REMOTE_CLUSTER_INDEX_SEPARATOR) > 0; + return RemoteClusterAware.isRemoteIndexName(indexWildcard); } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java index 42a1d89a9aa00..d5cbbe8b349a7 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java @@ -169,11 +169,9 @@ ResolvedIndices resolveIndicesAndAliasesWithoutWildcards(String action, IndicesR // and no remote clusters are configured that match it if (split.getLocal().isEmpty() && split.getRemote().isEmpty()) { for (String indexExpression : indices) { - String[] clusterAndIndex = indexExpression.split(":", 2); - if (clusterAndIndex.length == 2) { - if (clusterAndIndex[0].contains("*")) { - throw new NoSuchRemoteClusterException(clusterAndIndex[0]); - } + String[] clusterAndIndex = RemoteClusterAware.splitIndexName(indexExpression); + if (clusterAndIndex[0] != null && clusterAndIndex[0].contains("*")) { + throw new NoSuchRemoteClusterException(clusterAndIndex[0]); } } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/interceptor/SearchRequestCacheDisablingInterceptor.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/interceptor/SearchRequestCacheDisablingInterceptor.java index d10057ec7e740..d8ec078507bfe 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/interceptor/SearchRequestCacheDisablingInterceptor.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/interceptor/SearchRequestCacheDisablingInterceptor.java @@ -11,6 +11,7 @@ import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.RemoteClusterAware; import org.elasticsearch.transport.TransportActionProxy; import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine; import org.elasticsearch.xpack.core.security.authz.AuthorizationServiceField; @@ -18,7 +19,6 @@ import java.util.Arrays; -import static org.elasticsearch.transport.RemoteClusterAware.REMOTE_CLUSTER_INDEX_SEPARATOR; import static org.elasticsearch.xpack.core.security.SecurityField.DOCUMENT_LEVEL_SECURITY_FEATURE; import static org.elasticsearch.xpack.core.security.SecurityField.FIELD_LEVEL_SECURITY_FEATURE; @@ -55,6 +55,6 @@ && hasRemoteIndices(searchRequest) // package private for test static boolean hasRemoteIndices(SearchRequest request) { - return Arrays.stream(request.indices()).anyMatch(name -> name.indexOf(REMOTE_CLUSTER_INDEX_SEPARATOR) >= 0); + return Arrays.stream(request.indices()).anyMatch(RemoteClusterAware::isRemoteIndexName); } } From 3bc600e2cb30e9f1fceec558abaa58d823abf901 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Fri, 27 Sep 2024 09:20:19 +1000 Subject: [PATCH 38/43] Mute org.elasticsearch.xpack.ml.integration.MlJobIT testGetJobs_GivenMultipleJobs #113654 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 7f95a88219aa6..1bc17f6764c7d 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -314,6 +314,9 @@ tests: - class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT method: test {p0=search/540_ignore_above_synthetic_source/ignore_above mapping level setting on arrays} issue: https://github.com/elastic/elasticsearch/issues/113648 +- class: org.elasticsearch.xpack.ml.integration.MlJobIT + method: testGetJobs_GivenMultipleJobs + issue: https://github.com/elastic/elasticsearch/issues/113654 # Examples: # From 42f3ddeeafd5a0f3a7d413b9a19b15e303942ef3 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Fri, 27 Sep 2024 09:20:28 +1000 Subject: [PATCH 39/43] Mute org.elasticsearch.xpack.ml.integration.MlJobIT testGetJobs_GivenSingleJob #113655 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 1bc17f6764c7d..5530aa5ed541c 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -317,6 +317,9 @@ tests: - class: org.elasticsearch.xpack.ml.integration.MlJobIT method: testGetJobs_GivenMultipleJobs issue: https://github.com/elastic/elasticsearch/issues/113654 +- class: org.elasticsearch.xpack.ml.integration.MlJobIT + method: testGetJobs_GivenSingleJob + issue: https://github.com/elastic/elasticsearch/issues/113655 # Examples: # From 8259111fe05967606d85c7296d1a69d87f29e102 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Thu, 26 Sep 2024 18:56:11 -0500 Subject: [PATCH 40/43] fixing a typo in 80_ingest_simulate.yml --- .../rest-api-spec/test/ingest/80_ingest_simulate.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml index f3a977cd96f62..9c6a1ca2e96d2 100644 --- a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml +++ b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml @@ -375,7 +375,7 @@ setup: - do: allowed_warnings: - - "index template [test-composable-1] has index patterns [tsdb_templated_*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [test-composable-1] will take precedence during new index creation" + - "index template [test-composable-1] has index patterns [foo*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [test-composable-1] will take precedence during new index creation" indices.put_index_template: name: test-composable-1 body: @@ -527,7 +527,7 @@ setup: - do: allowed_warnings: - - "index template [test-composable-1] has index patterns [tsdb_templated_*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [test-composable-1] will take precedence during new index creation" + - "index template [test-composable-1] has index patterns [foo*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [test-composable-1] will take precedence during new index creation" indices.put_index_template: name: test-composable-1 body: From e77dc6dc1324b00ba2ddf15718686e8bd1bcb3a9 Mon Sep 17 00:00:00 2001 From: Jake Landis Date: Thu, 26 Sep 2024 20:05:44 -0500 Subject: [PATCH 41/43] Bump the min java version for examples (#112843) --- plugins/examples/build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/examples/build.gradle b/plugins/examples/build.gradle index 8be4dfd7d1ab7..b60485edd1cb8 100644 --- a/plugins/examples/build.gradle +++ b/plugins/examples/build.gradle @@ -16,8 +16,8 @@ subprojects { apply plugin: 'java' java { - sourceCompatibility = 17 - targetCompatibility = 17 + sourceCompatibility = 21 + targetCompatibility = 21 } repositories { From 34e2d6bc825b51053977c589845adea5298064b6 Mon Sep 17 00:00:00 2001 From: David Turner Date: Fri, 27 Sep 2024 09:25:41 +0100 Subject: [PATCH 42/43] Fix `testFollowIndexAndCloseNode` (#113618) It's possible that we index enough documents and restart the node while one of the follower shards is still initializing, and since this is like a snapshot recovery it won't retry if it fails when the node shuts down. This commit ensures that at least all the follower primaries have started before restarting the node. Closes #111941 --- .../java/org/elasticsearch/xpack/ccr/FollowerFailOverIT.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/FollowerFailOverIT.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/FollowerFailOverIT.java index 501a664d64698..274d723a37574 100644 --- a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/FollowerFailOverIT.java +++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/FollowerFailOverIT.java @@ -9,6 +9,7 @@ import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.delete.DeleteResponse; +import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.node.DiscoveryNode; @@ -157,6 +158,7 @@ public void testFollowIndexAndCloseNode() throws Exception { followRequest.getParameters().setMaxWriteRequestOperationCount(randomIntBetween(32, 2048)); followRequest.getParameters().setMaxWriteRequestSize(new ByteSizeValue(randomIntBetween(1, 4096), ByteSizeUnit.KB)); followRequest.getParameters().setMaxOutstandingWriteRequests(randomIntBetween(1, 10)); + followRequest.waitForActiveShards(ActiveShardCount.ALL); followerClient().execute(PutFollowAction.INSTANCE, followRequest).get(); disableDelayedAllocation("index2"); logger.info("--> follow request {}", Strings.toString(followRequest)); From 646725b50d7369d7693fc4b1e4a21448a11c311a Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Fri, 27 Sep 2024 18:41:12 +1000 Subject: [PATCH 43/43] Mute org.elasticsearch.xpack.security.authz.interceptor.SearchRequestCacheDisablingInterceptorTests testHasRemoteIndices #113660 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 5530aa5ed541c..05f6d3019dc19 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -320,6 +320,9 @@ tests: - class: org.elasticsearch.xpack.ml.integration.MlJobIT method: testGetJobs_GivenSingleJob issue: https://github.com/elastic/elasticsearch/issues/113655 +- class: org.elasticsearch.xpack.security.authz.interceptor.SearchRequestCacheDisablingInterceptorTests + method: testHasRemoteIndices + issue: https://github.com/elastic/elasticsearch/issues/113660 # Examples: #