diff --git a/build-tools-internal/version.properties b/build-tools-internal/version.properties index edb97a2968bc8..ac75a3a968ed1 100644 --- a/build-tools-internal/version.properties +++ b/build-tools-internal/version.properties @@ -1,5 +1,5 @@ elasticsearch = 9.0.0 -lucene = 9.11.1 +lucene = 9.12.0 bundled_jdk_vendor = openjdk bundled_jdk = 22.0.1+8@c7ec1332f7bb44aeba2eb341ae18aca4 diff --git a/docs/Versions.asciidoc b/docs/Versions.asciidoc index fb99ef498df17..b65b974cd6b69 100644 --- a/docs/Versions.asciidoc +++ b/docs/Versions.asciidoc @@ -1,8 +1,8 @@ include::{docs-root}/shared/versions/stack/{source_branch}.asciidoc[] -:lucene_version: 9.11.1 -:lucene_version_path: 9_11_1 +:lucene_version: 9.12.0 +:lucene_version_path: 9_12_0 :jdk: 11.0.2 :jdk_major: 11 :build_type: tar diff --git a/docs/changelog/111465.yaml b/docs/changelog/111465.yaml new file mode 100644 index 0000000000000..2a8df287427a9 --- /dev/null +++ b/docs/changelog/111465.yaml @@ -0,0 +1,5 @@ +pr: 111465 +summary: Add range and regexp Intervals +area: Search +type: enhancement +issues: [] diff --git a/docs/changelog/112826.yaml b/docs/changelog/112826.yaml new file mode 100644 index 0000000000000..65c05b4d6035a --- /dev/null +++ b/docs/changelog/112826.yaml @@ -0,0 +1,6 @@ +pr: 112826 +summary: "Multi term intervals: increase max_expansions" +area: Search +type: enhancement +issues: + - 110491 diff --git a/docs/changelog/113333.yaml b/docs/changelog/113333.yaml new file mode 100644 index 0000000000000..c6a3584845729 --- /dev/null +++ b/docs/changelog/113333.yaml @@ -0,0 +1,5 @@ +pr: 113333 +summary: Upgrade to Lucene 9.12 +area: Search +type: upgrade +issues: [] diff --git a/docs/reference/modules/threadpool.asciidoc b/docs/reference/modules/threadpool.asciidoc index 2d4110bdcb431..8ae8f59c22982 100644 --- a/docs/reference/modules/threadpool.asciidoc +++ b/docs/reference/modules/threadpool.asciidoc @@ -13,16 +13,10 @@ There are several thread pools, but the important ones include: [[search-threadpool]] `search`:: - For coordination of count/search operations at the shard level whose computation - is offloaded to the search_worker thread pool. Used also by fetch and other search + For count/search operations at the shard level. Used also by fetch and other search related operations Thread pool type is `fixed` with a size of `int((`<>`pass:[ * ]3) / 2) + 1`, and queue_size of `1000`. -`search_worker`:: - For the heavy workload of count/search operations that may be executed concurrently - across segments within the same shard when possible. Thread pool type is `fixed` - with a size of `int((`<>`pass:[ * ]3) / 2) + 1`, and unbounded queue_size . - [[search-throttled]]`search_throttled`:: For count/search/suggest/get operations on `search_throttled indices`. Thread pool type is `fixed` with a size of `1`, and queue_size of `100`. diff --git a/docs/reference/query-dsl/intervals-query.asciidoc b/docs/reference/query-dsl/intervals-query.asciidoc index 1e3380389d861..069021dddb69f 100644 --- a/docs/reference/query-dsl/intervals-query.asciidoc +++ b/docs/reference/query-dsl/intervals-query.asciidoc @@ -73,7 +73,9 @@ Valid rules include: * <> * <> * <> +* <> * <> +* <> * <> * <> -- @@ -122,8 +124,9 @@ unstemmed ones. ==== `prefix` rule parameters The `prefix` rule matches terms that start with a specified set of characters. -This prefix can expand to match at most 128 terms. If the prefix matches more -than 128 terms, {es} returns an error. You can use the +This prefix can expand to match at most `indices.query.bool.max_clause_count` +<> terms. If the prefix matches more terms, +{es} returns an error. You can use the <> option in the field mapping to avoid this limit. @@ -149,7 +152,8 @@ separate `analyzer` is specified. ==== `wildcard` rule parameters The `wildcard` rule matches terms using a wildcard pattern. This pattern can -expand to match at most 128 terms. If the pattern matches more than 128 terms, +expand to match at most `indices.query.bool.max_clause_count` +<> terms. If the pattern matches more terms, {es} returns an error. `pattern`:: @@ -178,12 +182,44 @@ The `pattern` is normalized using the search analyzer from this field, unless `analyzer` is specified separately. -- +[[intervals-regexp]] +==== `regexp` rule parameters + +The `regexp` rule matches terms using a regular expression pattern. +This pattern can expand to match at most `indices.query.bool.max_clause_count` +<> terms. +If the pattern matches more terms,{es} returns an error. + +`pattern`:: +(Required, string) Regexp pattern used to find matching terms. +For a list of operators supported by the +`regexp` pattern, see <>. + +WARNING: Avoid using wildcard patterns, such as `.*` or `.*?+``. This can +increase the iterations needed to find matching terms and slow search +performance. +-- +`analyzer`:: +(Optional, string) <> used to normalize the `pattern`. +Defaults to the top-level ``'s analyzer. +-- +`use_field`:: ++ +-- +(Optional, string) If specified, match intervals from this field rather than the +top-level ``. + +The `pattern` is normalized using the search analyzer from this field, unless +`analyzer` is specified separately. +-- + [[intervals-fuzzy]] ==== `fuzzy` rule parameters The `fuzzy` rule matches terms that are similar to the provided term, within an edit distance defined by <>. If the fuzzy expansion matches more than -128 terms, {es} returns an error. +`indices.query.bool.max_clause_count` +<> terms, {es} returns an error. `term`:: (Required, string) The term to match @@ -214,6 +250,41 @@ The `term` is normalized using the search analyzer from this field, unless `analyzer` is specified separately. -- +[[intervals-range]] +==== `range` rule parameters + +The `range` rule matches terms contained within a provided range. +This range can expand to match at most `indices.query.bool.max_clause_count` +<> terms. +If the range matches more terms,{es} returns an error. + +`gt`:: +(Optional, string) Greater than: match terms greater than the provided term. + +`gte`:: +(Optional, string) Greater than or equal to: match terms greater than or +equal to the provided term. + +`lt`:: +(Optional, string) Less than: match terms less than the provided term. + +`lte`:: +(Optional, string) Less than or equal to: match terms less than or +equal to the provided term. + +NOTE: It is required to provide one of `gt` or `gte` params. +It is required to provide one of `lt` or `lte` params. + + +`analyzer`:: +(Optional, string) <> used to normalize the `pattern`. +Defaults to the top-level ``'s analyzer. + +`use_field`:: +(Optional, string) If specified, match intervals from this field rather than the +top-level ``. + + [[intervals-all_of]] ==== `all_of` rule parameters diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index 472a65f9c6f24..f1c4b15ea5702 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -2814,124 +2814,129 @@ - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + + + + + + diff --git a/libs/simdvec/src/test/java/org/elasticsearch/simdvec/VectorScorerFactoryTests.java b/libs/simdvec/src/test/java/org/elasticsearch/simdvec/VectorScorerFactoryTests.java index e5d963995d748..db57dc936e794 100644 --- a/libs/simdvec/src/test/java/org/elasticsearch/simdvec/VectorScorerFactoryTests.java +++ b/libs/simdvec/src/test/java/org/elasticsearch/simdvec/VectorScorerFactoryTests.java @@ -237,6 +237,8 @@ void testRandomScorerImpl(long maxChunkSize, Function floatArr try (Directory dir = new MMapDirectory(createTempDir("testRandom"), maxChunkSize)) { for (var sim : List.of(COSINE, DOT_PRODUCT, EUCLIDEAN, MAXIMUM_INNER_PRODUCT)) { + // Use the random supplier for COSINE, which returns values in the normalized range + floatArraySupplier = sim == COSINE ? FLOAT_ARRAY_RANDOM_FUNC : floatArraySupplier; final int dims = randomIntBetween(1, 4096); final int size = randomIntBetween(2, 100); final float[][] vectors = new float[size][]; diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java index 6693d24fe78e2..5904169308fab 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java @@ -21,6 +21,7 @@ import org.apache.lucene.queries.intervals.IntervalsSource; import org.apache.lucene.search.ConstantScoreQuery; import org.apache.lucene.search.FuzzyQuery; +import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.PrefixQuery; @@ -270,7 +271,11 @@ public IntervalsSource termIntervals(BytesRef term, SearchExecutionContext conte @Override public IntervalsSource prefixIntervals(BytesRef term, SearchExecutionContext context) { - return toIntervalsSource(Intervals.prefix(term), new PrefixQuery(new Term(name(), term)), context); + return toIntervalsSource( + Intervals.prefix(term, IndexSearcher.getMaxClauseCount()), + new PrefixQuery(new Term(name(), term)), + context + ); } @Override @@ -285,23 +290,47 @@ public IntervalsSource fuzzyIntervals( new Term(name(), term), maxDistance, prefixLength, - 128, + IndexSearcher.getMaxClauseCount(), transpositions, MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE ); - IntervalsSource fuzzyIntervals = Intervals.multiterm(fuzzyQuery.getAutomata(), term); + IntervalsSource fuzzyIntervals = Intervals.multiterm(fuzzyQuery.getAutomata(), IndexSearcher.getMaxClauseCount(), term); return toIntervalsSource(fuzzyIntervals, fuzzyQuery, context); } @Override public IntervalsSource wildcardIntervals(BytesRef pattern, SearchExecutionContext context) { return toIntervalsSource( - Intervals.wildcard(pattern), + Intervals.wildcard(pattern, IndexSearcher.getMaxClauseCount()), new MatchAllDocsQuery(), // wildcard queries can be expensive, what should the approximation be? context ); } + @Override + public IntervalsSource regexpIntervals(BytesRef pattern, SearchExecutionContext context) { + return toIntervalsSource( + Intervals.regexp(pattern, IndexSearcher.getMaxClauseCount()), + new MatchAllDocsQuery(), // regexp queries can be expensive, what should the approximation be? + context + ); + } + + @Override + public IntervalsSource rangeIntervals( + BytesRef lowerTerm, + BytesRef upperTerm, + boolean includeLower, + boolean includeUpper, + SearchExecutionContext context + ) { + return toIntervalsSource( + Intervals.range(lowerTerm, upperTerm, includeLower, includeUpper, IndexSearcher.getMaxClauseCount()), + new MatchAllDocsQuery(), // range queries can be expensive, what should the approximation be? + context + ); + } + @Override public Query phraseQuery(TokenStream stream, int slop, boolean enablePosIncrements, SearchExecutionContext queryShardContext) throws IOException { diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldTypeTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldTypeTests.java index 4c20802a45058..6970dd6739ecf 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldTypeTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldTypeTests.java @@ -14,6 +14,7 @@ import org.apache.lucene.queries.intervals.IntervalsSource; import org.apache.lucene.search.ConstantScoreQuery; import org.apache.lucene.search.FuzzyQuery; +import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.MultiPhraseQuery; import org.apache.lucene.search.PhraseQuery; @@ -152,30 +153,56 @@ public void testPhrasePrefixQuery() throws IOException { assertNotEquals(new MatchAllDocsQuery(), SourceConfirmedTextQuery.approximate(delegate)); } - public void testTermIntervals() throws IOException { + public void testTermIntervals() { MappedFieldType ft = new MatchOnlyTextFieldType("field"); IntervalsSource termIntervals = ft.termIntervals(new BytesRef("foo"), MOCK_CONTEXT); assertThat(termIntervals, Matchers.instanceOf(SourceIntervalsSource.class)); assertEquals(Intervals.term(new BytesRef("foo")), ((SourceIntervalsSource) termIntervals).getIntervalsSource()); } - public void testPrefixIntervals() throws IOException { + public void testPrefixIntervals() { MappedFieldType ft = new MatchOnlyTextFieldType("field"); IntervalsSource prefixIntervals = ft.prefixIntervals(new BytesRef("foo"), MOCK_CONTEXT); assertThat(prefixIntervals, Matchers.instanceOf(SourceIntervalsSource.class)); - assertEquals(Intervals.prefix(new BytesRef("foo")), ((SourceIntervalsSource) prefixIntervals).getIntervalsSource()); + assertEquals( + Intervals.prefix(new BytesRef("foo"), IndexSearcher.getMaxClauseCount()), + ((SourceIntervalsSource) prefixIntervals).getIntervalsSource() + ); } - public void testWildcardIntervals() throws IOException { + public void testWildcardIntervals() { MappedFieldType ft = new MatchOnlyTextFieldType("field"); IntervalsSource wildcardIntervals = ft.wildcardIntervals(new BytesRef("foo"), MOCK_CONTEXT); assertThat(wildcardIntervals, Matchers.instanceOf(SourceIntervalsSource.class)); - assertEquals(Intervals.wildcard(new BytesRef("foo")), ((SourceIntervalsSource) wildcardIntervals).getIntervalsSource()); + assertEquals( + Intervals.wildcard(new BytesRef("foo"), IndexSearcher.getMaxClauseCount()), + ((SourceIntervalsSource) wildcardIntervals).getIntervalsSource() + ); + } + + public void testRegexpIntervals() { + MappedFieldType ft = new MatchOnlyTextFieldType("field"); + IntervalsSource regexpIntervals = ft.regexpIntervals(new BytesRef("foo"), MOCK_CONTEXT); + assertThat(regexpIntervals, Matchers.instanceOf(SourceIntervalsSource.class)); + assertEquals( + Intervals.regexp(new BytesRef("foo"), IndexSearcher.getMaxClauseCount()), + ((SourceIntervalsSource) regexpIntervals).getIntervalsSource() + ); } - public void testFuzzyIntervals() throws IOException { + public void testFuzzyIntervals() { MappedFieldType ft = new MatchOnlyTextFieldType("field"); IntervalsSource fuzzyIntervals = ft.fuzzyIntervals("foo", 1, 2, true, MOCK_CONTEXT); assertThat(fuzzyIntervals, Matchers.instanceOf(SourceIntervalsSource.class)); } + + public void testRangeIntervals() { + MappedFieldType ft = new MatchOnlyTextFieldType("field"); + IntervalsSource rangeIntervals = ft.rangeIntervals(new BytesRef("foo"), new BytesRef("foo1"), true, true, MOCK_CONTEXT); + assertThat(rangeIntervals, Matchers.instanceOf(SourceIntervalsSource.class)); + assertEquals( + Intervals.range(new BytesRef("foo"), new BytesRef("foo1"), true, true, IndexSearcher.getMaxClauseCount()), + ((SourceIntervalsSource) rangeIntervals).getIntervalsSource() + ); + } } diff --git a/qa/ccs-common-rest/build.gradle b/qa/ccs-common-rest/build.gradle index e5e8c5a489d5b..6121f7dcd4f82 100644 --- a/qa/ccs-common-rest/build.gradle +++ b/qa/ccs-common-rest/build.gradle @@ -10,7 +10,7 @@ apply plugin: 'elasticsearch.internal-yaml-rest-test' restResources { restApi { - include '_common', 'bulk', 'count', 'cluster', 'field_caps', 'get', 'knn_search', 'index', 'indices', 'msearch', + include 'capabilities', '_common', 'bulk', 'count', 'cluster', 'field_caps', 'get', 'knn_search', 'index', 'indices', 'msearch', 'search', 'async_search', 'graph', '*_point_in_time', 'info', 'scroll', 'clear_scroll', 'search_mvt', 'eql', 'sql' } restTests { diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/230_interval_query.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/230_interval_query.yml index 99bd001bd95e2..6a5f34b5207ce 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/230_interval_query.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/230_interval_query.yml @@ -476,3 +476,53 @@ setup: - match: { hits.hits.0._id: "6" } - match: { hits.hits.1._id: "5" } +--- +"Test regexp": + - requires: + capabilities: + - method: POST + path: /_search + capabilities: [ range_regexp_interval_queries ] + test_runner_features: capabilities + reason: "Support for range and regexp interval queries capability required" + - do: + search: + index: test + body: + query: + intervals: + text: + all_of: + intervals: + - match: + query: cold + - regexp: + pattern: ou.*ide + - match: { hits.total.value: 3 } + + +--- +"Test range": + - requires: + capabilities: + - method: POST + path: /_search + capabilities: [ range_regexp_interval_queries ] + test_runner_features: capabilities + reason: "Support for range and regexp interval queries capability required" + - do: + search: + index: test + body: + query: + intervals: + text: + all_of: + intervals: + - match: + query: cold + - range: + gte: out + lte: ouu + - match: { hits.total.value: 3 } + diff --git a/server/build.gradle b/server/build.gradle index 5c12d47da8102..963b3cfb2e747 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -44,6 +44,7 @@ dependencies { api "org.apache.lucene:lucene-core:${versions.lucene}" api "org.apache.lucene:lucene-analysis-common:${versions.lucene}" api "org.apache.lucene:lucene-backward-codecs:${versions.lucene}" + api "org.apache.lucene:lucene-facet:${versions.lucene}" api "org.apache.lucene:lucene-grouping:${versions.lucene}" api "org.apache.lucene:lucene-highlighter:${versions.lucene}" api "org.apache.lucene:lucene-join:${versions.lucene}" diff --git a/server/src/main/java/module-info.java b/server/src/main/java/module-info.java index 507fef10a5f44..56672957dd571 100644 --- a/server/src/main/java/module-info.java +++ b/server/src/main/java/module-info.java @@ -7,7 +7,6 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -import org.elasticsearch.index.codec.Elasticsearch814Codec; import org.elasticsearch.index.codec.tsdb.ES87TSDBDocValuesFormat; import org.elasticsearch.plugins.internal.RestExtension; @@ -455,7 +454,10 @@ org.elasticsearch.index.codec.vectors.ES815HnswBitVectorsFormat, org.elasticsearch.index.codec.vectors.ES815BitFlatVectorFormat; - provides org.apache.lucene.codecs.Codec with Elasticsearch814Codec; + provides org.apache.lucene.codecs.Codec + with + org.elasticsearch.index.codec.Elasticsearch814Codec, + org.elasticsearch.index.codec.Elasticsearch816Codec; provides org.apache.logging.log4j.core.util.ContextDataProvider with org.elasticsearch.common.logging.DynamicContextDataProvider; diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index be6d714c939de..998bc175dc6b6 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -230,6 +230,7 @@ static TransportVersion def(int id) { public static final TransportVersion ADD_DATA_STREAM_OPTIONS = def(8_754_00_0); public static final TransportVersion CCS_REMOTE_TELEMETRY_STATS = def(8_755_00_0); public static final TransportVersion ESQL_CCS_EXECUTION_INFO = def(8_756_00_0); + public static final TransportVersion REGEX_AND_RANGE_INTERVAL_QUERIES = def(8_757_00_0); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzer.java b/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzer.java index b45bce2d14d85..666708ea6ffde 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzer.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzer.java @@ -13,6 +13,7 @@ import org.apache.lucene.backward_codecs.lucene50.Lucene50PostingsFormat; import org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat; import org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat; +import org.apache.lucene.backward_codecs.lucene99.Lucene99PostingsFormat; import org.apache.lucene.codecs.DocValuesProducer; import org.apache.lucene.codecs.FieldsProducer; import org.apache.lucene.codecs.KnnVectorsReader; @@ -20,7 +21,7 @@ import org.apache.lucene.codecs.PointsReader; import org.apache.lucene.codecs.StoredFieldsReader; import org.apache.lucene.codecs.TermVectorsReader; -import org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat; +import org.apache.lucene.codecs.lucene912.Lucene912PostingsFormat; import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.ByteVectorValues; import org.apache.lucene.index.DirectoryReader; @@ -304,6 +305,9 @@ private static void readProximity(Terms terms, PostingsEnum postings) throws IOE private static BlockTermState getBlockTermState(TermsEnum termsEnum, BytesRef term) throws IOException { if (term != null && termsEnum.seekExact(term)) { final TermState termState = termsEnum.termState(); + if (termState instanceof final Lucene912PostingsFormat.IntBlockTermState blockTermState) { + return new BlockTermState(blockTermState.docStartFP, blockTermState.posStartFP, blockTermState.payStartFP); + } if (termState instanceof final ES812PostingsFormat.IntBlockTermState blockTermState) { return new BlockTermState(blockTermState.docStartFP, blockTermState.posStartFP, blockTermState.payStartFP); } diff --git a/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java b/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java index 3acd577aa42e3..c526652fc4e67 100644 --- a/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java +++ b/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java @@ -89,7 +89,7 @@ import java.util.Objects; public class Lucene { - public static final String LATEST_CODEC = "Lucene99"; + public static final String LATEST_CODEC = "Lucene912"; public static final String SOFT_DELETES_FIELD = "__soft_deletes"; @@ -242,7 +242,7 @@ public static void checkSegmentInfoIntegrity(final Directory directory) throws I @Override protected Object doBody(String segmentFileName) throws IOException { - try (IndexInput input = directory.openInput(segmentFileName, IOContext.READ)) { + try (IndexInput input = directory.openInput(segmentFileName, IOContext.READONCE)) { CodecUtil.checksumEntireFile(input); } return null; diff --git a/server/src/main/java/org/elasticsearch/index/IndexVersions.java b/server/src/main/java/org/elasticsearch/index/IndexVersions.java index 5f9cea7966560..9dde1a8d28e54 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexVersions.java +++ b/server/src/main/java/org/elasticsearch/index/IndexVersions.java @@ -116,6 +116,7 @@ private static IndexVersion def(int id, Version luceneVersion) { public static final IndexVersion LENIENT_UPDATEABLE_SYNONYMS = def(8_513_00_0, Version.LUCENE_9_11_1); public static final IndexVersion ENABLE_IGNORE_MALFORMED_LOGSDB = def(8_514_00_0, Version.LUCENE_9_11_1); public static final IndexVersion MERGE_ON_RECOVERY_VERSION = def(8_515_00_0, Version.LUCENE_9_11_1); + public static final IndexVersion UPGRADE_TO_LUCENE_9_12 = def(8_516_00_0, Version.LUCENE_9_12_0); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/index/codec/CodecService.java b/server/src/main/java/org/elasticsearch/index/codec/CodecService.java index f1a9d4ed2d211..144b99abe5644 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/CodecService.java +++ b/server/src/main/java/org/elasticsearch/index/codec/CodecService.java @@ -12,7 +12,7 @@ import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.FieldInfosFormat; import org.apache.lucene.codecs.FilterCodec; -import org.apache.lucene.codecs.lucene99.Lucene99Codec; +import org.apache.lucene.codecs.lucene912.Lucene912Codec; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.FeatureFlag; import org.elasticsearch.core.Nullable; @@ -46,7 +46,7 @@ public class CodecService implements CodecProvider { public CodecService(@Nullable MapperService mapperService, BigArrays bigArrays) { final var codecs = new HashMap(); - Codec legacyBestSpeedCodec = new LegacyPerFieldMapperCodec(Lucene99Codec.Mode.BEST_SPEED, mapperService, bigArrays); + Codec legacyBestSpeedCodec = new LegacyPerFieldMapperCodec(Lucene912Codec.Mode.BEST_SPEED, mapperService, bigArrays); if (ZSTD_STORED_FIELDS_FEATURE_FLAG.isEnabled()) { codecs.put(DEFAULT_CODEC, new PerFieldMapperCodec(Zstd814StoredFieldsFormat.Mode.BEST_SPEED, mapperService, bigArrays)); } else { @@ -58,7 +58,7 @@ public CodecService(@Nullable MapperService mapperService, BigArrays bigArrays) BEST_COMPRESSION_CODEC, new PerFieldMapperCodec(Zstd814StoredFieldsFormat.Mode.BEST_COMPRESSION, mapperService, bigArrays) ); - Codec legacyBestCompressionCodec = new LegacyPerFieldMapperCodec(Lucene99Codec.Mode.BEST_COMPRESSION, mapperService, bigArrays); + Codec legacyBestCompressionCodec = new LegacyPerFieldMapperCodec(Lucene912Codec.Mode.BEST_COMPRESSION, mapperService, bigArrays); codecs.put(LEGACY_BEST_COMPRESSION_CODEC, legacyBestCompressionCodec); codecs.put(LUCENE_DEFAULT_CODEC, Codec.getDefault()); diff --git a/server/src/main/java/org/elasticsearch/index/codec/Elasticsearch814Codec.java b/server/src/main/java/org/elasticsearch/index/codec/Elasticsearch814Codec.java index 44108109ad329..f3d758f4fc8b7 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/Elasticsearch814Codec.java +++ b/server/src/main/java/org/elasticsearch/index/codec/Elasticsearch814Codec.java @@ -9,14 +9,14 @@ package org.elasticsearch.index.codec; +import org.apache.lucene.backward_codecs.lucene99.Lucene99Codec; +import org.apache.lucene.backward_codecs.lucene99.Lucene99PostingsFormat; import org.apache.lucene.codecs.DocValuesFormat; import org.apache.lucene.codecs.KnnVectorsFormat; import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.codecs.StoredFieldsFormat; import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat; -import org.apache.lucene.codecs.lucene99.Lucene99Codec; import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat; -import org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat; import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat; import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat; diff --git a/server/src/main/java/org/elasticsearch/index/codec/Elasticsearch816Codec.java b/server/src/main/java/org/elasticsearch/index/codec/Elasticsearch816Codec.java new file mode 100644 index 0000000000000..00711c7ecc306 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/codec/Elasticsearch816Codec.java @@ -0,0 +1,131 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.codec; + +import org.apache.lucene.codecs.DocValuesFormat; +import org.apache.lucene.codecs.KnnVectorsFormat; +import org.apache.lucene.codecs.PostingsFormat; +import org.apache.lucene.codecs.StoredFieldsFormat; +import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat; +import org.apache.lucene.codecs.lucene912.Lucene912Codec; +import org.apache.lucene.codecs.lucene912.Lucene912PostingsFormat; +import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat; +import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat; +import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; +import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat; +import org.elasticsearch.index.codec.zstd.Zstd814StoredFieldsFormat; + +/** + * Elasticsearch codec as of 8.16. This extends the Lucene 9.12 codec to compressed stored fields with ZSTD instead of LZ4/DEFLATE. See + * {@link Zstd814StoredFieldsFormat}. + */ +public class Elasticsearch816Codec extends CodecService.DeduplicateFieldInfosCodec { + + private final StoredFieldsFormat storedFieldsFormat; + + private final PostingsFormat defaultPostingsFormat; + private final PostingsFormat postingsFormat = new PerFieldPostingsFormat() { + @Override + public PostingsFormat getPostingsFormatForField(String field) { + return Elasticsearch816Codec.this.getPostingsFormatForField(field); + } + }; + + private final DocValuesFormat defaultDVFormat; + private final DocValuesFormat docValuesFormat = new PerFieldDocValuesFormat() { + @Override + public DocValuesFormat getDocValuesFormatForField(String field) { + return Elasticsearch816Codec.this.getDocValuesFormatForField(field); + } + }; + + private final KnnVectorsFormat defaultKnnVectorsFormat; + private final KnnVectorsFormat knnVectorsFormat = new PerFieldKnnVectorsFormat() { + @Override + public KnnVectorsFormat getKnnVectorsFormatForField(String field) { + return Elasticsearch816Codec.this.getKnnVectorsFormatForField(field); + } + }; + + /** Public no-arg constructor, needed for SPI loading at read-time. */ + public Elasticsearch816Codec() { + this(Zstd814StoredFieldsFormat.Mode.BEST_SPEED); + } + + /** + * Constructor. Takes a {@link Zstd814StoredFieldsFormat.Mode} that describes whether to optimize for retrieval speed at the expense of + * worse space-efficiency or vice-versa. + */ + public Elasticsearch816Codec(Zstd814StoredFieldsFormat.Mode mode) { + super("Elasticsearch816", new Lucene912Codec()); + this.storedFieldsFormat = new Zstd814StoredFieldsFormat(mode); + this.defaultPostingsFormat = new Lucene912PostingsFormat(); + this.defaultDVFormat = new Lucene90DocValuesFormat(); + this.defaultKnnVectorsFormat = new Lucene99HnswVectorsFormat(); + } + + @Override + public StoredFieldsFormat storedFieldsFormat() { + return storedFieldsFormat; + } + + @Override + public final PostingsFormat postingsFormat() { + return postingsFormat; + } + + @Override + public final DocValuesFormat docValuesFormat() { + return docValuesFormat; + } + + @Override + public final KnnVectorsFormat knnVectorsFormat() { + return knnVectorsFormat; + } + + /** + * Returns the postings format that should be used for writing new segments of field. + * + *

The default implementation always returns "Lucene912". + * + *

WARNING: if you subclass, you are responsible for index backwards compatibility: + * future version of Lucene are only guaranteed to be able to read the default implementation, + */ + public PostingsFormat getPostingsFormatForField(String field) { + return defaultPostingsFormat; + } + + /** + * Returns the docvalues format that should be used for writing new segments of field + * . + * + *

The default implementation always returns "Lucene912". + * + *

WARNING: if you subclass, you are responsible for index backwards compatibility: + * future version of Lucene are only guaranteed to be able to read the default implementation. + */ + public DocValuesFormat getDocValuesFormatForField(String field) { + return defaultDVFormat; + } + + /** + * Returns the vectors format that should be used for writing new segments of field + * + *

The default implementation always returns "Lucene912". + * + *

WARNING: if you subclass, you are responsible for index backwards compatibility: + * future version of Lucene are only guaranteed to be able to read the default implementation. + */ + public KnnVectorsFormat getKnnVectorsFormatForField(String field) { + return defaultKnnVectorsFormat; + } + +} diff --git a/server/src/main/java/org/elasticsearch/index/codec/LegacyPerFieldMapperCodec.java b/server/src/main/java/org/elasticsearch/index/codec/LegacyPerFieldMapperCodec.java index 5d97f78e2747b..64c2ca788f63c 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/LegacyPerFieldMapperCodec.java +++ b/server/src/main/java/org/elasticsearch/index/codec/LegacyPerFieldMapperCodec.java @@ -13,7 +13,7 @@ import org.apache.lucene.codecs.DocValuesFormat; import org.apache.lucene.codecs.KnnVectorsFormat; import org.apache.lucene.codecs.PostingsFormat; -import org.apache.lucene.codecs.lucene99.Lucene99Codec; +import org.apache.lucene.codecs.lucene912.Lucene912Codec; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.index.mapper.MapperService; @@ -22,11 +22,11 @@ * Legacy version of {@link PerFieldMapperCodec}. This codec is preserved to give an escape hatch in case we encounter issues with new * changes in {@link PerFieldMapperCodec}. */ -public final class LegacyPerFieldMapperCodec extends Lucene99Codec { +public final class LegacyPerFieldMapperCodec extends Lucene912Codec { private final PerFieldFormatSupplier formatSupplier; - public LegacyPerFieldMapperCodec(Lucene99Codec.Mode compressionMode, MapperService mapperService, BigArrays bigArrays) { + public LegacyPerFieldMapperCodec(Lucene912Codec.Mode compressionMode, MapperService mapperService, BigArrays bigArrays) { super(compressionMode); this.formatSupplier = new PerFieldFormatSupplier(mapperService, bigArrays); // If the below assertion fails, it is a sign that Lucene released a new codec. You must create a copy of the current Elasticsearch diff --git a/server/src/main/java/org/elasticsearch/index/codec/PerFieldMapperCodec.java b/server/src/main/java/org/elasticsearch/index/codec/PerFieldMapperCodec.java index 46b05fdd282db..83c5cb396d88b 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/PerFieldMapperCodec.java +++ b/server/src/main/java/org/elasticsearch/index/codec/PerFieldMapperCodec.java @@ -26,7 +26,7 @@ * per index in real time via the mapping API. If no specific postings format or vector format is * configured for a specific field the default postings or vector format is used. */ -public final class PerFieldMapperCodec extends Elasticsearch814Codec { +public final class PerFieldMapperCodec extends Elasticsearch816Codec { private final PerFieldFormatSupplier formatSupplier; diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813FlatVectorFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813FlatVectorFormat.java index 5818af87feac7..7a8d09c02ba3b 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813FlatVectorFormat.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813FlatVectorFormat.java @@ -66,7 +66,7 @@ static class ES813FlatVectorWriter extends KnnVectorsWriter { @Override public KnnFieldVectorsWriter addField(FieldInfo fieldInfo) throws IOException { - return writer.addField(fieldInfo, null); + return writer.addField(fieldInfo); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813Int8FlatVectorFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813Int8FlatVectorFormat.java index d2c40a890e246..248421fb99d1c 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813Int8FlatVectorFormat.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813Int8FlatVectorFormat.java @@ -74,7 +74,7 @@ public ES813FlatVectorWriter(FlatVectorsWriter writer) { @Override public KnnFieldVectorsWriter addField(FieldInfo fieldInfo) throws IOException { - return writer.addField(fieldInfo, null); + return writer.addField(fieldInfo); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES814ScalarQuantizedVectorsFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES814ScalarQuantizedVectorsFormat.java index 9d993bd948f0f..4313aa40cf13e 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES814ScalarQuantizedVectorsFormat.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES814ScalarQuantizedVectorsFormat.java @@ -9,7 +9,6 @@ package org.elasticsearch.index.codec.vectors; -import org.apache.lucene.codecs.KnnFieldVectorsWriter; import org.apache.lucene.codecs.hnsw.DefaultFlatVectorScorer; import org.apache.lucene.codecs.hnsw.FlatFieldVectorsWriter; import org.apache.lucene.codecs.hnsw.FlatVectorsFormat; @@ -67,6 +66,7 @@ public class ES814ScalarQuantizedVectorsFormat extends FlatVectorsFormat { private final boolean compress; public ES814ScalarQuantizedVectorsFormat(Float confidenceInterval, int bits, boolean compress) { + super(NAME); if (confidenceInterval != null && confidenceInterval != DYNAMIC_CONFIDENCE_INTERVAL && (confidenceInterval < MINIMUM_CONFIDENCE_INTERVAL || confidenceInterval > MAXIMUM_CONFIDENCE_INTERVAL)) { @@ -137,8 +137,8 @@ static final class ES814ScalarQuantizedVectorsWriter extends FlatVectorsWriter { } @Override - public FlatFieldVectorsWriter addField(FieldInfo fieldInfo, KnnFieldVectorsWriter knnFieldVectorsWriter) throws IOException { - return delegate.addField(fieldInfo, knnFieldVectorsWriter); + public FlatFieldVectorsWriter addField(FieldInfo fieldInfo) throws IOException { + return delegate.addField(fieldInfo); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES815BitFlatVectorsFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES815BitFlatVectorsFormat.java index cc6479cf1e2bf..f1ae4e3fdeded 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES815BitFlatVectorsFormat.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES815BitFlatVectorsFormat.java @@ -29,6 +29,10 @@ class ES815BitFlatVectorsFormat extends FlatVectorsFormat { private final FlatVectorsFormat delegate = new Lucene99FlatVectorsFormat(FlatBitVectorScorer.INSTANCE); + protected ES815BitFlatVectorsFormat() { + super("ES815BitFlatVectorsFormat"); + } + @Override public FlatVectorsWriter fieldsWriter(SegmentWriteState segmentWriteState) throws IOException { return delegate.fieldsWriter(segmentWriteState); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java index 9f60d99e0ded4..53ccccdbd4bab 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java @@ -345,7 +345,7 @@ public CompletionFieldType fieldType() { } static PostingsFormat postingsFormat() { - return PostingsFormat.forName("Completion99"); + return PostingsFormat.forName("Completion912"); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java index 2c851b70d2606..35722be20b9be 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java @@ -440,6 +440,30 @@ public IntervalsSource wildcardIntervals(BytesRef pattern, SearchExecutionContex ); } + /** + * Create a regexp {@link IntervalsSource} for the given pattern. + */ + public IntervalsSource regexpIntervals(BytesRef pattern, SearchExecutionContext context) { + throw new IllegalArgumentException( + "Can only use interval queries on text fields - not on [" + name + "] which is of type [" + typeName() + "]" + ); + } + + /** + * Create a range {@link IntervalsSource} for the given ranges + */ + public IntervalsSource rangeIntervals( + BytesRef lowerTerm, + BytesRef upperTerm, + boolean includeLower, + boolean includeUpper, + SearchExecutionContext context + ) { + throw new IllegalArgumentException( + "Can only use interval queries on text fields - not on [" + name + "] which is of type [" + typeName() + "]" + ); + } + /** * An enum used to describe the relation between the range of terms in a * shard when compared with a query range diff --git a/server/src/main/java/org/elasticsearch/index/mapper/PlaceHolderFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/PlaceHolderFieldMapper.java index 9a6dd1d127651..670ddc4d5ccda 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/PlaceHolderFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/PlaceHolderFieldMapper.java @@ -248,6 +248,22 @@ public IntervalsSource wildcardIntervals(BytesRef pattern, SearchExecutionContex throw new QueryShardException(context, fail("wildcard intervals query")); } + @Override + public IntervalsSource regexpIntervals(BytesRef pattern, SearchExecutionContext context) { + throw new QueryShardException(context, fail("regexp intervals query")); + } + + @Override + public IntervalsSource rangeIntervals( + BytesRef lowerTerm, + BytesRef upperTerm, + boolean includeLower, + boolean includeUpper, + SearchExecutionContext context + ) { + throw new QueryShardException(context, fail("range intervals query")); + } + @Override public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext) { throw new IllegalArgumentException(fail("aggregation or sorts")); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java index a30793cdc5d97..2c55fc35db57d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java @@ -36,6 +36,7 @@ import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.ConstantScoreQuery; import org.apache.lucene.search.FuzzyQuery; +import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MultiPhraseQuery; import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.PhraseQuery; @@ -620,7 +621,10 @@ public IntervalsSource intervals(BytesRef term) { return Intervals.fixField(name(), Intervals.term(term)); } String wildcardTerm = term.utf8ToString() + "?".repeat(Math.max(0, minChars - term.length)); - return Intervals.or(Intervals.fixField(name(), Intervals.wildcard(new BytesRef(wildcardTerm))), Intervals.term(term)); + return Intervals.or( + Intervals.fixField(name(), Intervals.wildcard(new BytesRef(wildcardTerm), IndexSearcher.getMaxClauseCount())), + Intervals.term(term) + ); } @Override @@ -822,7 +826,7 @@ public IntervalsSource prefixIntervals(BytesRef term, SearchExecutionContext con if (prefixFieldType != null) { return prefixFieldType.intervals(term); } - return Intervals.prefix(term); + return Intervals.prefix(term, IndexSearcher.getMaxClauseCount()); } @Override @@ -836,8 +840,14 @@ public IntervalsSource fuzzyIntervals( if (getTextSearchInfo().hasPositions() == false) { throw new IllegalArgumentException("Cannot create intervals over field [" + name() + "] with no positions indexed"); } - FuzzyQuery fq = new FuzzyQuery(new Term(name(), term), maxDistance, prefixLength, 128, transpositions); - return Intervals.multiterm(fq.getAutomata(), term); + FuzzyQuery fq = new FuzzyQuery( + new Term(name(), term), + maxDistance, + prefixLength, + IndexSearcher.getMaxClauseCount(), + transpositions + ); + return Intervals.multiterm(fq.getAutomata(), IndexSearcher.getMaxClauseCount(), term); } @Override @@ -845,7 +855,29 @@ public IntervalsSource wildcardIntervals(BytesRef pattern, SearchExecutionContex if (getTextSearchInfo().hasPositions() == false) { throw new IllegalArgumentException("Cannot create intervals over field [" + name() + "] with no positions indexed"); } - return Intervals.wildcard(pattern); + return Intervals.wildcard(pattern, IndexSearcher.getMaxClauseCount()); + } + + @Override + public IntervalsSource regexpIntervals(BytesRef pattern, SearchExecutionContext context) { + if (getTextSearchInfo().hasPositions() == false) { + throw new IllegalArgumentException("Cannot create intervals over field [" + name() + "] with no positions indexed"); + } + return Intervals.regexp(pattern, IndexSearcher.getMaxClauseCount()); + } + + @Override + public IntervalsSource rangeIntervals( + BytesRef lowerTerm, + BytesRef upperTerm, + boolean includeLower, + boolean includeUpper, + SearchExecutionContext context + ) { + if (getTextSearchInfo().hasPositions() == false) { + throw new IllegalArgumentException("Cannot create intervals over field [" + name() + "] with no positions indexed"); + } + return Intervals.range(lowerTerm, upperTerm, includeLower, includeUpper, IndexSearcher.getMaxClauseCount()); } private void checkForPositions() { diff --git a/server/src/main/java/org/elasticsearch/index/query/IntervalsSourceProvider.java b/server/src/main/java/org/elasticsearch/index/query/IntervalsSourceProvider.java index 9b579c97f197a..647e45d1beda1 100644 --- a/server/src/main/java/org/elasticsearch/index/query/IntervalsSourceProvider.java +++ b/server/src/main/java/org/elasticsearch/index/query/IntervalsSourceProvider.java @@ -14,11 +14,13 @@ import org.apache.lucene.queries.intervals.Intervals; import org.apache.lucene.queries.intervals.IntervalsSource; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.io.stream.NamedWriteable; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.VersionedNamedWriteable; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.index.analysis.NamedAnalyzer; @@ -77,10 +79,16 @@ public static IntervalsSourceProvider fromXContent(XContentParser parser) throws return Wildcard.fromXContent(parser); case "fuzzy": return Fuzzy.fromXContent(parser); + case "regexp": + return Regexp.fromXContent(parser); + case "range": + return Range.fromXContent(parser); } throw new ParsingException( parser.getTokenLocation(), - "Unknown interval type [" + parser.currentName() + "], expecting one of [match, any_of, all_of, prefix, wildcard]" + "Unknown interval type [" + + parser.currentName() + + "], expecting one of [match, any_of, all_of, prefix, wildcard, regexp, range]" ); } @@ -747,6 +755,129 @@ String getUseField() { } } + public static class Regexp extends IntervalsSourceProvider implements VersionedNamedWriteable { + + public static final String NAME = "regexp"; + + private final String pattern; + private final String analyzer; + private final String useField; + + public Regexp(String pattern, String analyzer, String useField) { + this.pattern = pattern; + this.analyzer = analyzer; + this.useField = useField; + } + + public Regexp(StreamInput in) throws IOException { + this.pattern = in.readString(); + this.analyzer = in.readOptionalString(); + this.useField = in.readOptionalString(); + } + + @Override + public IntervalsSource getSource(SearchExecutionContext context, MappedFieldType fieldType) { + NamedAnalyzer analyzer = null; + if (this.analyzer != null) { + analyzer = context.getIndexAnalyzers().get(this.analyzer); + } + if (useField != null) { + fieldType = context.getFieldType(useField); + assert fieldType != null; + } + if (analyzer == null) { + analyzer = fieldType.getTextSearchInfo().searchAnalyzer(); + } + BytesRef normalizedPattern = analyzer.normalize(fieldType.name(), pattern); + IntervalsSource source = fieldType.regexpIntervals(normalizedPattern, context); + if (useField != null) { + source = Intervals.fixField(useField, source); + } + return source; + } + + @Override + public void extractFields(Set fields) { + if (useField != null) { + fields.add(useField); + } + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Regexp regexp = (Regexp) o; + return Objects.equals(pattern, regexp.pattern) + && Objects.equals(analyzer, regexp.analyzer) + && Objects.equals(useField, regexp.useField); + } + + @Override + public int hashCode() { + return Objects.hash(pattern, analyzer, useField); + } + + @Override + public String getWriteableName() { + return NAME; + } + + @Override + public TransportVersion getMinimalSupportedVersion() { + return TransportVersions.REGEX_AND_RANGE_INTERVAL_QUERIES; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(pattern); + out.writeOptionalString(analyzer); + out.writeOptionalString(useField); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(NAME); + builder.field("pattern", pattern); + if (analyzer != null) { + builder.field("analyzer", analyzer); + } + if (useField != null) { + builder.field("use_field", useField); + } + builder.endObject(); + return builder; + } + + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>(NAME, args -> { + String term = (String) args[0]; + String analyzer = (String) args[1]; + String useField = (String) args[2]; + return new Regexp(term, analyzer, useField); + }); + static { + PARSER.declareString(constructorArg(), new ParseField("pattern")); + PARSER.declareString(optionalConstructorArg(), new ParseField("analyzer")); + PARSER.declareString(optionalConstructorArg(), new ParseField("use_field")); + } + + public static Regexp fromXContent(XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } + + String getPattern() { + return pattern; + } + + String getAnalyzer() { + return analyzer; + } + + String getUseField() { + return useField; + } + } + public static class Fuzzy extends IntervalsSourceProvider { public static final String NAME = "fuzzy"; @@ -908,6 +1039,190 @@ String getUseField() { } } + public static class Range extends IntervalsSourceProvider implements VersionedNamedWriteable { + + public static final String NAME = "range"; + + private final String lowerTerm; + private final String upperTerm; + private final boolean includeLower; + private final boolean includeUpper; + private final String analyzer; + private final String useField; + + public Range(String lowerTerm, String upperTerm, boolean includeLower, boolean includeUpper, String analyzer, String useField) { + this.lowerTerm = lowerTerm; + this.upperTerm = upperTerm; + this.includeLower = includeLower; + this.includeUpper = includeUpper; + this.analyzer = analyzer; + this.useField = useField; + } + + public Range(StreamInput in) throws IOException { + this.lowerTerm = in.readString(); + this.upperTerm = in.readString(); + this.includeLower = in.readBoolean(); + this.includeUpper = in.readBoolean(); + this.analyzer = in.readOptionalString(); + this.useField = in.readOptionalString(); + } + + @Override + public IntervalsSource getSource(SearchExecutionContext context, MappedFieldType fieldType) { + NamedAnalyzer analyzer = null; + if (this.analyzer != null) { + analyzer = context.getIndexAnalyzers().get(this.analyzer); + } + if (useField != null) { + fieldType = context.getFieldType(useField); + assert fieldType != null; + } + if (analyzer == null) { + analyzer = fieldType.getTextSearchInfo().searchAnalyzer(); + } + BytesRef normalizedLowerTerm = analyzer.normalize(fieldType.name(), lowerTerm); + BytesRef normalizedUpperTerm = analyzer.normalize(fieldType.name(), upperTerm); + + IntervalsSource source = fieldType.rangeIntervals( + normalizedLowerTerm, + normalizedUpperTerm, + includeLower, + includeUpper, + context + ); + if (useField != null) { + source = Intervals.fixField(useField, source); + } + return source; + } + + @Override + public void extractFields(Set fields) { + if (useField != null) { + fields.add(useField); + } + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Range range = (Range) o; + return includeLower == range.includeLower + && includeUpper == range.includeUpper + && Objects.equals(lowerTerm, range.lowerTerm) + && Objects.equals(upperTerm, range.upperTerm) + && Objects.equals(analyzer, range.analyzer) + && Objects.equals(useField, range.useField); + } + + @Override + public int hashCode() { + return Objects.hash(lowerTerm, upperTerm, includeLower, includeUpper, analyzer, useField); + } + + @Override + public String getWriteableName() { + return NAME; + } + + @Override + public TransportVersion getMinimalSupportedVersion() { + return TransportVersions.REGEX_AND_RANGE_INTERVAL_QUERIES; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(lowerTerm); + out.writeString(upperTerm); + out.writeBoolean(includeLower); + out.writeBoolean(includeUpper); + out.writeOptionalString(analyzer); + out.writeOptionalString(useField); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(NAME); + if (includeLower) { + builder.field("gte", lowerTerm); + } else { + builder.field("gt", lowerTerm); + } + if (includeUpper) { + builder.field("lte", upperTerm); + } else { + builder.field("lt", upperTerm); + } + if (analyzer != null) { + builder.field("analyzer", analyzer); + } + if (useField != null) { + builder.field("use_field", useField); + } + builder.endObject(); + return builder; + } + + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>(NAME, args -> { + String gte = (String) args[0]; + String gt = (String) args[1]; + String lte = (String) args[2]; + String lt = (String) args[3]; + if ((gte == null && gt == null) || (gte != null && gt != null)) { + throw new IllegalArgumentException("Either [gte] or [gt], one of them must be provided"); + } + if ((lte == null && lt == null) || (lte != null && lt != null)) { + throw new IllegalArgumentException("Either [lte] or [lt], one of them must be provided"); + } + boolean includeLower = gte != null ? true : false; + String lowerTerm = gte != null ? gte : gt; + boolean includeUpper = lte != null ? true : false; + String upperTerm = lte != null ? lte : lt; + String analyzer = (String) args[4]; + String useField = (String) args[5]; + return new Range(lowerTerm, upperTerm, includeLower, includeUpper, analyzer, useField); + }); + + static { + PARSER.declareString(optionalConstructorArg(), new ParseField("gte")); + PARSER.declareString(optionalConstructorArg(), new ParseField("gt")); + PARSER.declareString(optionalConstructorArg(), new ParseField("lte")); + PARSER.declareString(optionalConstructorArg(), new ParseField("lt")); + PARSER.declareString(optionalConstructorArg(), new ParseField("analyzer")); + PARSER.declareString(optionalConstructorArg(), new ParseField("use_field")); + } + + public static Range fromXContent(XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } + + String getLowerTerm() { + return lowerTerm; + } + + String getUpperTerm() { + return upperTerm; + } + + boolean getIncludeLower() { + return includeLower; + } + + boolean getIncludeUpper() { + return includeUpper; + } + + String getAnalyzer() { + return analyzer; + } + + String getUseField() { + return useField; + } + } + static class ScriptFilterSource extends FilteredIntervalsSource { final IntervalFilterScript script; diff --git a/server/src/main/java/org/elasticsearch/index/store/LuceneFilesExtensions.java b/server/src/main/java/org/elasticsearch/index/store/LuceneFilesExtensions.java index fdc508098d82e..186aff230b8d0 100644 --- a/server/src/main/java/org/elasticsearch/index/store/LuceneFilesExtensions.java +++ b/server/src/main/java/org/elasticsearch/index/store/LuceneFilesExtensions.java @@ -57,6 +57,7 @@ public enum LuceneFilesExtensions { NVM("nvm", "Norms Metadata", true, false), PAY("pay", "Payloads", false, false), POS("pos", "Positions", false, false), + PSM("psm", "Postings Metadata", true, false), SI("si", "Segment Info", true, false), // Term dictionaries are typically performance-sensitive and hot in the page // cache, so we use mmap, which provides better performance. diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java b/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java index 3b0e4e048613c..556bf43425bd8 100644 --- a/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java +++ b/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java @@ -1376,7 +1376,7 @@ protected void onNewResource(StoreFileMetadata md) throws IOException { // we already have the file contents on heap no need to open the file again currentInput = null; } else { - currentInput = store.directory().openInput(md.name(), IOContext.READONCE); + currentInput = store.directory().openInput(md.name(), IOContext.READ); } } diff --git a/server/src/main/java/org/elasticsearch/lucene/similarity/LegacyBM25Similarity.java b/server/src/main/java/org/elasticsearch/lucene/similarity/LegacyBM25Similarity.java index 7421579d643e4..d420e519a30e7 100644 --- a/server/src/main/java/org/elasticsearch/lucene/similarity/LegacyBM25Similarity.java +++ b/server/src/main/java/org/elasticsearch/lucene/similarity/LegacyBM25Similarity.java @@ -43,7 +43,7 @@ public final class LegacyBM25Similarity extends Similarity { * */ public LegacyBM25Similarity() { - this.bm25Similarity = new BM25Similarity(); + this(new BM25Similarity()); } /** @@ -54,7 +54,12 @@ public LegacyBM25Similarity() { * not within the range {@code [0..1]} */ public LegacyBM25Similarity(float k1, float b, boolean discountOverlaps) { - this.bm25Similarity = new BM25Similarity(k1, b, discountOverlaps); + this(new BM25Similarity(k1, b, discountOverlaps)); + } + + private LegacyBM25Similarity(BM25Similarity bm25Similarity) { + super(bm25Similarity.getDiscountOverlaps()); + this.bm25Similarity = bm25Similarity; } @Override @@ -81,13 +86,6 @@ public float getB() { return bm25Similarity.getB(); } - /** - * Returns true if overlap tokens are discounted from the document's length. - */ - public boolean getDiscountOverlaps() { - return bm25Similarity.getDiscountOverlaps(); - } - @Override public String toString() { return bm25Similarity.toString(); diff --git a/server/src/main/java/org/elasticsearch/lucene/util/CombinedBitSet.java b/server/src/main/java/org/elasticsearch/lucene/util/CombinedBitSet.java index 2a2c816a9ce54..be41959417f14 100644 --- a/server/src/main/java/org/elasticsearch/lucene/util/CombinedBitSet.java +++ b/server/src/main/java/org/elasticsearch/lucene/util/CombinedBitSet.java @@ -77,6 +77,19 @@ public int nextSetBit(int index) { return next; } + @Override + public int nextSetBit(int index, int upperBound) { + assert index >= 0 && index < length : "index=" + index + " numBits=" + length(); + int next = first.nextSetBit(index, upperBound); + while (next != DocIdSetIterator.NO_MORE_DOCS && second.get(next) == false) { + if (next == length() - 1) { + return DocIdSetIterator.NO_MORE_DOCS; + } + next = first.nextSetBit(next + 1, upperBound); + } + return next; + } + @Override public long ramBytesUsed() { return first.ramBytesUsed(); diff --git a/server/src/main/java/org/elasticsearch/lucene/util/MatchAllBitSet.java b/server/src/main/java/org/elasticsearch/lucene/util/MatchAllBitSet.java index e315dc046fe92..e46bb78bd7954 100644 --- a/server/src/main/java/org/elasticsearch/lucene/util/MatchAllBitSet.java +++ b/server/src/main/java/org/elasticsearch/lucene/util/MatchAllBitSet.java @@ -69,6 +69,12 @@ public int nextSetBit(int index) { return index; } + @Override + public int nextSetBit(int index, int upperBound) { + assert index < upperBound; + return index; + } + @Override public long ramBytesUsed() { return RAM_BYTES_USED; diff --git a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java index b3c95186b6037..9a7dfba095723 100644 --- a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java +++ b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java @@ -85,6 +85,7 @@ import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.core.TimeValue; import org.elasticsearch.core.Tuple; +import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.discovery.DiscoveryModule; import org.elasticsearch.env.Environment; import org.elasticsearch.env.NodeEnvironment; @@ -509,6 +510,7 @@ private SettingsModule validateSettings(Settings envSettings, Settings settings, for (final ExecutorBuilder builder : threadPool.builders()) { additionalSettings.addAll(builder.getRegisteredSettings()); } + addBwcSearchWorkerSettings(additionalSettings); SettingsExtension.load().forEach(e -> additionalSettings.addAll(e.getSettings())); // this is as early as we can validate settings at this point. we already pass them to ThreadPool @@ -539,6 +541,17 @@ private SettingsModule validateSettings(Settings envSettings, Settings settings, return settingsModule; } + @UpdateForV9 + private static void addBwcSearchWorkerSettings(List> additionalSettings) { + // TODO remove the below settings, they are unused and only here to enable BwC for deployments that still use them + additionalSettings.add( + Setting.intSetting("thread_pool.search_worker.queue_size", 0, Setting.Property.NodeScope, Setting.Property.DeprecatedWarning) + ); + additionalSettings.add( + Setting.intSetting("thread_pool.search_worker.size", 0, Setting.Property.NodeScope, Setting.Property.DeprecatedWarning) + ); + } + private SearchModule createSearchModule(Settings settings, ThreadPool threadPool, TelemetryProvider telemetryProvider) { IndexSearcher.setMaxClauseCount(SearchUtils.calculateMaxClauseValue(threadPool)); return new SearchModule(settings, pluginsService.filterPlugins(SearchPlugin.class).toList(), telemetryProvider); diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index a53db3c5cc2de..1975e6cb55940 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -4030,7 +4030,7 @@ protected void snapshotFile(SnapshotShardContext context, FileInfo fileInfo) thr final String file = fileInfo.physicalName(); try ( Releasable ignored = context.withCommitRef(); - IndexInput indexInput = store.openVerifyingInput(file, IOContext.READONCE, fileInfo.metadata()) + IndexInput indexInput = store.openVerifyingInput(file, IOContext.READ, fileInfo.metadata()) ) { for (int i = 0; i < fileInfo.numberOfParts(); i++) { final long partBytes = fileInfo.partBytes(i); diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java b/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java index 28330c7c45479..38157efd8a370 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java @@ -95,6 +95,11 @@ public List routes() { ); } + @Override + public Set supportedCapabilities() { + return SearchCapabilities.CAPABILITIES; + } + @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java b/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java new file mode 100644 index 0000000000000..45fd6afe4fca6 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.rest.action.search; + +import java.util.Set; + +/** + * A {@link Set} of "capabilities" supported by the {@link RestSearchAction}. + */ +public final class SearchCapabilities { + + private SearchCapabilities() {} + + /** Support regex and range match rules in interval queries. */ + private static final String RANGE_REGEX_INTERVAL_QUERY_CAPABILITY = "range_regexp_interval_queries"; + + public static final Set CAPABILITIES = Set.of(RANGE_REGEX_INTERVAL_QUERY_CAPABILITY); +} diff --git a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java index de4cde5393c69..1521b17a81766 100644 --- a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java +++ b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java @@ -180,7 +180,14 @@ final class DefaultSearchContext extends SearchContext { this.indexShard = readerContext.indexShard(); Engine.Searcher engineSearcher = readerContext.acquireSearcher("search"); - if (executor == null) { + int maximumNumberOfSlices = determineMaximumNumberOfSlices( + executor, + request, + resultsType, + enableQueryPhaseParallelCollection, + field -> getFieldCardinality(field, readerContext.indexService(), engineSearcher.getDirectoryReader()) + ); + if (executor == null || maximumNumberOfSlices <= 1) { this.searcher = new ContextIndexSearcher( engineSearcher.getIndexReader(), engineSearcher.getSimilarity(), @@ -196,13 +203,7 @@ final class DefaultSearchContext extends SearchContext { engineSearcher.getQueryCachingPolicy(), lowLevelCancellation, executor, - determineMaximumNumberOfSlices( - executor, - request, - resultsType, - enableQueryPhaseParallelCollection, - field -> getFieldCardinality(field, readerContext.indexService(), engineSearcher.getDirectoryReader()) - ), + maximumNumberOfSlices, minimumDocsPerSlice ); } @@ -290,6 +291,7 @@ static int determineMaximumNumberOfSlices( ToLongFunction fieldCardinality ) { return executor instanceof ThreadPoolExecutor tpe + && tpe.getQueue().isEmpty() && isParallelCollectionSupportedForResults(resultsType, request.source(), fieldCardinality, enableQueryPhaseParallelCollection) ? tpe.getMaximumPoolSize() : 1; diff --git a/server/src/main/java/org/elasticsearch/search/SearchModule.java b/server/src/main/java/org/elasticsearch/search/SearchModule.java index 4afcc57b7b15a..6308b19358410 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchModule.java +++ b/server/src/main/java/org/elasticsearch/search/SearchModule.java @@ -1263,6 +1263,16 @@ public static List getIntervalsSourceProviderNamed IntervalsSourceProvider.class, IntervalsSourceProvider.Fuzzy.NAME, IntervalsSourceProvider.Fuzzy::new + ), + new NamedWriteableRegistry.Entry( + IntervalsSourceProvider.class, + IntervalsSourceProvider.Regexp.NAME, + IntervalsSourceProvider.Regexp::new + ), + new NamedWriteableRegistry.Entry( + IntervalsSourceProvider.class, + IntervalsSourceProvider.Range.NAME, + IntervalsSourceProvider.Range::new ) ); } diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java index a0b91261236b0..70101bbc7fc54 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchService.java +++ b/server/src/main/java/org/elasticsearch/search/SearchService.java @@ -142,7 +142,6 @@ import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.Executor; -import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -228,7 +227,8 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv "search.worker_threads_enabled", true, Property.NodeScope, - Property.Dynamic + Property.Dynamic, + Property.DeprecatedWarning ); public static final Setting QUERY_PHASE_PARALLEL_COLLECTION_ENABLED = Setting.boolSetting( @@ -279,7 +279,7 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv private final FetchPhase fetchPhase; private final RankFeatureShardPhase rankFeatureShardPhase; - private volatile boolean enableSearchWorkerThreads; + private volatile Executor searchExecutor; private volatile boolean enableQueryPhaseParallelCollection; private volatile long defaultKeepAlive; @@ -373,7 +373,10 @@ public SearchService( clusterService.getClusterSettings() .addSettingsUpdateConsumer(ENABLE_REWRITE_AGGS_TO_FILTER_BY_FILTER, this::setEnableRewriteAggsToFilterByFilter); - enableSearchWorkerThreads = SEARCH_WORKER_THREADS_ENABLED.get(settings); + if (SEARCH_WORKER_THREADS_ENABLED.get(settings)) { + searchExecutor = threadPool.executor(Names.SEARCH); + } + clusterService.getClusterSettings().addSettingsUpdateConsumer(SEARCH_WORKER_THREADS_ENABLED, this::setEnableSearchWorkerThreads); enableQueryPhaseParallelCollection = QUERY_PHASE_PARALLEL_COLLECTION_ENABLED.get(settings); @@ -382,7 +385,11 @@ public SearchService( } private void setEnableSearchWorkerThreads(boolean enableSearchWorkerThreads) { - this.enableSearchWorkerThreads = enableSearchWorkerThreads; + if (enableSearchWorkerThreads) { + searchExecutor = threadPool.executor(Names.SEARCH); + } else { + searchExecutor = null; + } } private void setEnableQueryPhaseParallelCollection(boolean enableQueryPhaseParallelCollection) { @@ -1111,7 +1118,6 @@ private DefaultSearchContext createSearchContext( reader.indexShard().shardId(), request.getClusterAlias() ); - ExecutorService executor = this.enableSearchWorkerThreads ? threadPool.executor(Names.SEARCH_WORKER) : null; searchContext = new DefaultSearchContext( reader, request, @@ -1120,7 +1126,7 @@ private DefaultSearchContext createSearchContext( timeout, fetchPhase, lowLevelCancellation, - executor, + searchExecutor, resultsType, enableQueryPhaseParallelCollection, minimumDocsPerSlice diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/TimeSeriesIndexSearcher.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/TimeSeriesIndexSearcher.java index ac8cc5c8232eb..742d366efa7a3 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/support/TimeSeriesIndexSearcher.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/TimeSeriesIndexSearcher.java @@ -22,7 +22,6 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefBuilder; import org.apache.lucene.util.PriorityQueue; -import org.apache.lucene.util.ThreadInterruptedException; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.common.lucene.search.function.MinScoreScorer; import org.elasticsearch.index.mapper.DataStreamTimestampFieldMapper; @@ -38,9 +37,6 @@ import java.util.ArrayList; import java.util.Iterator; import java.util.List; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.FutureTask; -import java.util.concurrent.RunnableFuture; import java.util.function.IntSupplier; import static org.elasticsearch.index.IndexSortConfig.TIME_SERIES_SORT; @@ -68,10 +64,7 @@ public TimeSeriesIndexSearcher(IndexSearcher searcher, List cancellati searcher.getSimilarity(), searcher.getQueryCache(), searcher.getQueryCachingPolicy(), - false, - searcher.getExecutor(), - 1, - -1 + false ); } catch (IOException e) { // IOException from wrapping the index searcher which should never happen. @@ -94,28 +87,8 @@ public void setMinimumScore(Float minimumScore) { public void search(Query query, BucketCollector bucketCollector) throws IOException { query = searcher.rewrite(query); Weight weight = searcher.createWeight(query, bucketCollector.scoreMode(), 1); - if (searcher.getExecutor() == null) { - search(bucketCollector, weight); - bucketCollector.postCollection(); - return; - } - // offload to the search worker thread pool whenever possible. It will be null only when search.worker_threads_enabled is false - RunnableFuture task = new FutureTask<>(() -> { - search(bucketCollector, weight); - bucketCollector.postCollection(); - return null; - }); - searcher.getExecutor().execute(task); - try { - task.get(); - } catch (InterruptedException e) { - throw new ThreadInterruptedException(e); - } catch (ExecutionException e) { - if (e.getCause() instanceof RuntimeException runtimeException) { - throw runtimeException; - } - throw new RuntimeException(e.getCause()); - } + search(bucketCollector, weight); + bucketCollector.postCollection(); } private void search(BucketCollector bucketCollector, Weight weight) throws IOException { diff --git a/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java b/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java index 208ca613a350b..18de4b81cbf8c 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java +++ b/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java @@ -322,12 +322,9 @@ public T search(Query query, CollectorManager col } /** - * Similar to the lucene implementation, with the following changes made: - * 1) postCollection is performed after each segment is collected. This is needed for aggregations, performed by search worker threads - * so it can be parallelized. Also, it needs to happen in the same thread where doc_values are read, as it consumes them and Lucene - * does not allow consuming them from a different thread. - * 2) handles the ES TimeExceededException - * */ + * Same implementation as the default one in Lucene, with an additional call to postCollection in cased there are no segments. + * The rest is a plain copy from Lucene. + */ private T search(Weight weight, CollectorManager collectorManager, C firstCollector) throws IOException { LeafSlice[] leafSlices = getSlices(); if (leafSlices.length == 0) { @@ -359,14 +356,18 @@ private T search(Weight weight, CollectorManager } } + /** + * Similar to the lucene implementation, with the following changes made: + * 1) postCollection is performed after each segment is collected. This is needed for aggregations, performed by search threads + * so it can be parallelized. Also, it needs to happen in the same thread where doc_values are read, as it consumes them and Lucene + * does not allow consuming them from a different thread. + * 2) handles the ES TimeExceededException + */ @Override public void search(List leaves, Weight weight, Collector collector) throws IOException { - collector.setWeight(weight); boolean success = false; try { - for (LeafReaderContext ctx : leaves) { // search each subreader - searchLeaf(ctx, weight, collector); - } + super.search(leaves, weight, collector); success = true; } catch (@SuppressWarnings("unused") TimeExceededException e) { timeExceeded = true; @@ -410,13 +411,8 @@ private static class TimeExceededException extends RuntimeException { // This exception should never be re-thrown, but we fill in the stacktrace to be able to trace where it does not get properly caught } - /** - * Lower-level search API. - * - * {@link LeafCollector#collect(int)} is called for every matching document in - * the provided ctx. - */ - private void searchLeaf(LeafReaderContext ctx, Weight weight, Collector collector) throws IOException { + @Override + protected void searchLeaf(LeafReaderContext ctx, Weight weight, Collector collector) throws IOException { cancellable.checkCancelled(); final LeafCollector leafCollector; try { diff --git a/server/src/main/java/org/elasticsearch/threadpool/DefaultBuiltInExecutorBuilders.java b/server/src/main/java/org/elasticsearch/threadpool/DefaultBuiltInExecutorBuilders.java index 134766fbeae57..c3a24d012c013 100644 --- a/server/src/main/java/org/elasticsearch/threadpool/DefaultBuiltInExecutorBuilders.java +++ b/server/src/main/java/org/elasticsearch/threadpool/DefaultBuiltInExecutorBuilders.java @@ -72,16 +72,6 @@ public Map getBuilders(Settings settings, int allocated new EsExecutors.TaskTrackingConfig(true, searchAutoscalingEWMA) ) ); - result.put( - ThreadPool.Names.SEARCH_WORKER, - new FixedExecutorBuilder( - settings, - ThreadPool.Names.SEARCH_WORKER, - searchOrGetThreadPoolSize, - -1, - EsExecutors.TaskTrackingConfig.DEFAULT - ) - ); result.put( ThreadPool.Names.SEARCH_COORDINATION, new FixedExecutorBuilder( diff --git a/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java b/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java index b3f8f2e02fc06..9eb994896cbff 100644 --- a/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java +++ b/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java @@ -88,7 +88,6 @@ public static class Names { public static final String ANALYZE = "analyze"; public static final String WRITE = "write"; public static final String SEARCH = "search"; - public static final String SEARCH_WORKER = "search_worker"; public static final String SEARCH_COORDINATION = "search_coordination"; public static final String AUTO_COMPLETE = "auto_complete"; public static final String SEARCH_THROTTLED = "search_throttled"; @@ -158,7 +157,6 @@ public static ThreadPoolType fromType(String type) { entry(Names.ANALYZE, ThreadPoolType.FIXED), entry(Names.WRITE, ThreadPoolType.FIXED), entry(Names.SEARCH, ThreadPoolType.FIXED), - entry(Names.SEARCH_WORKER, ThreadPoolType.FIXED), entry(Names.SEARCH_COORDINATION, ThreadPoolType.FIXED), entry(Names.AUTO_COMPLETE, ThreadPoolType.FIXED), entry(Names.MANAGEMENT, ThreadPoolType.SCALING), diff --git a/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec b/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec index b99a15507f742..4e85ba2cf479f 100644 --- a/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec +++ b/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec @@ -1 +1,2 @@ org.elasticsearch.index.codec.Elasticsearch814Codec +org.elasticsearch.index.codec.Elasticsearch816Codec diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerTests.java index 2515c3e680789..65464c7f14a5c 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerTests.java @@ -13,7 +13,7 @@ import org.apache.lucene.codecs.KnnVectorsFormat; import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat; -import org.apache.lucene.codecs.lucene99.Lucene99Codec; +import org.apache.lucene.codecs.lucene912.Lucene912Codec; import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat; import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat; import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; @@ -54,7 +54,7 @@ import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.Weight; -import org.apache.lucene.search.suggest.document.Completion99PostingsFormat; +import org.apache.lucene.search.suggest.document.Completion912PostingsFormat; import org.apache.lucene.search.suggest.document.CompletionPostingsFormat; import org.apache.lucene.search.suggest.document.SuggestField; import org.apache.lucene.store.Directory; @@ -327,11 +327,11 @@ public void testTriangle() throws Exception { public void testCompletionField() throws Exception { IndexWriterConfig config = new IndexWriterConfig().setCommitOnClose(true) .setUseCompoundFile(false) - .setCodec(new Lucene99Codec(Lucene99Codec.Mode.BEST_SPEED) { + .setCodec(new Lucene912Codec(Lucene912Codec.Mode.BEST_SPEED) { @Override public PostingsFormat getPostingsFormatForField(String field) { if (field.startsWith("suggest_")) { - return new Completion99PostingsFormat(randomFrom(CompletionPostingsFormat.FSTLoadMode.values())); + return new Completion912PostingsFormat(randomFrom(CompletionPostingsFormat.FSTLoadMode.values())); } else { return super.postingsFormat(); } @@ -414,25 +414,25 @@ private static void addFieldsToDoc(Document doc, IndexableField[] fields) { enum CodecMode { BEST_SPEED { @Override - Lucene99Codec.Mode mode() { - return Lucene99Codec.Mode.BEST_SPEED; + Lucene912Codec.Mode mode() { + return Lucene912Codec.Mode.BEST_SPEED; } }, BEST_COMPRESSION { @Override - Lucene99Codec.Mode mode() { - return Lucene99Codec.Mode.BEST_COMPRESSION; + Lucene912Codec.Mode mode() { + return Lucene912Codec.Mode.BEST_COMPRESSION; } }; - abstract Lucene99Codec.Mode mode(); + abstract Lucene912Codec.Mode mode(); } static void indexRandomly(Directory directory, CodecMode codecMode, int numDocs, Consumer addFields) throws IOException { IndexWriterConfig config = new IndexWriterConfig().setCommitOnClose(true) .setUseCompoundFile(randomBoolean()) - .setCodec(new Lucene99Codec(codecMode.mode())); + .setCodec(new Lucene912Codec(codecMode.mode())); try (IndexWriter writer = new IndexWriter(directory, config)) { for (int i = 0; i < numDocs; i++) { final Document doc = new Document(); @@ -640,7 +640,7 @@ static void rewriteIndexWithPerFieldCodec(Directory source, CodecMode mode, Dire try (DirectoryReader reader = DirectoryReader.open(source)) { IndexWriterConfig config = new IndexWriterConfig().setSoftDeletesField(Lucene.SOFT_DELETES_FIELD) .setUseCompoundFile(randomBoolean()) - .setCodec(new Lucene99Codec(mode.mode()) { + .setCodec(new Lucene912Codec(mode.mode()) { @Override public PostingsFormat getPostingsFormatForField(String field) { return new ES812PostingsFormat(); diff --git a/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java b/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java index 64160c83646fa..10b0b54d2d7e2 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java @@ -52,7 +52,7 @@ public void testResolveDefaultCodecs() throws Exception { assumeTrue("Only when zstd_stored_fields feature flag is enabled", CodecService.ZSTD_STORED_FIELDS_FEATURE_FLAG.isEnabled()); CodecService codecService = createCodecService(); assertThat(codecService.codec("default"), instanceOf(PerFieldMapperCodec.class)); - assertThat(codecService.codec("default"), instanceOf(Elasticsearch814Codec.class)); + assertThat(codecService.codec("default"), instanceOf(Elasticsearch816Codec.class)); } public void testDefault() throws Exception { diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/ES813FlatVectorFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/ES813FlatVectorFormatTests.java index 5a00e90e6ffa8..aa50bc26c4443 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/vectors/ES813FlatVectorFormatTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/ES813FlatVectorFormatTests.java @@ -11,7 +11,7 @@ import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.KnnVectorsFormat; -import org.apache.lucene.codecs.lucene99.Lucene99Codec; +import org.apache.lucene.codecs.lucene912.Lucene912Codec; import org.apache.lucene.tests.index.BaseKnnVectorsFormatTestCase; import org.elasticsearch.common.logging.LogConfigurator; @@ -24,7 +24,7 @@ public class ES813FlatVectorFormatTests extends BaseKnnVectorsFormatTestCase { @Override protected Codec getCodec() { - return new Lucene99Codec() { + return new Lucene912Codec() { @Override public KnnVectorsFormat getKnnVectorsFormatForField(String field) { return new ES813FlatVectorFormat(); diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/ES813Int8FlatVectorFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/ES813Int8FlatVectorFormatTests.java index 2b70ad657ea3c..8cb927036588a 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/vectors/ES813Int8FlatVectorFormatTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/ES813Int8FlatVectorFormatTests.java @@ -11,7 +11,7 @@ import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.KnnVectorsFormat; -import org.apache.lucene.codecs.lucene99.Lucene99Codec; +import org.apache.lucene.codecs.lucene912.Lucene912Codec; import org.apache.lucene.tests.index.BaseKnnVectorsFormatTestCase; import org.elasticsearch.common.logging.LogConfigurator; @@ -24,7 +24,7 @@ public class ES813Int8FlatVectorFormatTests extends BaseKnnVectorsFormatTestCase @Override protected Codec getCodec() { - return new Lucene99Codec() { + return new Lucene912Codec() { @Override public KnnVectorsFormat getKnnVectorsFormatForField(String field) { return new ES813Int8FlatVectorFormat(); diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/ES814HnswScalarQuantizedVectorsFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/ES814HnswScalarQuantizedVectorsFormatTests.java index 8d7c5b5e4343f..cee60efb57327 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/vectors/ES814HnswScalarQuantizedVectorsFormatTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/ES814HnswScalarQuantizedVectorsFormatTests.java @@ -11,7 +11,7 @@ import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.KnnVectorsFormat; -import org.apache.lucene.codecs.lucene99.Lucene99Codec; +import org.apache.lucene.codecs.lucene912.Lucene912Codec; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.KnnFloatVectorField; @@ -41,7 +41,7 @@ public class ES814HnswScalarQuantizedVectorsFormatTests extends BaseKnnVectorsFo @Override protected Codec getCodec() { - return new Lucene99Codec() { + return new Lucene912Codec() { @Override public KnnVectorsFormat getKnnVectorsFormatForField(String field) { return new ES814HnswScalarQuantizedVectorsFormat(); diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/ES815BitFlatVectorFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/ES815BitFlatVectorFormatTests.java index bae73cc40f5d4..90d2584feb3f2 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/vectors/ES815BitFlatVectorFormatTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/ES815BitFlatVectorFormatTests.java @@ -11,7 +11,7 @@ import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.KnnVectorsFormat; -import org.apache.lucene.codecs.lucene99.Lucene99Codec; +import org.apache.lucene.codecs.lucene912.Lucene912Codec; import org.apache.lucene.index.VectorSimilarityFunction; import org.junit.Before; @@ -19,7 +19,7 @@ public class ES815BitFlatVectorFormatTests extends BaseKnnBitVectorsFormatTestCa @Override protected Codec getCodec() { - return new Lucene99Codec() { + return new Lucene912Codec() { @Override public KnnVectorsFormat getKnnVectorsFormatForField(String field) { return new ES815BitFlatVectorFormat(); diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/ES815HnswBitVectorsFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/ES815HnswBitVectorsFormatTests.java index 2561d17965bc4..add90ea271fa1 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/vectors/ES815HnswBitVectorsFormatTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/ES815HnswBitVectorsFormatTests.java @@ -11,7 +11,7 @@ import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.KnnVectorsFormat; -import org.apache.lucene.codecs.lucene99.Lucene99Codec; +import org.apache.lucene.codecs.lucene912.Lucene912Codec; import org.apache.lucene.index.VectorSimilarityFunction; import org.junit.Before; @@ -19,7 +19,7 @@ public class ES815HnswBitVectorsFormatTests extends BaseKnnBitVectorsFormatTestC @Override protected Codec getCodec() { - return new Lucene99Codec() { + return new Lucene912Codec() { @Override public KnnVectorsFormat getKnnVectorsFormatForField(String field) { return new ES815HnswBitVectorsFormat(); diff --git a/server/src/test/java/org/elasticsearch/index/codec/zstd/StoredFieldCodecDuelTests.java b/server/src/test/java/org/elasticsearch/index/codec/zstd/StoredFieldCodecDuelTests.java index a56d5f1c8084b..c3fea6c7a189b 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/zstd/StoredFieldCodecDuelTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/zstd/StoredFieldCodecDuelTests.java @@ -10,7 +10,7 @@ package org.elasticsearch.index.codec.zstd; import org.apache.lucene.codecs.Codec; -import org.apache.lucene.codecs.lucene99.Lucene99Codec; +import org.apache.lucene.codecs.lucene912.Lucene912Codec; import org.apache.lucene.document.Document; import org.apache.lucene.document.StoredField; import org.apache.lucene.index.DirectoryReader; @@ -35,13 +35,13 @@ public class StoredFieldCodecDuelTests extends ESTestCase { private static final String DOUBLE_FIELD = "double_field_5"; public void testDuelBestSpeed() throws IOException { - var baseline = new LegacyPerFieldMapperCodec(Lucene99Codec.Mode.BEST_SPEED, null, BigArrays.NON_RECYCLING_INSTANCE); + var baseline = new LegacyPerFieldMapperCodec(Lucene912Codec.Mode.BEST_SPEED, null, BigArrays.NON_RECYCLING_INSTANCE); var contender = new PerFieldMapperCodec(Zstd814StoredFieldsFormat.Mode.BEST_SPEED, null, BigArrays.NON_RECYCLING_INSTANCE); doTestDuel(baseline, contender); } public void testDuelBestCompression() throws IOException { - var baseline = new LegacyPerFieldMapperCodec(Lucene99Codec.Mode.BEST_COMPRESSION, null, BigArrays.NON_RECYCLING_INSTANCE); + var baseline = new LegacyPerFieldMapperCodec(Lucene912Codec.Mode.BEST_COMPRESSION, null, BigArrays.NON_RECYCLING_INSTANCE); var contender = new PerFieldMapperCodec(Zstd814StoredFieldsFormat.Mode.BEST_COMPRESSION, null, BigArrays.NON_RECYCLING_INSTANCE); doTestDuel(baseline, contender); } diff --git a/server/src/test/java/org/elasticsearch/index/codec/zstd/Zstd814BestCompressionStoredFieldsFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/zstd/Zstd814BestCompressionStoredFieldsFormatTests.java index 211c564650317..71c7464657e72 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/zstd/Zstd814BestCompressionStoredFieldsFormatTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/zstd/Zstd814BestCompressionStoredFieldsFormatTests.java @@ -11,11 +11,11 @@ import org.apache.lucene.codecs.Codec; import org.apache.lucene.tests.index.BaseStoredFieldsFormatTestCase; -import org.elasticsearch.index.codec.Elasticsearch814Codec; +import org.elasticsearch.index.codec.Elasticsearch816Codec; public class Zstd814BestCompressionStoredFieldsFormatTests extends BaseStoredFieldsFormatTestCase { - private final Codec codec = new Elasticsearch814Codec(Zstd814StoredFieldsFormat.Mode.BEST_COMPRESSION); + private final Codec codec = new Elasticsearch816Codec(Zstd814StoredFieldsFormat.Mode.BEST_COMPRESSION); @Override protected Codec getCodec() { diff --git a/server/src/test/java/org/elasticsearch/index/codec/zstd/Zstd814BestSpeedStoredFieldsFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/zstd/Zstd814BestSpeedStoredFieldsFormatTests.java index 077569d150daa..02a1b10697907 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/zstd/Zstd814BestSpeedStoredFieldsFormatTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/zstd/Zstd814BestSpeedStoredFieldsFormatTests.java @@ -11,11 +11,11 @@ import org.apache.lucene.codecs.Codec; import org.apache.lucene.tests.index.BaseStoredFieldsFormatTestCase; -import org.elasticsearch.index.codec.Elasticsearch814Codec; +import org.elasticsearch.index.codec.Elasticsearch816Codec; public class Zstd814BestSpeedStoredFieldsFormatTests extends BaseStoredFieldsFormatTestCase { - private final Codec codec = new Elasticsearch814Codec(Zstd814StoredFieldsFormat.Mode.BEST_SPEED); + private final Codec codec = new Elasticsearch816Codec(Zstd814StoredFieldsFormat.Mode.BEST_SPEED); @Override protected Codec getCodec() { diff --git a/server/src/test/java/org/elasticsearch/index/engine/CompletionStatsCacheTests.java b/server/src/test/java/org/elasticsearch/index/engine/CompletionStatsCacheTests.java index 9837eba25f5b7..6565a11a860ec 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/CompletionStatsCacheTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/CompletionStatsCacheTests.java @@ -9,12 +9,12 @@ package org.elasticsearch.index.engine; import org.apache.lucene.codecs.PostingsFormat; -import org.apache.lucene.codecs.lucene99.Lucene99Codec; +import org.apache.lucene.codecs.lucene912.Lucene912Codec; import org.apache.lucene.document.Document; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.search.suggest.document.Completion99PostingsFormat; +import org.apache.lucene.search.suggest.document.Completion912PostingsFormat; import org.apache.lucene.search.suggest.document.SuggestField; import org.apache.lucene.store.Directory; import org.elasticsearch.ElasticsearchException; @@ -44,8 +44,8 @@ public void testExceptionsAreNotCached() { public void testCompletionStatsCache() throws IOException, InterruptedException { final IndexWriterConfig indexWriterConfig = newIndexWriterConfig(); - final PostingsFormat postingsFormat = new Completion99PostingsFormat(); - indexWriterConfig.setCodec(new Lucene99Codec() { + final PostingsFormat postingsFormat = new Completion912PostingsFormat(); + indexWriterConfig.setCodec(new Lucene912Codec() { @Override public PostingsFormat getPostingsFormatForField(String field) { return postingsFormat; // all fields are suggest fields diff --git a/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java index f9f4cba7848a5..134d21ba475b7 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java @@ -16,7 +16,7 @@ import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; import org.apache.lucene.search.Query; -import org.apache.lucene.search.suggest.document.Completion99PostingsFormat; +import org.apache.lucene.search.suggest.document.Completion912PostingsFormat; import org.apache.lucene.search.suggest.document.CompletionAnalyzer; import org.apache.lucene.search.suggest.document.ContextSuggestField; import org.apache.lucene.search.suggest.document.FuzzyCompletionQuery; @@ -151,7 +151,7 @@ public void testPostingsFormat() throws IOException { Codec codec = codecService.codec("default"); if (CodecService.ZSTD_STORED_FIELDS_FEATURE_FLAG.isEnabled()) { assertThat(codec, instanceOf(PerFieldMapperCodec.class)); - assertThat(((PerFieldMapperCodec) codec).getPostingsFormatForField("field"), instanceOf(Completion99PostingsFormat.class)); + assertThat(((PerFieldMapperCodec) codec).getPostingsFormatForField("field"), instanceOf(Completion912PostingsFormat.class)); } else { if (codec instanceof CodecService.DeduplicateFieldInfosCodec deduplicateFieldInfosCodec) { codec = deduplicateFieldInfosCodec.delegate(); @@ -159,7 +159,7 @@ public void testPostingsFormat() throws IOException { assertThat(codec, instanceOf(LegacyPerFieldMapperCodec.class)); assertThat( ((LegacyPerFieldMapperCodec) codec).getPostingsFormatForField("field"), - instanceOf(Completion99PostingsFormat.class) + instanceOf(Completion912PostingsFormat.class) ); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ConstantScoreTextFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ConstantScoreTextFieldTypeTests.java index 2627ae9a39839..e454a4ffa0c8d 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ConstantScoreTextFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ConstantScoreTextFieldTypeTests.java @@ -16,6 +16,7 @@ import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.ConstantScoreQuery; import org.apache.lucene.search.FuzzyQuery; +import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.PrefixQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.RegexpQuery; @@ -231,20 +232,26 @@ public void testTermIntervals() throws IOException { public void testPrefixIntervals() throws IOException { MappedFieldType ft = createFieldType(); IntervalsSource prefixIntervals = ft.prefixIntervals(new BytesRef("foo"), MOCK_CONTEXT); - assertEquals(Intervals.prefix(new BytesRef("foo")), prefixIntervals); + assertEquals(Intervals.prefix(new BytesRef("foo"), IndexSearcher.getMaxClauseCount()), prefixIntervals); } public void testWildcardIntervals() throws IOException { MappedFieldType ft = createFieldType(); IntervalsSource wildcardIntervals = ft.wildcardIntervals(new BytesRef("foo"), MOCK_CONTEXT); - assertEquals(Intervals.wildcard(new BytesRef("foo")), wildcardIntervals); + assertEquals(Intervals.wildcard(new BytesRef("foo"), IndexSearcher.getMaxClauseCount()), wildcardIntervals); + } + + public void testRegexpIntervals() { + MappedFieldType ft = createFieldType(); + IntervalsSource regexpIntervals = ft.regexpIntervals(new BytesRef("foo"), MOCK_CONTEXT); + assertEquals(Intervals.regexp(new BytesRef("foo"), IndexSearcher.getMaxClauseCount()), regexpIntervals); } public void testFuzzyIntervals() throws IOException { MappedFieldType ft = createFieldType(); IntervalsSource fuzzyIntervals = ft.fuzzyIntervals("foo", 1, 2, true, MOCK_CONTEXT); FuzzyQuery fq = new FuzzyQuery(new Term("field", "foo"), 1, 2, 128, true); - IntervalsSource expectedIntervals = Intervals.multiterm(fq.getAutomata(), "foo"); + IntervalsSource expectedIntervals = Intervals.multiterm(fq.getAutomata(), IndexSearcher.getMaxClauseCount(), "foo"); assertEquals(expectedIntervals, fuzzyIntervals); } @@ -259,6 +266,15 @@ public void testWildcardIntervalsWithIndexedPrefixes() { ConstantScoreTextFieldType ft = createFieldType(); ft.setIndexPrefixes(1, 4); IntervalsSource wildcardIntervals = ft.wildcardIntervals(new BytesRef("foo"), MOCK_CONTEXT); - assertEquals(Intervals.wildcard(new BytesRef("foo")), wildcardIntervals); + assertEquals(Intervals.wildcard(new BytesRef("foo"), IndexSearcher.getMaxClauseCount()), wildcardIntervals); + } + + public void testRangeIntervals() { + MappedFieldType ft = createFieldType(); + IntervalsSource rangeIntervals = ft.rangeIntervals(new BytesRef("foo"), new BytesRef("foo1"), true, true, MOCK_CONTEXT); + assertEquals( + Intervals.range(new BytesRef("foo"), new BytesRef("foo1"), true, true, IndexSearcher.getMaxClauseCount()), + rangeIntervals + ); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java index d73e8546a726a..4d246d3c557a6 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java @@ -16,6 +16,7 @@ import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.ConstantScoreQuery; import org.apache.lucene.search.FuzzyQuery; +import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.PrefixQuery; import org.apache.lucene.search.Query; @@ -243,20 +244,26 @@ public void testTermIntervals() throws IOException { public void testPrefixIntervals() throws IOException { MappedFieldType ft = createFieldType(); IntervalsSource prefixIntervals = ft.prefixIntervals(new BytesRef("foo"), MOCK_CONTEXT); - assertEquals(Intervals.prefix(new BytesRef("foo")), prefixIntervals); + assertEquals(Intervals.prefix(new BytesRef("foo"), IndexSearcher.getMaxClauseCount()), prefixIntervals); } - public void testWildcardIntervals() throws IOException { + public void testWildcardIntervals() { MappedFieldType ft = createFieldType(); IntervalsSource wildcardIntervals = ft.wildcardIntervals(new BytesRef("foo"), MOCK_CONTEXT); - assertEquals(Intervals.wildcard(new BytesRef("foo")), wildcardIntervals); + assertEquals(Intervals.wildcard(new BytesRef("foo"), IndexSearcher.getMaxClauseCount()), wildcardIntervals); } - public void testFuzzyIntervals() throws IOException { + public void testRegexpIntervals() { + MappedFieldType ft = createFieldType(); + IntervalsSource regexpIntervals = ft.regexpIntervals(new BytesRef("foo"), MOCK_CONTEXT); + assertEquals(Intervals.regexp(new BytesRef("foo"), IndexSearcher.getMaxClauseCount()), regexpIntervals); + } + + public void testFuzzyIntervals() { MappedFieldType ft = createFieldType(); IntervalsSource fuzzyIntervals = ft.fuzzyIntervals("foo", 1, 2, true, MOCK_CONTEXT); FuzzyQuery fq = new FuzzyQuery(new Term("field", "foo"), 1, 2, 128, true); - IntervalsSource expectedIntervals = Intervals.multiterm(fq.getAutomata(), "foo"); + IntervalsSource expectedIntervals = Intervals.multiterm(fq.getAutomata(), IndexSearcher.getMaxClauseCount(), "foo"); assertEquals(expectedIntervals, fuzzyIntervals); } @@ -271,6 +278,15 @@ public void testWildcardIntervalsWithIndexedPrefixes() { TextFieldType ft = createFieldType(); ft.setIndexPrefixes(1, 4); IntervalsSource wildcardIntervals = ft.wildcardIntervals(new BytesRef("foo"), MOCK_CONTEXT); - assertEquals(Intervals.wildcard(new BytesRef("foo")), wildcardIntervals); + assertEquals(Intervals.wildcard(new BytesRef("foo"), IndexSearcher.getMaxClauseCount()), wildcardIntervals); + } + + public void testRangeIntervals() { + MappedFieldType ft = createFieldType(); + IntervalsSource rangeIntervals = ft.rangeIntervals(new BytesRef("foo"), new BytesRef("foo1"), true, true, MOCK_CONTEXT); + assertEquals( + Intervals.range(new BytesRef("foo"), new BytesRef("foo1"), true, true, IndexSearcher.getMaxClauseCount()), + rangeIntervals + ); } } diff --git a/server/src/test/java/org/elasticsearch/index/query/IntervalQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/IntervalQueryBuilderTests.java index 6d78f5fffd4b1..aad8275f4749d 100644 --- a/server/src/test/java/org/elasticsearch/index/query/IntervalQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/IntervalQueryBuilderTests.java @@ -9,14 +9,22 @@ package org.elasticsearch.index.query; +import org.apache.lucene.analysis.core.KeywordAnalyzer; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.TextField; +import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; import org.apache.lucene.queries.intervals.IntervalQuery; import org.apache.lucene.queries.intervals.Intervals; import org.apache.lucene.queries.intervals.IntervalsSource; import org.apache.lucene.search.BoostQuery; import org.apache.lucene.search.FuzzyQuery; +import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; +import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.index.RandomIndexWriter; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.Strings; @@ -34,7 +42,9 @@ import java.util.Collections; import java.util.List; +import static java.util.Collections.singleton; import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; +import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; @@ -606,7 +616,7 @@ public void testPrefixes() throws IOException { } }""", TEXT_FIELD_NAME); IntervalQueryBuilder builder = (IntervalQueryBuilder) parseQuery(json); - Query expected = new IntervalQuery(TEXT_FIELD_NAME, Intervals.prefix(new BytesRef("term"))); + Query expected = new IntervalQuery(TEXT_FIELD_NAME, Intervals.prefix(new BytesRef("term"), IndexSearcher.getMaxClauseCount())); assertEquals(expected, builder.toQuery(createSearchExecutionContext())); String no_positions_json = Strings.format(""" @@ -667,7 +677,13 @@ public void testPrefixes() throws IOException { builder = (IntervalQueryBuilder) parseQuery(short_prefix_json); expected = new IntervalQuery( PREFIXED_FIELD, - Intervals.or(Intervals.fixField(PREFIXED_FIELD + "._index_prefix", Intervals.wildcard(new BytesRef("t?"))), Intervals.term("t")) + Intervals.or( + Intervals.fixField( + PREFIXED_FIELD + "._index_prefix", + Intervals.wildcard(new BytesRef("t?"), IndexSearcher.getMaxClauseCount()) + ), + Intervals.term("t") + ) ); assertEquals(expected, builder.toQuery(createSearchExecutionContext())); @@ -726,8 +742,109 @@ public void testPrefixes() throws IOException { assertEquals(expected, builder.toQuery(createSearchExecutionContext())); } - public void testWildcard() throws IOException { + public void testRegexp() throws IOException { + String json = Strings.format(""" + { + "intervals": { + "%s": { + "regexp": { + "pattern": "Te.*m" + } + } + } + }""", TEXT_FIELD_NAME); + + IntervalQueryBuilder builder = (IntervalQueryBuilder) parseQuery(json); + Query expected = new IntervalQuery(TEXT_FIELD_NAME, Intervals.regexp(new BytesRef("te.*m"), IndexSearcher.getMaxClauseCount())); + assertEquals(expected, builder.toQuery(createSearchExecutionContext())); + + String no_positions_json = Strings.format(""" + { + "intervals": { + "%s": { + "regexp": { + "pattern": "Te.*m" + } + } + } + } + """, NO_POSITIONS_FIELD); + expectThrows(IllegalArgumentException.class, () -> { + IntervalQueryBuilder builder1 = (IntervalQueryBuilder) parseQuery(no_positions_json); + builder1.toQuery(createSearchExecutionContext()); + }); + + String fixed_field_json = Strings.format(""" + { + "intervals": { + "%s": { + "regexp": { + "pattern": "Te.*m", + "use_field": "masked_field" + } + } + } + }""", TEXT_FIELD_NAME); + + builder = (IntervalQueryBuilder) parseQuery(fixed_field_json); + expected = new IntervalQuery( + TEXT_FIELD_NAME, + Intervals.fixField(MASKED_FIELD, Intervals.regexp(new BytesRef("te.*m"), IndexSearcher.getMaxClauseCount())) + ); + assertEquals(expected, builder.toQuery(createSearchExecutionContext())); + String fixed_field_json_no_positions = Strings.format(""" + { + "intervals": { + "%s": { + "regexp": { + "pattern": "Te.*m", + "use_field": "%s" + } + } + } + }""", TEXT_FIELD_NAME, NO_POSITIONS_FIELD); + expectThrows(IllegalArgumentException.class, () -> { + IntervalQueryBuilder builder1 = (IntervalQueryBuilder) parseQuery(fixed_field_json_no_positions); + builder1.toQuery(createSearchExecutionContext()); + }); + } + + public void testMaxExpansionExceptionFailure() throws Exception { + IntervalsSourceProvider provider1 = new IntervalsSourceProvider.Prefix("bar", "keyword", null); + IntervalsSourceProvider provider2 = new IntervalsSourceProvider.Wildcard("bar*", "keyword", null); + IntervalsSourceProvider provider3 = new IntervalsSourceProvider.Fuzzy("bar", 0, true, Fuzziness.fromEdits(1), "keyword", null); + IntervalsSourceProvider provider4 = new IntervalsSourceProvider.Regexp("bar.*", "keyword", null); + IntervalsSourceProvider provider5 = new IntervalsSourceProvider.Range("bar", "bar2", true, true, "keyword", null); + IntervalsSourceProvider provider = randomFrom(provider1, provider2, provider3, provider4, provider5); + + try (Directory directory = newDirectory()) { + try (RandomIndexWriter iw = new RandomIndexWriter(random(), directory, new KeywordAnalyzer())) { + for (int i = 0; i < 3; i++) { + iw.addDocument(singleton(new TextField(TEXT_FIELD_NAME, "bar" + i, Field.Store.NO))); + } + try (IndexReader reader = iw.getReader()) { + int origBoolMaxClauseCount = IndexSearcher.getMaxClauseCount(); + IndexSearcher.setMaxClauseCount(1); + try { + + IntervalQueryBuilder queryBuilder = new IntervalQueryBuilder(TEXT_FIELD_NAME, provider); + IndexSearcher searcher = newSearcher(reader); + Query query = queryBuilder.toQuery(createSearchExecutionContext(searcher)); + RuntimeException exc = expectThrows( + RuntimeException.class, + () -> query.createWeight(searcher, ScoreMode.COMPLETE, 1.0f).scorer(searcher.getLeafContexts().get(0)) + ); + assertThat(exc.getMessage(), containsString("expanded to too many terms (limit 1)")); + } finally { + IndexSearcher.setMaxClauseCount(origBoolMaxClauseCount); + } + } + } + } + } + + public void testWildcard() throws IOException { String json = Strings.format(""" { "intervals": { @@ -740,7 +857,7 @@ public void testWildcard() throws IOException { }""", TEXT_FIELD_NAME); IntervalQueryBuilder builder = (IntervalQueryBuilder) parseQuery(json); - Query expected = new IntervalQuery(TEXT_FIELD_NAME, Intervals.wildcard(new BytesRef("te?m"))); + Query expected = new IntervalQuery(TEXT_FIELD_NAME, Intervals.wildcard(new BytesRef("te?m"), IndexSearcher.getMaxClauseCount())); assertEquals(expected, builder.toQuery(createSearchExecutionContext())); String no_positions_json = Strings.format(""" @@ -772,7 +889,7 @@ public void testWildcard() throws IOException { }""", TEXT_FIELD_NAME); builder = (IntervalQueryBuilder) parseQuery(keyword_json); - expected = new IntervalQuery(TEXT_FIELD_NAME, Intervals.wildcard(new BytesRef("Te?m"))); + expected = new IntervalQuery(TEXT_FIELD_NAME, Intervals.wildcard(new BytesRef("Te?m"), IndexSearcher.getMaxClauseCount())); assertEquals(expected, builder.toQuery(createSearchExecutionContext())); String fixed_field_json = Strings.format(""" @@ -788,7 +905,10 @@ public void testWildcard() throws IOException { }""", TEXT_FIELD_NAME); builder = (IntervalQueryBuilder) parseQuery(fixed_field_json); - expected = new IntervalQuery(TEXT_FIELD_NAME, Intervals.fixField(MASKED_FIELD, Intervals.wildcard(new BytesRef("te?m")))); + expected = new IntervalQuery( + TEXT_FIELD_NAME, + Intervals.fixField(MASKED_FIELD, Intervals.wildcard(new BytesRef("te?m"), IndexSearcher.getMaxClauseCount())) + ); assertEquals(expected, builder.toQuery(createSearchExecutionContext())); String fixed_field_json_no_positions = Strings.format(""" @@ -821,13 +941,22 @@ public void testWildcard() throws IOException { }""", TEXT_FIELD_NAME); builder = (IntervalQueryBuilder) parseQuery(fixed_field_analyzer_json); - expected = new IntervalQuery(TEXT_FIELD_NAME, Intervals.fixField(MASKED_FIELD, Intervals.wildcard(new BytesRef("Te?m")))); + expected = new IntervalQuery( + TEXT_FIELD_NAME, + Intervals.fixField(MASKED_FIELD, Intervals.wildcard(new BytesRef("Te?m"), IndexSearcher.getMaxClauseCount())) + ); assertEquals(expected, builder.toQuery(createSearchExecutionContext())); } private static IntervalsSource buildFuzzySource(String term, String label, int prefixLength, boolean transpositions, int editDistance) { - FuzzyQuery fq = new FuzzyQuery(new Term("field", term), editDistance, prefixLength, 128, transpositions); - return Intervals.multiterm(fq.getAutomata(), label); + FuzzyQuery fq = new FuzzyQuery( + new Term("field", term), + editDistance, + prefixLength, + IndexSearcher.getMaxClauseCount(), + transpositions + ); + return Intervals.multiterm(fq.getAutomata(), IndexSearcher.getMaxClauseCount(), label); } public void testFuzzy() throws IOException { @@ -932,7 +1061,77 @@ public void testFuzzy() throws IOException { Intervals.fixField(MASKED_FIELD, buildFuzzySource("term", "term", 2, true, Fuzziness.ONE.asDistance("term"))) ); assertEquals(expected, builder.toQuery(createSearchExecutionContext())); - } + public void testRange() throws IOException { + String json = Strings.format(""" + { + "intervals": { + "%s": { + "range": { + "gte": "aaa", + "lte": "aab" + } + } + } + }""", TEXT_FIELD_NAME); + IntervalQueryBuilder builder = (IntervalQueryBuilder) parseQuery(json); + Query expected = new IntervalQuery( + TEXT_FIELD_NAME, + Intervals.range(new BytesRef("aaa"), new BytesRef("aab"), true, true, IndexSearcher.getMaxClauseCount()) + ); + assertEquals(expected, builder.toQuery(createSearchExecutionContext())); + + json = Strings.format(""" + { + "intervals": { + "%s": { + "range": { + "gt": "aaa", + "lt": "aab" + } + } + } + }""", TEXT_FIELD_NAME); + builder = (IntervalQueryBuilder) parseQuery(json); + expected = new IntervalQuery( + TEXT_FIELD_NAME, + Intervals.range(new BytesRef("aaa"), new BytesRef("aab"), false, false, IndexSearcher.getMaxClauseCount()) + ); + assertEquals(expected, builder.toQuery(createSearchExecutionContext())); + + String incomplete_range = Strings.format(""" + { + "intervals": { + "%s": { + "range": { + "gt": "aaa" + } + } + } + } + """, TEXT_FIELD_NAME); + IllegalArgumentException exc = expectThrows(IllegalArgumentException.class, () -> { + IntervalQueryBuilder builder1 = (IntervalQueryBuilder) parseQuery(incomplete_range); + builder1.toQuery(createSearchExecutionContext()); + }); + assertEquals("Either [lte] or [lt], one of them must be provided", exc.getCause().getMessage()); + + String incomplete_range2 = Strings.format(""" + { + "intervals": { + "%s": { + "range": { + "lt": "aaa" + } + } + } + } + """, TEXT_FIELD_NAME); + exc = expectThrows(IllegalArgumentException.class, () -> { + IntervalQueryBuilder builder1 = (IntervalQueryBuilder) parseQuery(incomplete_range2); + builder1.toQuery(createSearchExecutionContext()); + }); + assertEquals("Either [gte] or [gt], one of them must be provided", exc.getCause().getMessage()); + } } diff --git a/server/src/test/java/org/elasticsearch/index/query/RangeIntervalsSourceProviderTests.java b/server/src/test/java/org/elasticsearch/index/query/RangeIntervalsSourceProviderTests.java new file mode 100644 index 0000000000000..e170faf8043be --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/query/RangeIntervalsSourceProviderTests.java @@ -0,0 +1,71 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.query; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractXContentSerializingTestCase; +import org.elasticsearch.xcontent.XContentParser; + +import java.io.IOException; + +import static org.elasticsearch.index.query.IntervalsSourceProvider.Range; + +public class RangeIntervalsSourceProviderTests extends AbstractXContentSerializingTestCase { + + @Override + protected Range createTestInstance() { + return createRandomRange(); + } + + static Range createRandomRange() { + return new Range( + "a" + randomAlphaOfLengthBetween(1, 10), + "z" + randomAlphaOfLengthBetween(1, 10), + randomBoolean(), + randomBoolean(), + randomBoolean() ? randomAlphaOfLength(10) : null, + randomBoolean() ? randomAlphaOfLength(10) : null + ); + } + + @Override + protected Range mutateInstance(Range instance) { + String lowerTerm = instance.getLowerTerm(); + String upperTerm = instance.getUpperTerm(); + boolean includeLower = instance.getIncludeLower(); + boolean includeUpper = instance.getIncludeUpper(); + String analyzer = instance.getAnalyzer(); + String useField = instance.getUseField(); + switch (between(0, 5)) { + case 0 -> lowerTerm = "a" + lowerTerm; + case 1 -> upperTerm = "z" + upperTerm; + case 2 -> includeLower = includeLower == false; + case 3 -> includeUpper = includeUpper == false; + case 4 -> analyzer = randomAlphaOfLength(5); + case 5 -> useField = useField == null ? randomAlphaOfLength(5) : null; + } + return new Range(lowerTerm, upperTerm, includeLower, includeUpper, analyzer, useField); + } + + @Override + protected Writeable.Reader instanceReader() { + return Range::new; + } + + @Override + protected Range doParseInstance(XContentParser parser) throws IOException { + if (parser.nextToken() == XContentParser.Token.START_OBJECT) { + parser.nextToken(); + } + Range range = (Range) IntervalsSourceProvider.fromXContent(parser); + assertEquals(XContentParser.Token.END_OBJECT, parser.nextToken()); + return range; + } +} diff --git a/server/src/test/java/org/elasticsearch/index/query/RegexpIntervalsSourceProviderTests.java b/server/src/test/java/org/elasticsearch/index/query/RegexpIntervalsSourceProviderTests.java new file mode 100644 index 0000000000000..ace7350d8d796 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/query/RegexpIntervalsSourceProviderTests.java @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.query; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractXContentSerializingTestCase; +import org.elasticsearch.xcontent.XContentParser; + +import java.io.IOException; + +import static org.elasticsearch.index.query.IntervalsSourceProvider.Regexp; + +public class RegexpIntervalsSourceProviderTests extends AbstractXContentSerializingTestCase { + + @Override + protected Regexp createTestInstance() { + return createRandomRegexp(); + } + + static Regexp createRandomRegexp() { + return new Regexp( + randomAlphaOfLengthBetween(1, 10), + randomBoolean() ? randomAlphaOfLength(10) : null, + randomBoolean() ? randomAlphaOfLength(10) : null + ); + } + + @Override + protected Regexp mutateInstance(Regexp instance) { + String regexp = instance.getPattern(); + String analyzer = instance.getAnalyzer(); + String useField = instance.getUseField(); + switch (between(0, 2)) { + case 0 -> regexp += "a"; + case 1 -> analyzer = randomAlphaOfLength(5); + case 2 -> useField = useField == null ? randomAlphaOfLength(5) : null; + } + return new Regexp(regexp, analyzer, useField); + } + + @Override + protected Writeable.Reader instanceReader() { + return Regexp::new; + } + + @Override + protected Regexp doParseInstance(XContentParser parser) throws IOException { + if (parser.nextToken() == XContentParser.Token.START_OBJECT) { + parser.nextToken(); + } + Regexp regexp = (Regexp) IntervalsSourceProvider.fromXContent(parser); + assertEquals(XContentParser.Token.END_OBJECT, parser.nextToken()); + return regexp; + } +} diff --git a/server/src/test/java/org/elasticsearch/index/store/StoreTests.java b/server/src/test/java/org/elasticsearch/index/store/StoreTests.java index 0b936384bf343..36ece00ccc0ca 100644 --- a/server/src/test/java/org/elasticsearch/index/store/StoreTests.java +++ b/server/src/test/java/org/elasticsearch/index/store/StoreTests.java @@ -274,7 +274,7 @@ public IndexInput openInput(String name, IOContext context) throws IOException { metadata = store.getMetadata(randomBoolean() ? indexCommit : null); assertThat(metadata.fileMetadataMap().isEmpty(), is(false)); for (StoreFileMetadata meta : metadata) { - try (IndexInput input = store.directory().openInput(meta.name(), IOContext.DEFAULT)) { + try (IndexInput input = store.directory().openInput(meta.name(), IOContext.READONCE)) { String checksum = Store.digestToString(CodecUtil.retrieveChecksum(input)); assertThat("File: " + meta.name() + " has a different checksum", meta.checksum(), equalTo(checksum)); assertThat(meta.writtenBy(), equalTo(Version.LATEST.toString())); diff --git a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java index 2cf45e463346b..642804730a144 100644 --- a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java +++ b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java @@ -2742,13 +2742,13 @@ public void testEnableSearchWorkerThreads() throws IOException { } /** - * Verify that a single slice is created for requests that don't support parallel collection, while computation - * is still offloaded to the worker threads. Also ensure multiple slices are created for requests that do support + * Verify that a single slice is created for requests that don't support parallel collection, while an executor is still + * provided to the searcher to parallelize other operations. Also ensure multiple slices are created for requests that do support * parallel collection. */ public void testSlicingBehaviourForParallelCollection() throws Exception { IndexService indexService = createIndex("index", Settings.EMPTY); - ThreadPoolExecutor executor = (ThreadPoolExecutor) indexService.getThreadPool().executor(ThreadPool.Names.SEARCH_WORKER); + ThreadPoolExecutor executor = (ThreadPoolExecutor) indexService.getThreadPool().executor(ThreadPool.Names.SEARCH); final int configuredMaxPoolSize = 10; executor.setMaximumPoolSize(configuredMaxPoolSize); // We set this explicitly to be independent of CPU cores. int numDocs = randomIntBetween(50, 100); @@ -2799,7 +2799,7 @@ public void testSlicingBehaviourForParallelCollection() throws Exception { assertBusy( () -> assertEquals( "DFS supports parallel collection, so the number of slices should be > 1.", - expectedSlices, + expectedSlices - 1, // one slice executes on the calling thread executor.getCompletedTaskCount() - priorExecutorTaskCount ) ); @@ -2829,7 +2829,7 @@ public void testSlicingBehaviourForParallelCollection() throws Exception { assertBusy( () -> assertEquals( "QUERY supports parallel collection when enabled, so the number of slices should be > 1.", - expectedSlices, + expectedSlices - 1, // one slice executes on the calling thread executor.getCompletedTaskCount() - priorExecutorTaskCount ) ); @@ -2838,13 +2838,14 @@ public void testSlicingBehaviourForParallelCollection() throws Exception { { try (SearchContext searchContext = service.createContext(readerContext, request, task, ResultsType.FETCH, true)) { ContextIndexSearcher searcher = searchContext.searcher(); - assertNotNull(searcher.getExecutor()); + assertNull(searcher.getExecutor()); final long priorExecutorTaskCount = executor.getCompletedTaskCount(); searcher.search(termQuery, new TotalHitCountCollectorManager()); assertBusy( () -> assertEquals( - "The number of slices should be 1 as FETCH does not support parallel collection.", - 1, + "The number of slices should be 1 as FETCH does not support parallel collection and thus runs on the calling" + + " thread.", + 0, executor.getCompletedTaskCount() - priorExecutorTaskCount ) ); @@ -2853,13 +2854,13 @@ public void testSlicingBehaviourForParallelCollection() throws Exception { { try (SearchContext searchContext = service.createContext(readerContext, request, task, ResultsType.NONE, true)) { ContextIndexSearcher searcher = searchContext.searcher(); - assertNotNull(searcher.getExecutor()); + assertNull(searcher.getExecutor()); final long priorExecutorTaskCount = executor.getCompletedTaskCount(); searcher.search(termQuery, new TotalHitCountCollectorManager()); assertBusy( () -> assertEquals( "The number of slices should be 1 as NONE does not support parallel collection.", - 1, + 0, // zero since one slice executes on the calling thread executor.getCompletedTaskCount() - priorExecutorTaskCount ) ); @@ -2876,13 +2877,13 @@ public void testSlicingBehaviourForParallelCollection() throws Exception { { try (SearchContext searchContext = service.createContext(readerContext, request, task, ResultsType.QUERY, true)) { ContextIndexSearcher searcher = searchContext.searcher(); - assertNotNull(searcher.getExecutor()); + assertNull(searcher.getExecutor()); final long priorExecutorTaskCount = executor.getCompletedTaskCount(); searcher.search(termQuery, new TotalHitCountCollectorManager()); assertBusy( () -> assertEquals( "The number of slices should be 1 when QUERY parallel collection is disabled.", - 1, + 0, // zero since one slice executes on the calling thread executor.getCompletedTaskCount() - priorExecutorTaskCount ) ); @@ -2919,7 +2920,7 @@ public void testSlicingBehaviourForParallelCollection() throws Exception { assertBusy( () -> assertEquals( "QUERY supports parallel collection when enabled, so the number of slices should be > 1.", - expectedSlices, + expectedSlices - 1, // one slice executes on the calling thread executor.getCompletedTaskCount() - priorExecutorTaskCount ) ); diff --git a/server/src/test/java/org/elasticsearch/search/dfs/DfsPhaseTests.java b/server/src/test/java/org/elasticsearch/search/dfs/DfsPhaseTests.java index 5cf40309f9bc0..0abf34d800dca 100644 --- a/server/src/test/java/org/elasticsearch/search/dfs/DfsPhaseTests.java +++ b/server/src/test/java/org/elasticsearch/search/dfs/DfsPhaseTests.java @@ -40,7 +40,7 @@ public class DfsPhaseTests extends ESTestCase { @Before public final void init() { threadPool = new TestThreadPool(DfsPhaseTests.class.getName()); - threadPoolExecutor = (ThreadPoolExecutor) threadPool.executor(ThreadPool.Names.SEARCH_WORKER); + threadPoolExecutor = (ThreadPoolExecutor) threadPool.executor(ThreadPool.Names.SEARCH); } @After diff --git a/server/src/test/java/org/elasticsearch/search/internal/ContextIndexSearcherTests.java b/server/src/test/java/org/elasticsearch/search/internal/ContextIndexSearcherTests.java index 14e81c0414865..34ee0eec101b6 100644 --- a/server/src/test/java/org/elasticsearch/search/internal/ContextIndexSearcherTests.java +++ b/server/src/test/java/org/elasticsearch/search/internal/ContextIndexSearcherTests.java @@ -224,7 +224,8 @@ public void testConcurrentRewrite() throws Exception { int numSegments = directoryReader.getContext().leaves().size(); KnnFloatVectorQuery vectorQuery = new KnnFloatVectorQuery("float_vector", new float[] { 0, 0, 0 }, 10, null); vectorQuery.rewrite(searcher); - assertBusy(() -> assertEquals(numSegments, executor.getCompletedTaskCount())); + // 1 task gets executed on the caller thread + assertBusy(() -> assertEquals(numSegments - 1, executor.getCompletedTaskCount())); } } finally { terminate(executor); @@ -253,8 +254,9 @@ public void testConcurrentCollection() throws Exception { Integer totalHits = searcher.search(new MatchAllDocsQuery(), new TotalHitCountCollectorManager()); assertEquals(numDocs, totalHits.intValue()); int numExpectedTasks = ContextIndexSearcher.computeSlices(searcher.getIndexReader().leaves(), Integer.MAX_VALUE, 1).length; - // check that each slice goes to the executor, no matter the queue size or the number of slices - assertBusy(() -> assertEquals(numExpectedTasks, executor.getCompletedTaskCount())); + // check that each slice except for one that executes on the calling thread goes to the executor, no matter the queue size + // or the number of slices + assertBusy(() -> assertEquals(numExpectedTasks - 1, executor.getCompletedTaskCount())); } } finally { terminate(executor); diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java index 1509cfa08b400..f8c3edcbb9d42 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java @@ -1850,8 +1850,6 @@ private Environment createEnvironment(String nodeName) { Settings.builder() .put(NODE_NAME_SETTING.getKey(), nodeName) .put(PATH_HOME_SETTING.getKey(), tempDir.resolve(nodeName).toAbsolutePath()) - // test uses the same executor service for all thread pools, search worker would need to be a different one - .put(SearchService.SEARCH_WORKER_THREADS_ENABLED.getKey(), false) .put(Environment.PATH_REPO_SETTING.getKey(), tempDir.resolve("repo").toAbsolutePath()) .putList( ClusterBootstrapService.INITIAL_MASTER_NODES_SETTING.getKey(), diff --git a/server/src/test/java/org/elasticsearch/threadpool/ThreadPoolTests.java b/server/src/test/java/org/elasticsearch/threadpool/ThreadPoolTests.java index 310cf467a8391..808c0a5b88b7e 100644 --- a/server/src/test/java/org/elasticsearch/threadpool/ThreadPoolTests.java +++ b/server/src/test/java/org/elasticsearch/threadpool/ThreadPoolTests.java @@ -25,8 +25,6 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; -import java.util.concurrent.LinkedTransferQueue; -import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import static org.elasticsearch.common.util.concurrent.EsExecutors.TaskTrackingConfig.DEFAULT; @@ -371,25 +369,6 @@ public void testWriteThreadPoolUsesTaskExecutionTimeTrackingEsThreadPoolExecutor } } - public void testSearchWorkedThreadPool() { - final int allocatedProcessors = randomIntBetween(1, EsExecutors.allocatedProcessors(Settings.EMPTY)); - final ThreadPool threadPool = new TestThreadPool( - "test", - Settings.builder().put(EsExecutors.NODE_PROCESSORS_SETTING.getKey(), allocatedProcessors).build() - ); - try { - ExecutorService executor = threadPool.executor(ThreadPool.Names.SEARCH_WORKER); - assertThat(executor, instanceOf(ThreadPoolExecutor.class)); - ThreadPoolExecutor threadPoolExecutor = (ThreadPoolExecutor) executor; - int expectedPoolSize = allocatedProcessors * 3 / 2 + 1; - assertEquals(expectedPoolSize, threadPoolExecutor.getCorePoolSize()); - assertEquals(expectedPoolSize, threadPoolExecutor.getMaximumPoolSize()); - assertThat(threadPoolExecutor.getQueue(), instanceOf(LinkedTransferQueue.class)); - } finally { - assertTrue(terminate(threadPool)); - } - } - public void testScheduledOneShotRejection() { final var name = "fixed-bounded"; final var threadPool = new TestThreadPool( diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index f885442373d70..d35d5282238ee 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -212,7 +212,7 @@ public abstract class AggregatorTestCase extends ESTestCase { @Before public final void initPlugins() { threadPool = new TestThreadPool(AggregatorTestCase.class.getName()); - threadPoolExecutor = (ThreadPoolExecutor) threadPool.executor(ThreadPool.Names.SEARCH_WORKER); + threadPoolExecutor = (ThreadPoolExecutor) threadPool.executor(ThreadPool.Names.SEARCH); List plugins = new ArrayList<>(getSearchPlugins()); plugins.add(new AggCardinalityUpperBoundPlugin()); SearchModule searchModule = new SearchModule(Settings.EMPTY, plugins); diff --git a/test/framework/src/test/java/org/elasticsearch/search/internal/ConcurrentSearchSingleNodeTests.java b/test/framework/src/test/java/org/elasticsearch/search/internal/ConcurrentSearchSingleNodeTests.java index b73066c6f4d38..c9c9759bd0cec 100644 --- a/test/framework/src/test/java/org/elasticsearch/search/internal/ConcurrentSearchSingleNodeTests.java +++ b/test/framework/src/test/java/org/elasticsearch/search/internal/ConcurrentSearchSingleNodeTests.java @@ -9,33 +9,22 @@ package org.elasticsearch.search.internal; -import org.elasticsearch.core.TimeValue; -import org.elasticsearch.index.IndexService; -import org.elasticsearch.index.shard.IndexShard; -import org.elasticsearch.indices.IndicesService; +import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.search.SearchService; import org.elasticsearch.test.ESSingleNodeTestCase; -import java.io.IOException; - public class ConcurrentSearchSingleNodeTests extends ESSingleNodeTestCase { private final boolean concurrentSearch = randomBoolean(); - public void testConcurrentSearch() throws IOException { + public void testConcurrentSearch() { client().admin().indices().prepareCreate("index").get(); - IndicesService indicesService = getInstanceFromNode(IndicesService.class); - IndexService indexService = indicesService.iterator().next(); - IndexShard shard = indexService.getShard(0); - SearchService searchService = getInstanceFromNode(SearchService.class); - ShardSearchRequest shardSearchRequest = new ShardSearchRequest(shard.shardId(), 0L, AliasFilter.EMPTY); - try (SearchContext searchContext = searchService.createSearchContext(shardSearchRequest, TimeValue.MINUS_ONE)) { - ContextIndexSearcher searcher = searchContext.searcher(); - if (concurrentSearch) { - assertEquals(1, searcher.getMinimumDocsPerSlice()); - } else { - assertEquals(50_000, searcher.getMinimumDocsPerSlice()); - } + ClusterService clusterService = getInstanceFromNode(ClusterService.class); + int minDocsPerSlice = SearchService.MINIMUM_DOCS_PER_SLICE.get(clusterService.getSettings()); + if (concurrentSearch) { + assertEquals(1, minDocsPerSlice); + } else { + assertEquals(50_000, minDocsPerSlice); } } diff --git a/test/framework/src/test/java/org/elasticsearch/search/internal/ConcurrentSearchTestPluginTests.java b/test/framework/src/test/java/org/elasticsearch/search/internal/ConcurrentSearchTestPluginTests.java index f99efe33af09b..6b983d47bdf42 100644 --- a/test/framework/src/test/java/org/elasticsearch/search/internal/ConcurrentSearchTestPluginTests.java +++ b/test/framework/src/test/java/org/elasticsearch/search/internal/ConcurrentSearchTestPluginTests.java @@ -9,34 +9,23 @@ package org.elasticsearch.search.internal; -import org.elasticsearch.core.TimeValue; -import org.elasticsearch.index.IndexService; -import org.elasticsearch.index.shard.IndexShard; -import org.elasticsearch.indices.IndicesService; +import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.search.SearchService; import org.elasticsearch.test.ESIntegTestCase; -import java.io.IOException; - @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 1) public class ConcurrentSearchTestPluginTests extends ESIntegTestCase { private final boolean concurrentSearch = randomBoolean(); - public void testConcurrentSearch() throws IOException { + public void testConcurrentSearch() { client().admin().indices().prepareCreate("index").get(); - IndicesService indicesService = internalCluster().getDataNodeInstance(IndicesService.class); - IndexService indexService = indicesService.iterator().next(); - IndexShard shard = indexService.getShard(0); - SearchService searchService = internalCluster().getDataNodeInstance(SearchService.class); - ShardSearchRequest shardSearchRequest = new ShardSearchRequest(shard.shardId(), 0L, AliasFilter.EMPTY); - try (SearchContext searchContext = searchService.createSearchContext(shardSearchRequest, TimeValue.MINUS_ONE)) { - ContextIndexSearcher searcher = searchContext.searcher(); - if (concurrentSearch) { - assertEquals(1, searcher.getMinimumDocsPerSlice()); - } else { - assertEquals(50_000, searcher.getMinimumDocsPerSlice()); - } + ClusterService clusterService = internalCluster().getDataNodeInstance(ClusterService.class); + int minDocsPerSlice = SearchService.MINIMUM_DOCS_PER_SLICE.get(clusterService.getSettings()); + if (concurrentSearch) { + assertEquals(1, minDocsPerSlice); + } else { + assertEquals(50_000, minDocsPerSlice); } } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRestoreSourceService.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRestoreSourceService.java index fa9438353779f..6b390ab5747a8 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRestoreSourceService.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRestoreSourceService.java @@ -9,6 +9,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; import org.elasticsearch.common.component.AbstractLifecycleComponent; @@ -244,9 +245,10 @@ private Store.MetadataSnapshot getMetadata() throws IOException { private long readFileBytes(String fileName, ByteArray reference) throws IOException { try (Releasable ignored = keyedLock.acquire(fileName)) { + var context = fileName.startsWith(IndexFileNames.SEGMENTS) ? IOContext.READONCE : IOContext.READ; final IndexInput indexInput = cachedInputs.computeIfAbsent(fileName, f -> { try { - return commitRef.getIndexCommit().getDirectory().openInput(fileName, IOContext.READONCE); + return commitRef.getIndexCommit().getDirectory().openInput(fileName, context); } catch (IOException e) { throw new UncheckedIOException(e); } @@ -256,7 +258,7 @@ private long readFileBytes(String fileName, ByteArray reference) throws IOExcept long offsetAfterRead = indexInput.getFilePointer(); - if (offsetAfterRead == indexInput.length()) { + if (offsetAfterRead == indexInput.length() || context == IOContext.READONCE) { cachedInputs.remove(fileName); IOUtils.close(indexInput); } diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/OldSegmentInfos.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/OldSegmentInfos.java index e5de349203b3d..d1455eaa2f1c4 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/OldSegmentInfos.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/OldSegmentInfos.java @@ -196,7 +196,7 @@ static final OldSegmentInfos readCommit(Directory directory, String segmentFileN long generation = generationFromSegmentsFileName(segmentFileName); // System.out.println(Thread.currentThread() + ": SegmentInfos.readCommit " + segmentFileName); - try (ChecksumIndexInput input = directory.openChecksumInput(segmentFileName, IOContext.READ)) { + try (ChecksumIndexInput input = directory.openChecksumInput(segmentFileName, IOContext.READONCE)) { try { return readCommit(directory, input, generation, minSupportedMajorVersion); } catch (EOFException | NoSuchFileException | FileNotFoundException e) { diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/MetadataCachingIndexInput.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/MetadataCachingIndexInput.java index 94ba06a00cc4e..c1468f2e45df0 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/MetadataCachingIndexInput.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/MetadataCachingIndexInput.java @@ -221,7 +221,6 @@ public static boolean assertCurrentThreadMayAccessBlobStore() { ThreadPool.Names.SNAPSHOT, ThreadPool.Names.GENERIC, ThreadPool.Names.SEARCH, - ThreadPool.Names.SEARCH_WORKER, ThreadPool.Names.SEARCH_THROTTLED, // Cache asynchronous fetching runs on a dedicated thread pool. diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/store/SearchableSnapshotDirectoryTests.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/store/SearchableSnapshotDirectoryTests.java index 1452847c65b4c..e65c4a60f89d5 100644 --- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/store/SearchableSnapshotDirectoryTests.java +++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/store/SearchableSnapshotDirectoryTests.java @@ -692,7 +692,7 @@ private void testDirectories( private void testIndexInputs(final CheckedBiConsumer consumer) throws Exception { testDirectories((directory, snapshotDirectory) -> { for (String fileName : randomSubsetOf(Arrays.asList(snapshotDirectory.listAll()))) { - final IOContext context = randomIOContext(); + final IOContext context = fileName.startsWith(IndexFileNames.SEGMENTS) ? IOContext.READONCE : randomIOContext(); try (IndexInput indexInput = directory.openInput(fileName, context)) { final List closeables = new ArrayList<>(); try { diff --git a/x-pack/qa/runtime-fields/build.gradle b/x-pack/qa/runtime-fields/build.gradle index 5add595d64e3f..43d6d9463e0d1 100644 --- a/x-pack/qa/runtime-fields/build.gradle +++ b/x-pack/qa/runtime-fields/build.gradle @@ -29,7 +29,7 @@ subprojects { restResources { restApi { - include '_common', 'bulk', 'count', 'cluster', 'index', 'indices', 'field_caps', 'msearch', + include 'capabilities', '_common', 'bulk', 'count', 'cluster', 'index', 'indices', 'field_caps', 'msearch', 'search', 'async_search', 'graph', '*_point_in_time', 'put_script', 'scripts_painless_execute' } restTests {