diff --git a/docs/changelog/130849.yaml b/docs/changelog/130849.yaml new file mode 100644 index 0000000000000..6dcc5908f7fb1 --- /dev/null +++ b/docs/changelog/130849.yaml @@ -0,0 +1,6 @@ +pr: 130849 +summary: Fix behavior for `_index` LIKE for ESQL +area: ES|QL +type: bug +issues: + - 129511 diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index da26fb61eedd6..69ce2d827f805 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -256,6 +256,7 @@ static TransportVersion def(int id) { public static final TransportVersion ESQL_DOCUMENTS_FOUND_AND_VALUES_LOADED_8_19 = def(8_841_0_61); public static final TransportVersion ESQL_PROFILE_INCLUDE_PLAN_8_19 = def(8_841_0_62); public static final TransportVersion ESQL_SPLIT_ON_BIG_VALUES_8_19 = def(8_841_0_63); + public static final TransportVersion ESQL_FIXED_INDEX_LIKE_8_19 = def(8_841_0_64); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ConstantFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/ConstantFieldType.java index 5ecb75b09408c..0b246c2492fdb 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ConstantFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ConstantFieldType.java @@ -15,6 +15,8 @@ import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.automaton.Automaton; +import org.apache.lucene.util.automaton.CharacterRunAutomaton; import org.elasticsearch.common.lucene.search.Queries; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.core.Nullable; @@ -23,6 +25,7 @@ import java.util.Collection; import java.util.Map; +import java.util.function.Supplier; /** * A {@link MappedFieldType} that has the same value for all documents. @@ -135,9 +138,47 @@ public final Query wildcardQuery(String value, boolean caseInsensitive, QueryRew } } + /** + * Returns a query that matches all documents or no documents + * It usually calls {@link #wildcardQuery(String, boolean, QueryRewriteContext)} + * except for IndexFieldType which overrides this method to use its own matching logic. + */ + public Query wildcardLikeQuery(String value, boolean caseInsensitive, QueryRewriteContext context) { + return wildcardQuery(value, caseInsensitive, context); + } + @Override public final boolean fieldHasValue(FieldInfos fieldInfos) { // We consider constant field types to always have value. return true; } + + /** + * Returns the constant value of this field as a string. + * Based on the field type, we need to get it in a different way. + */ + public abstract String getConstantFieldValue(SearchExecutionContext context); + + /** + * Returns a query that matches all documents or no documents + * depending on whether the constant value of this field matches or not + */ + @Override + public Query automatonQuery( + Supplier automatonSupplier, + Supplier characterRunAutomatonSupplier, + @Nullable MultiTermQuery.RewriteMethod method, + SearchExecutionContext context, + String description + ) { + CharacterRunAutomaton compiled = characterRunAutomatonSupplier.get(); + boolean matches = compiled.run(getConstantFieldValue(context)); + if (matches) { + return new MatchAllDocsQuery(); + } else { + return new MatchNoDocsQuery( + "The \"" + context.getFullyQualifiedIndex().getName() + "\" query was rewritten to a \"match_none\" query." + ); + } + } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java index 33c6ff15cccfd..79754a4c63b30 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java @@ -10,9 +10,13 @@ package org.elasticsearch.index.mapper; import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.MatchNoDocsQuery; +import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.regex.Regex; +import org.elasticsearch.core.Nullable; import org.elasticsearch.index.fielddata.FieldData; import org.elasticsearch.index.fielddata.FieldDataContext; import org.elasticsearch.index.fielddata.IndexFieldData; @@ -27,6 +31,7 @@ import java.util.Collections; import java.util.List; +import java.util.Locale; public class IndexFieldMapper extends MetadataFieldMapper { @@ -102,6 +107,38 @@ public StoredFieldsSpec storedFieldsSpec() { }; } + @Override + public Query wildcardLikeQuery( + String value, + @Nullable MultiTermQuery.RewriteMethod method, + boolean caseInsensitve, + SearchExecutionContext context + ) { + String indexName = context.getFullyQualifiedIndex().getName(); + return getWildcardLikeQuery(value, caseInsensitve, indexName); + } + + @Override + public Query wildcardLikeQuery(String value, boolean caseInsensitive, QueryRewriteContext context) { + String indexName = context.getFullyQualifiedIndex().getName(); + return getWildcardLikeQuery(value, caseInsensitive, indexName); + } + + private static Query getWildcardLikeQuery(String value, boolean caseInsensitve, String indexName) { + if (caseInsensitve) { + value = value.toLowerCase(Locale.ROOT); + indexName = indexName.toLowerCase(Locale.ROOT); + } + if (Regex.simpleMatch(value, indexName)) { + return new MatchAllDocsQuery(); + } + return new MatchNoDocsQuery("The \"" + indexName + "\" query was rewritten to a \"match_none\" query."); + } + + @Override + public String getConstantFieldValue(SearchExecutionContext context) { + return context.getFullyQualifiedIndex().getName(); + } } public IndexFieldMapper() { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IndexModeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IndexModeFieldMapper.java index e539c07caef61..2ddfbb84f4c76 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IndexModeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IndexModeFieldMapper.java @@ -60,6 +60,11 @@ protected boolean matches(String pattern, boolean caseInsensitive, QueryRewriteC return Regex.simpleMatch(pattern, indexMode, caseInsensitive); } + @Override + public String getConstantFieldValue(SearchExecutionContext context) { + return context.getIndexSettings().getMode().getName(); + } + @Override public Query existsQuery(SearchExecutionContext context) { return new MatchAllDocsQuery(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 0da583de1ad79..b6de0c4350d8b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -23,6 +23,7 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.MultiTerms; +import org.apache.lucene.index.Term; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.MultiTermQuery; @@ -30,6 +31,7 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.automaton.Automata; import org.apache.lucene.util.automaton.Automaton; +import org.apache.lucene.util.automaton.CharacterRunAutomaton; import org.apache.lucene.util.automaton.CompiledAutomaton; import org.apache.lucene.util.automaton.CompiledAutomaton.AUTOMATON_TYPE; import org.apache.lucene.util.automaton.MinimizationOperations; @@ -50,6 +52,7 @@ import org.elasticsearch.index.fielddata.SourceValueFetcherSortedBinaryIndexFieldData; import org.elasticsearch.index.fielddata.StoredFieldSortedBinaryIndexFieldData; import org.elasticsearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData; +import org.elasticsearch.index.query.AutomatonQueryWithDescription; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.similarity.SimilarityProvider; import org.elasticsearch.script.Script; @@ -81,6 +84,7 @@ import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.function.Supplier; import static org.apache.lucene.index.IndexWriter.MAX_TERM_LENGTH; import static org.elasticsearch.core.Strings.format; @@ -930,6 +934,17 @@ public boolean hasScriptValues() { public boolean hasNormalizer() { return normalizer != Lucene.KEYWORD_ANALYZER; } + } + + @Override + public Query automatonQuery( + Supplier automatonSupplier, + Supplier characterRunAutomatonSupplier, + @Nullable MultiTermQuery.RewriteMethod method, + SearchExecutionContext context, + String description + ) { + return new AutomatonQueryWithDescription(new Term(name()), automatonSupplier.get(), description); } private final boolean indexed; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java index ed750f4515473..e8ae8391dcfbb 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java @@ -29,6 +29,8 @@ import org.apache.lucene.search.TermInSetQuery; import org.apache.lucene.search.TermQuery; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.automaton.Automaton; +import org.apache.lucene.util.automaton.CharacterRunAutomaton; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.cluster.metadata.IndexMetadata; @@ -58,6 +60,7 @@ import java.util.Objects; import java.util.Set; import java.util.function.Function; +import java.util.function.Supplier; import static org.elasticsearch.search.SearchService.ALLOW_EXPENSIVE_QUERIES; @@ -333,6 +336,19 @@ public final Query wildcardQuery(String value, @Nullable MultiTermQuery.RewriteM return wildcardQuery(value, method, false, context); } + /** + * Similar to wildcardQuery, except that we change the behavior for ESQL + * to behave like a string LIKE query, where the value is matched as a string + */ + public Query wildcardLikeQuery( + String value, + @Nullable MultiTermQuery.RewriteMethod method, + boolean caseInsensitve, + SearchExecutionContext context + ) { + return wildcardQuery(value, method, caseInsensitve, context); + } + public Query wildcardQuery( String value, @Nullable MultiTermQuery.RewriteMethod method, @@ -374,6 +390,23 @@ public Query regexpQuery( ); } + /** + * Returns a Lucine pushable Query for the current field + * For now can only be AutomatonQuery or MatchAllDocsQuery() or MatchNoDocsQuery() + */ + public Query automatonQuery( + Supplier automatonSupplier, + Supplier characterRunAutomatonSupplier, + @Nullable MultiTermQuery.RewriteMethod method, + SearchExecutionContext context, + String description + ) { + throw new QueryShardException( + context, + "Can only use automaton queries on keyword fields - not on [" + name + "] which is of type [" + typeName() + "]" + ); + } + public Query existsQuery(SearchExecutionContext context) { if (hasDocValues() || getTextSearchInfo().hasNorms()) { return new FieldExistsQuery(name()); diff --git a/server/src/main/java/org/elasticsearch/index/query/AutomatonQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/AutomatonQueryBuilder.java deleted file mode 100644 index 9c6331044e6d5..0000000000000 --- a/server/src/main/java/org/elasticsearch/index/query/AutomatonQueryBuilder.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.index.query; - -import org.apache.lucene.index.Term; -import org.apache.lucene.search.AutomatonQuery; -import org.apache.lucene.search.Query; -import org.apache.lucene.util.automaton.Automaton; -import org.elasticsearch.TransportVersion; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.xcontent.XContentBuilder; - -import java.io.IOException; -import java.io.UnsupportedEncodingException; -import java.util.Objects; - -/** - * Implements an Automaton query, which matches documents based on a Lucene Automaton. - * It does not support serialization or XContent representation. - */ -public class AutomatonQueryBuilder extends AbstractQueryBuilder implements MultiTermQueryBuilder { - private final String fieldName; - private final Automaton automaton; - private final String description; - - public AutomatonQueryBuilder(String fieldName, Automaton automaton, String description) { - if (Strings.isEmpty(fieldName)) { - throw new IllegalArgumentException("field name is null or empty"); - } - if (automaton == null) { - throw new IllegalArgumentException("automaton cannot be null"); - } - this.fieldName = fieldName; - this.automaton = automaton; - this.description = description; - } - - @Override - public String fieldName() { - return fieldName; - } - - @Override - public String getWriteableName() { - throw new UnsupportedOperationException("AutomatonQueryBuilder does not support getWriteableName"); - } - - @Override - protected void doWriteTo(StreamOutput out) throws IOException { - throw new UnsupportedEncodingException("AutomatonQueryBuilder does not support doWriteTo"); - } - - @Override - protected void doXContent(XContentBuilder builder, Params params) throws IOException { - throw new UnsupportedEncodingException("AutomatonQueryBuilder does not support doXContent"); - } - - @Override - protected Query doToQuery(SearchExecutionContext context) throws IOException { - return new AutomatonQueryWithDescription(new Term(fieldName), automaton, description); - } - - @Override - protected int doHashCode() { - return Objects.hash(fieldName, automaton, description); - } - - @Override - protected boolean doEquals(AutomatonQueryBuilder other) { - return Objects.equals(fieldName, other.fieldName) - && Objects.equals(automaton, other.automaton) - && Objects.equals(description, other.description); - } - - @Override - public TransportVersion getMinimalSupportedVersion() { - throw new UnsupportedOperationException("AutomatonQueryBuilder does not support getMinimalSupportedVersion"); - } - - static class AutomatonQueryWithDescription extends AutomatonQuery { - private final String description; - - AutomatonQueryWithDescription(Term term, Automaton automaton, String description) { - super(term, automaton); - this.description = description; - } - - @Override - public String toString(String field) { - if (this.field.equals(field)) { - return description; - } - return this.field + ":" + description; - } - } -} diff --git a/server/src/main/java/org/elasticsearch/index/query/AutomatonQueryWithDescription.java b/server/src/main/java/org/elasticsearch/index/query/AutomatonQueryWithDescription.java new file mode 100644 index 0000000000000..78c285470e3b6 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/query/AutomatonQueryWithDescription.java @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.query; + +import org.apache.lucene.index.Term; +import org.apache.lucene.search.AutomatonQuery; +import org.apache.lucene.util.automaton.Automaton; + +/** + * A specialized {@link AutomatonQuery} that includes a description of the query. + * This can be useful for debugging or logging purposes, providing more context + * about the query being executed. + */ +public class AutomatonQueryWithDescription extends AutomatonQuery { + private final String description; + + public AutomatonQueryWithDescription(Term term, Automaton automaton, String description) { + super(term, automaton); + this.description = description; + } + + @Override + public String toString(String field) { + if (this.field.equals(field)) { + return description; + } + return this.field + ":" + description; + } +} diff --git a/server/src/main/java/org/elasticsearch/index/query/CoordinatorRewriteContext.java b/server/src/main/java/org/elasticsearch/index/query/CoordinatorRewriteContext.java index 823679276edac..1e2f33a2d1b84 100644 --- a/server/src/main/java/org/elasticsearch/index/query/CoordinatorRewriteContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/CoordinatorRewriteContext.java @@ -72,6 +72,11 @@ protected boolean matches(String pattern, boolean caseInsensitive, QueryRewriteC return Regex.simpleMatch(pattern, tierPreference); } + @Override + public String getConstantFieldValue(SearchExecutionContext context) { + return context.getTierPreference(); + } + @Override public Query existsQuery(SearchExecutionContext context) { throw new UnsupportedOperationException("field exists query is not supported on the coordinator node"); diff --git a/server/src/main/java/org/elasticsearch/index/query/WildcardQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/WildcardQueryBuilder.java index 419195e5e5ba5..f51dce10ea96c 100644 --- a/server/src/main/java/org/elasticsearch/index/query/WildcardQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/WildcardQueryBuilder.java @@ -56,6 +56,13 @@ public class WildcardQueryBuilder extends AbstractQueryBuilder runEsql(RestEsqlTestCase.RequestObjectBuilder reques } } + private void assertResultMapForLike( + boolean includeCCSMetadata, + Map result, + C columns, + V values, + boolean remoteOnly, + boolean requireLikeListCapability + ) throws IOException { + List requiredCapabilities = new ArrayList<>(List.of("like_on_index_fields")); + if (requireLikeListCapability) { + requiredCapabilities.add("like_list_on_index_fields"); + } + // the feature is completely supported if both local and remote clusters support it + boolean isSupported = capabilitiesSupportedNewAndOld(requiredCapabilities); + + if (isSupported) { + assertResultMap(includeCCSMetadata, result, columns, values, remoteOnly); + } else { + logger.info("--> skipping data check for like index test, cluster does not support like index feature"); + // just verify that we did not get a partial result + var clusters = result.get("_clusters"); + var reason = "unexpected partial results" + (clusters != null ? ": _clusters=" + clusters : ""); + assertThat(reason, result.get("is_partial"), anyOf(nullValue(), is(false))); + } + } + + private boolean capabilitiesSupportedNewAndOld(List requiredCapabilities) throws IOException { + boolean isSupported = clusterHasCapability("POST", "/_query", List.of(), requiredCapabilities).orElse(false); + try (RestClient remoteClient = remoteClusterClient()) { + isSupported = isSupported + && clusterHasCapability(remoteClient, "POST", "/_query", List.of(), requiredCapabilities).orElse(false); + } + return isSupported; + } + private void assertResultMap(boolean includeCCSMetadata, Map result, C columns, V values, boolean remoteOnly) { MapMatcher mapMatcher = getResultMatcher( ccsMetadataAvailable(), @@ -371,6 +406,53 @@ public void testStats() throws IOException { assertThat(clusterData, hasKey("took")); } + public void testLikeIndexLegacySettingNoResults() throws Exception { + // the feature is completely supported if both local and remote clusters support it + assumeTrue("not supported", capabilitiesSupportedNewAndOld(List.of("like_on_index_fields"))); + try ( + ClusterSettingToggle ignored = new ClusterSettingToggle(adminClient(), "esql.query.string_like_on_index", false, true); + RestClient remoteClient = remoteClusterClient(); + ClusterSettingToggle ignored2 = new ClusterSettingToggle(remoteClient, "esql.query.string_like_on_index", false, true) + ) { + // test code with the setting changed + boolean includeCCSMetadata = includeCCSMetadata(); + Map result = run(""" + FROM test-local-index,*:test-remote-index METADATA _index + | WHERE _index LIKE "*remote*" + | STATS c = COUNT(*) BY _index + | SORT _index ASC + """, includeCCSMetadata); + var columns = List.of(Map.of("name", "c", "type", "long"), Map.of("name", "_index", "type", "keyword")); + // we expect empty result, since the setting is false + var values = List.of(); + assertResultMapForLike(includeCCSMetadata, result, columns, values, false, false); + } + } + + public void testLikeIndexLegacySettingResults() throws Exception { + // we require that the admin client supports the like_on_index_fields capability + // otherwise we will get an error when trying to toggle the setting + // the remote client does not have to support it + assumeTrue("not supported", capabilitiesSupportedNewAndOld(List.of("like_on_index_fields"))); + try ( + ClusterSettingToggle ignored = new ClusterSettingToggle(adminClient(), "esql.query.string_like_on_index", false, true); + RestClient remoteClient = remoteClusterClient(); + ClusterSettingToggle ignored2 = new ClusterSettingToggle(remoteClient, "esql.query.string_like_on_index", false, true) + ) { + boolean includeCCSMetadata = includeCCSMetadata(); + Map result = run(""" + FROM test-local-index,*:test-remote-index METADATA _index + | WHERE _index LIKE "*remote*:*remote*" + | STATS c = COUNT(*) BY _index + | SORT _index ASC + """, includeCCSMetadata); + var columns = List.of(Map.of("name", "c", "type", "long"), Map.of("name", "_index", "type", "keyword")); + // we expect results, since the setting is false, but there is : in the LIKE query + var values = List.of(List.of(remoteDocs.size(), REMOTE_CLUSTER_NAME + ":" + remoteIndex)); + assertResultMapForLike(includeCCSMetadata, result, columns, values, false, false); + } + } + private RestClient remoteClusterClient() throws IOException { var clusterHosts = parseClusterHosts(remoteCluster.getHttpAddresses()); return buildClient(restClientSettings(), clusterHosts.toArray(new HttpHost[0])); @@ -387,4 +469,28 @@ private static boolean capabilitiesEndpointAvailable() { private static boolean includeCCSMetadata() { return ccsMetadataAvailable() && randomBoolean(); } + + public static class ClusterSettingToggle implements AutoCloseable { + private final RestClient client; + private final String settingKey; + private final Object originalValue; + + public ClusterSettingToggle(RestClient client, String settingKey, Object newValue, Object restoreValue) throws IOException { + this.client = client; + this.settingKey = settingKey; + this.originalValue = restoreValue; + setValue(newValue); + } + + private void setValue(Object value) throws IOException { + Request set = new Request("PUT", "/_cluster/settings"); + set.setJsonEntity("{\"persistent\": {\"" + settingKey + "\": " + value + "}}"); + ESRestTestCase.assertOK(client.performRequest(set)); + } + + @Override + public void close() throws IOException { + setValue(originalValue == null ? "null" : originalValue); + } + } } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java index 86a07cc7cd9a4..d4de404db61be 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java @@ -861,7 +861,7 @@ static Version randomVersion() { } public static WildcardLike wildcardLike(Expression left, String exp) { - return new WildcardLike(EMPTY, left, new WildcardPattern(exp)); + return new WildcardLike(EMPTY, left, new WildcardPattern(exp), false); } public static RLike rlike(Expression left, String exp) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 37b3216b3f26e..ea6fab469db35 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -986,6 +986,8 @@ public enum Cap { */ LIKE_WITH_LIST_OF_PATTERNS, + LIKE_LIST_ON_INDEX_FIELDS, + /** * Support for the SAMPLE command */ @@ -1010,6 +1012,9 @@ public enum Cap { NO_PLAIN_STRINGS_IN_LITERALS, /** + * Support improved behavior for LIKE operator when used with index fields. + */ + LIKE_ON_INDEX_FIELDS, * Support avg with aggregate metric doubles */ AGGREGATE_METRIC_DOUBLE_AVG(AGGREGATE_METRIC_DOUBLE_FEATURE_FLAG); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/capabilities/TranslationAware.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/capabilities/TranslationAware.java index 1a2fd81db4b6c..730cccb4dce45 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/capabilities/TranslationAware.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/capabilities/TranslationAware.java @@ -7,8 +7,11 @@ package org.elasticsearch.xpack.esql.capabilities; +import org.apache.lucene.search.MultiTermQuery.RewriteMethod; import org.elasticsearch.compute.lucene.LuceneTopNSourceOperator; import org.elasticsearch.compute.operator.FilterOperator; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.querydsl.query.Query; import org.elasticsearch.xpack.esql.optimizer.rules.physical.local.LucenePushdownPredicates; @@ -47,6 +50,23 @@ static TranslationAware.Translatable translatable(Expression exp, LucenePushdown */ Query asQuery(LucenePushdownPredicates pushdownPredicates, TranslatorHandler handler); + /** + * Translates this expression into a Lucene {@link org.apache.lucene.search.Query}. + *

+ * Implementations should use the provided field type, rewrite method, and search execution context + * to construct an appropriate Lucene query for this expression. + * By default, this method throws {@link UnsupportedOperationException}; override it in subclasses + * that support Lucene query translation. + *

+ */ + default org.apache.lucene.search.Query asLuceneQuery( + MappedFieldType fieldType, + RewriteMethod constantScoreRewrite, + SearchExecutionContext context + ) { + throw new UnsupportedOperationException("asLuceneQuery is not implemented for " + getClass().getName()); + } + /** * Subinterface for expressions that can only process single values (and null out on MVs). */ diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWith.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWith.java index 52dedcb670372..5a165d7c822a0 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWith.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWith.java @@ -151,7 +151,7 @@ public Query asQuery(LucenePushdownPredicates pushdownPredicates, TranslatorHand // TODO: Get the real FoldContext here var wildcardQuery = "*" + QueryParser.escape(BytesRefs.toString(suffix.fold(FoldContext.small()))); - return new WildcardQuery(source(), fieldName, wildcardQuery); + return new WildcardQuery(source(), fieldName, wildcardQuery, false, false); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWith.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWith.java index f457bacf44268..3dc43e29f3fa9 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWith.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWith.java @@ -148,7 +148,7 @@ public Query asQuery(LucenePushdownPredicates pushdownPredicates, TranslatorHand // TODO: Get the real FoldContext here var wildcardQuery = QueryParser.escape(BytesRefs.toString(prefix.fold(FoldContext.small()))) + "*"; - return new WildcardQuery(source(), fieldName, wildcardQuery); + return new WildcardQuery(source(), fieldName, wildcardQuery, false, false); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/regex/WildcardLike.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/regex/WildcardLike.java index 55d6961b8015f..949fba0604245 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/regex/WildcardLike.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/regex/WildcardLike.java @@ -121,11 +121,14 @@ public Translatable translatable(LucenePushdownPredicates pushdownPredicates) { public Query asQuery(LucenePushdownPredicates pushdownPredicates, TranslatorHandler handler) { var field = field(); LucenePushdownPredicates.checkIsPushableAttribute(field); - return translateField(handler.nameOf(field instanceof FieldAttribute fa ? fa.exactAttribute() : field)); + return translateField( + handler.nameOf(field instanceof FieldAttribute fa ? fa.exactAttribute() : field), + pushdownPredicates.flags().stringLikeOnIndex() + ); } // TODO: see whether escaping is needed - private Query translateField(String targetFieldName) { - return new WildcardQuery(source(), targetFieldName, pattern().asLuceneWildcard(), caseInsensitive()); + private Query translateField(String targetFieldName, boolean forceStringMatch) { + return new WildcardQuery(source(), targetFieldName, pattern().asLuceneWildcard(), caseInsensitive(), forceStringMatch); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/regex/WildcardLikeList.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/regex/WildcardLikeList.java index 0b58594779408..d38e315b58b4f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/regex/WildcardLikeList.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/regex/WildcardLikeList.java @@ -7,26 +7,35 @@ package org.elasticsearch.xpack.esql.expression.function.scalar.string.regex; +import org.apache.lucene.search.MultiTermQuery.RewriteMethod; +import org.apache.lucene.util.automaton.Automaton; +import org.apache.lucene.util.automaton.CharacterRunAutomaton; +import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.expression.predicate.regex.WildcardPattern; import org.elasticsearch.xpack.esql.core.expression.predicate.regex.WildcardPatternList; -import org.elasticsearch.xpack.esql.core.querydsl.query.AutomatonQuery; import org.elasticsearch.xpack.esql.core.querydsl.query.Query; import org.elasticsearch.xpack.esql.core.querydsl.query.WildcardQuery; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.expression.function.Param; +import org.elasticsearch.xpack.esql.io.stream.ExpressionQuery; import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; import org.elasticsearch.xpack.esql.optimizer.rules.physical.local.LucenePushdownPredicates; import org.elasticsearch.xpack.esql.planner.TranslatorHandler; import java.io.IOException; +import java.util.function.Supplier; import java.util.stream.Collectors; +import static org.elasticsearch.index.query.WildcardQueryBuilder.expressionTransportSupported; + public class WildcardLikeList extends RegexMatch { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( Expression.class, @@ -34,6 +43,30 @@ public class WildcardLikeList extends RegexMatch { WildcardLikeList::new ); + Supplier automatonSupplier = new Supplier<>() { + Automaton cached; + + @Override + public Automaton get() { + if (cached == null) { + cached = pattern().createAutomaton(caseInsensitive()); + } + return cached; + } + }; + + Supplier characterRunAutomatonSupplier = new Supplier<>() { + CharacterRunAutomaton cached; + + @Override + public CharacterRunAutomaton get() { + if (cached == null) { + cached = new CharacterRunAutomaton(automatonSupplier.get()); + } + return cached; + } + }; + /** * The documentation for this function is in WildcardLike, and shown to the users `LIKE` in the docs. */ @@ -92,8 +125,12 @@ protected WildcardLikeList replaceChild(Expression newLeft) { */ @Override public Translatable translatable(LucenePushdownPredicates pushdownPredicates) { - return pushdownPredicates.isPushableAttribute(field()) ? Translatable.YES : Translatable.NO; - + if (supportsPushdown(pushdownPredicates.minTransportVersion())) { + return pushdownPredicates.isPushableAttribute(field()) ? Translatable.YES : Translatable.NO; + } else { + // The ExpressionQuery we use isn't serializable to all nodes in the cluster. + return Translatable.NO; + } } /** @@ -104,20 +141,40 @@ public Translatable translatable(LucenePushdownPredicates pushdownPredicates) { public Query asQuery(LucenePushdownPredicates pushdownPredicates, TranslatorHandler handler) { var field = field(); LucenePushdownPredicates.checkIsPushableAttribute(field); - return translateField(handler.nameOf(field instanceof FieldAttribute fa ? fa.exactAttribute() : field)); + String targetFieldName = handler.nameOf(field instanceof FieldAttribute fa ? fa.exactAttribute() : field); + return translateField(targetFieldName); } - /** - * Translates the field to a {@link WildcardQuery} using the first pattern in the list. - * Throws an {@link IllegalArgumentException} if the pattern list contains more than one pattern. - */ - private Query translateField(String targetFieldName) { - return new AutomatonQuery(source(), targetFieldName, pattern().createAutomaton(caseInsensitive()), getAutomatonDescription()); + private boolean supportsPushdown(TransportVersion version) { + return version == null || expressionTransportSupported(version); + } + + @Override + public org.apache.lucene.search.Query asLuceneQuery( + MappedFieldType fieldType, + RewriteMethod constantScoreRewrite, + SearchExecutionContext context + ) { + return fieldType.automatonQuery( + automatonSupplier, + characterRunAutomatonSupplier, + constantScoreRewrite, + context, + getLuceneQueryDescription() + ); } - private String getAutomatonDescription() { + private String getLuceneQueryDescription() { // we use the information used to create the automaton to describe the query here String patternDesc = pattern().patternList().stream().map(WildcardPattern::pattern).collect(Collectors.joining("\", \"")); return "LIKE(\"" + patternDesc + "\"), caseInsensitive=" + caseInsensitive(); } + + /** + * Translates the field to a {@link WildcardQuery} using the first pattern in the list. + * Throws an {@link IllegalArgumentException} if the pattern list contains more than one pattern. + */ + private Query translateField(String targetFieldName) { + return new ExpressionQuery(source(), targetFieldName, this); + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/ExpressionQuery.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/ExpressionQuery.java new file mode 100644 index 0000000000000..f3051b36adc06 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/ExpressionQuery.java @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.esql.io.stream; + +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.querydsl.query.Query; +import org.elasticsearch.xpack.esql.core.tree.Source; + +import java.util.Objects; + +/** + * Implements an Expression query, which matches documents based on a given expression. + */ +public class ExpressionQuery extends Query { + + private final String targetFieldName; + private final Expression expression; + + public ExpressionQuery(Source source, String targetFieldName, Expression expression) { + super(source); + this.targetFieldName = targetFieldName; + this.expression = expression; + } + + public String field() { + return targetFieldName; + } + + @Override + protected QueryBuilder asBuilder() { + return new ExpressionQueryBuilder(targetFieldName, expression); + } + + @Override + public int hashCode() { + return Objects.hash(targetFieldName, expression); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (obj == null || getClass() != obj.getClass()) { + return false; + } + + ExpressionQuery other = (ExpressionQuery) obj; + return Objects.equals(targetFieldName, other.targetFieldName) && Objects.equals(expression, other.expression); + } + + @Override + protected String innerToString() { + return "ExpressionQuery{" + "field='" + targetFieldName + '\'' + '}'; + } + + @Override + public boolean containsPlan() { + return true; + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/ExpressionQueryBuilder.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/ExpressionQueryBuilder.java new file mode 100644 index 0000000000000..7c4d26f2dff86 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/ExpressionQueryBuilder.java @@ -0,0 +1,123 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.io.stream; + +import org.apache.lucene.search.MatchNoDocsQuery; +import org.apache.lucene.search.Query; +import org.elasticsearch.TransportVersion; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.query.AbstractQueryBuilder; +import org.elasticsearch.index.query.MultiTermQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xpack.esql.capabilities.TranslationAware; +import org.elasticsearch.xpack.esql.core.expression.Expression; + +import java.io.IOException; +import java.util.Objects; + +import static org.apache.lucene.search.MultiTermQuery.CONSTANT_SCORE_REWRITE; + +/** + * Implements an Expression query builder, which matches documents based on a given expression. + * The expression itself must provide the {@link TranslationAware#asLuceneQuery} interface to be translated into a Lucene query. + * It allows for serialization of the expression and generate an AutomatonQuery on the data node + * as Automaton does not support serialization. + */ +public class ExpressionQueryBuilder extends AbstractQueryBuilder implements MultiTermQueryBuilder { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( + QueryBuilder.class, + "expressionQueryBuilder", + ExpressionQueryBuilder::new + ); + private final String fieldName; + private final Expression expression; + + public ExpressionQueryBuilder(String fieldName, Expression expression) { + if (Strings.isEmpty(fieldName)) { + throw new IllegalArgumentException("field name is null or empty"); + } + if (expression == null) { + throw new IllegalArgumentException("expression cannot be null"); + } + this.fieldName = fieldName; + this.expression = expression; + } + + /** + * Read from a stream. + */ + private ExpressionQueryBuilder(StreamInput in) throws IOException { + super(in); + fieldName = in.readString(); + assert in instanceof PlanStreamInput; + this.expression = in.readNamedWriteable(Expression.class); + } + + public Expression getExpression() { + return expression; + } + + @Override + protected void doWriteTo(StreamOutput out) throws IOException { + out.writeString(this.fieldName); + assert out instanceof PlanStreamOutput; + out.writeNamedWriteable(expression); + } + + @Override + public String fieldName() { + return fieldName; + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + + @Override + protected void doXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(ENTRY.name); // Use the appropriate query name + builder.field("field", fieldName); + builder.field("expression", expression.toString()); + builder.endObject(); + } + + @Override + protected Query doToQuery(SearchExecutionContext context) { + if (expression instanceof TranslationAware translationAware) { + MappedFieldType fieldType = context.getFieldType(fieldName); + if (fieldType == null) { + return new MatchNoDocsQuery("Field [" + fieldName + "] does not exist"); + } + return translationAware.asLuceneQuery(fieldType, CONSTANT_SCORE_REWRITE, context); + } else { + throw new UnsupportedOperationException("ExpressionQueryBuilder does not support non-automaton expressions"); + } + } + + @Override + protected int doHashCode() { + return Objects.hash(fieldName, expression); + } + + @Override + protected boolean doEquals(ExpressionQueryBuilder other) { + return Objects.equals(fieldName, other.fieldName) && Objects.equals(expression, other.expression); + } + + @Override + public TransportVersion getMinimalSupportedVersion() { + throw new UnsupportedOperationException("AutomatonQueryBuilder does not support getMinimalSupportedVersion"); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamWrapperQueryBuilder.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamWrapperQueryBuilder.java new file mode 100644 index 0000000000000..a99e42496c5db --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamWrapperQueryBuilder.java @@ -0,0 +1,111 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.io.stream; + +import org.apache.lucene.search.Query; +import org.elasticsearch.TransportVersion; +import org.elasticsearch.TransportVersions; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xpack.esql.session.Configuration; + +import java.io.IOException; + +import static org.elasticsearch.index.query.WildcardQueryBuilder.expressionTransportSupported; + +/** + * A {@link QueryBuilder} that wraps another {@linkplain QueryBuilder} + * so it read with a {@link PlanStreamInput}. + */ +public class PlanStreamWrapperQueryBuilder implements QueryBuilder { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( + QueryBuilder.class, + "planwrapper", + PlanStreamWrapperQueryBuilder::new + ); + + private final Configuration configuration; + private final QueryBuilder next; + + public PlanStreamWrapperQueryBuilder(Configuration configuration, QueryBuilder next) { + this.configuration = configuration; + this.next = next; + } + + public PlanStreamWrapperQueryBuilder(StreamInput in) throws IOException { + configuration = Configuration.readWithoutTables(in); + PlanStreamInput planStreamInput = new PlanStreamInput(in, in.namedWriteableRegistry(), configuration); + next = planStreamInput.readNamedWriteable(QueryBuilder.class); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + configuration.withoutTables().writeTo(out); + new PlanStreamOutput(out, configuration).writeNamedWriteable(next); + } + + @Override + public TransportVersion getMinimalSupportedVersion() { + return TransportVersions.ESQL_FIXED_INDEX_LIKE_8_19; + } + + @Override + public boolean supportsVersion(TransportVersion version) { + return expressionTransportSupported(version); + } + + @Override + public Query toQuery(SearchExecutionContext context) throws IOException { + return next.toQuery(context); + } + + @Override + public QueryBuilder queryName(String queryName) { + next.queryName(queryName); + return this; + } + + @Override + public String queryName() { + return next.queryName(); + } + + @Override + public float boost() { + return next.boost(); + } + + @Override + public QueryBuilder boost(float boost) { + next.boost(boost); + return this; + } + + @Override + public String getName() { + return getWriteableName(); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return next.toXContent(builder, params); + } + + public QueryBuilder next() { + return next; + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalOptimizerContext.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalOptimizerContext.java index 22e07b45310fb..1be024c9af76a 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalOptimizerContext.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalOptimizerContext.java @@ -8,7 +8,8 @@ package org.elasticsearch.xpack.esql.optimizer; import org.elasticsearch.xpack.esql.core.expression.FoldContext; +import org.elasticsearch.xpack.esql.plugin.EsqlFlags; import org.elasticsearch.xpack.esql.session.Configuration; import org.elasticsearch.xpack.esql.stats.SearchStats; -public record LocalPhysicalOptimizerContext(Configuration configuration, FoldContext foldCtx, SearchStats searchStats) {} +public record LocalPhysicalOptimizerContext(EsqlFlags flags, Configuration configuration, FoldContext foldCtx, SearchStats searchStats) {} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/EnableSpatialDistancePushdown.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/EnableSpatialDistancePushdown.java index 1e976ca2e6263..3e087bf64c1a0 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/EnableSpatialDistancePushdown.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/EnableSpatialDistancePushdown.java @@ -77,14 +77,14 @@ public class EnableSpatialDistancePushdown extends PhysicalOptimizerRules.Parame protected PhysicalPlan rule(FilterExec filterExec, LocalPhysicalOptimizerContext ctx) { PhysicalPlan plan = filterExec; if (filterExec.child() instanceof EsQueryExec esQueryExec) { - plan = rewrite(ctx.foldCtx(), filterExec, esQueryExec, LucenePushdownPredicates.from(ctx.searchStats())); + plan = rewrite(ctx.foldCtx(), filterExec, esQueryExec, LucenePushdownPredicates.from(ctx.searchStats(), ctx.flags())); } else if (filterExec.child() instanceof EvalExec evalExec && evalExec.child() instanceof EsQueryExec esQueryExec) { plan = rewriteBySplittingFilter( ctx.foldCtx(), filterExec, evalExec, esQueryExec, - LucenePushdownPredicates.from(ctx.searchStats()) + LucenePushdownPredicates.from(ctx.searchStats(), ctx.flags()) ); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/LucenePushdownPredicates.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/LucenePushdownPredicates.java index a476086980534..39e7da7d06d33 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/LucenePushdownPredicates.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/LucenePushdownPredicates.java @@ -13,6 +13,7 @@ import org.elasticsearch.xpack.esql.core.expression.TypedAttribute; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.util.Check; +import org.elasticsearch.xpack.esql.plugin.EsqlFlags; import org.elasticsearch.xpack.esql.stats.SearchStats; /** @@ -31,6 +32,8 @@ */ public interface LucenePushdownPredicates { /** + EsqlFlags flags(); + * For TEXT fields, we need to check if the field has a subfield of type KEYWORD that can be used instead. */ boolean hasExactSubfield(FieldAttribute attr); @@ -101,8 +104,23 @@ static String pushableAttributeName(TypedAttribute attribute) { * In particular, it assumes TEXT fields have no exact subfields (underlying keyword field), * and that isAggregatable means indexed and has hasDocValues. */ - LucenePushdownPredicates DEFAULT = new LucenePushdownPredicates() { + LucenePushdownPredicates DEFAULT = forCanMatch(null, new EsqlFlags(true)); + + /** + * A {@link LucenePushdownPredicates} for use with the {@code can_match} phase. + */ + static LucenePushdownPredicates forCanMatch(TransportVersion minTransportVersion, EsqlFlags flags) { + return new LucenePushdownPredicates() { + @Override + public TransportVersion minTransportVersion() { + return minTransportVersion; + } @Override + public EsqlFlags flags() { + return flags; + } + + @Override public boolean hasExactSubfield(FieldAttribute attr) { return false; } @@ -129,11 +147,16 @@ public boolean canUseEqualityOnSyntheticSourceDelegate(FieldAttribute attr, Stri * If we have access to {@link SearchStats} over a collection of shards, we can make more fine-grained decisions about what can be * pushed down. This should open up more opportunities for lucene pushdown. */ - static LucenePushdownPredicates from(SearchStats stats) { + static LucenePushdownPredicates from(SearchStats stats, EsqlFlags flags) { // TODO: use FieldAttribute#fieldName, otherwise this doesn't apply to field attributes used for union types. // C.f. https://github.com/elastic/elasticsearch/issues/128905 return new LucenePushdownPredicates() { @Override + } + + @Override + public EsqlFlags flags() { + return flags; public boolean hasExactSubfield(FieldAttribute attr) { return stats.hasExactSubfield(new FieldAttribute.FieldName(attr.name())); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushFiltersToSource.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushFiltersToSource.java index 1f8341c4768d2..ba382b9800ece 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushFiltersToSource.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushFiltersToSource.java @@ -53,7 +53,7 @@ protected PhysicalPlan rule(FilterExec filterExec, LocalPhysicalOptimizerContext } private static PhysicalPlan planFilterExec(FilterExec filterExec, EsQueryExec queryExec, LocalPhysicalOptimizerContext ctx) { - LucenePushdownPredicates pushdownPredicates = LucenePushdownPredicates.from(ctx.searchStats()); + LucenePushdownPredicates pushdownPredicates = LucenePushdownPredicates.from(ctx.searchStats(), ctx.flags()); List pushable = new ArrayList<>(); List nonPushable = new ArrayList<>(); for (Expression exp : splitAnd(filterExec.condition())) { @@ -75,7 +75,7 @@ private static PhysicalPlan planFilterExec( EsQueryExec queryExec, LocalPhysicalOptimizerContext ctx ) { - LucenePushdownPredicates pushdownPredicates = LucenePushdownPredicates.from(ctx.searchStats()); + LucenePushdownPredicates pushdownPredicates = LucenePushdownPredicates.from(ctx.searchStats(), ctx.flags()); AttributeMap aliasReplacedBy = getAliasReplacedBy(evalExec); List pushable = new ArrayList<>(); List nonPushable = new ArrayList<>(); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushTopNToSource.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushTopNToSource.java index 02d2f49605ced..8ec6d6b4bee39 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushTopNToSource.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushTopNToSource.java @@ -63,7 +63,7 @@ public class PushTopNToSource extends PhysicalOptimizerRules.ParameterizedOptimi @Override protected PhysicalPlan rule(TopNExec topNExec, LocalPhysicalOptimizerContext ctx) { - Pushable pushable = evaluatePushable(ctx.foldCtx(), topNExec, LucenePushdownPredicates.from(ctx.searchStats())); + Pushable pushable = evaluatePushable(ctx.foldCtx(), topNExec, LucenePushdownPredicates.from(ctx.searchStats(), ctx.flags())); return pushable.rewrite(topNExec); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlParser.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlParser.java index 53a421e0307b6..6b7d483a99aaa 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlParser.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlParser.java @@ -21,6 +21,7 @@ import org.elasticsearch.xpack.esql.core.util.StringUtils; import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; +import org.elasticsearch.xpack.esql.session.Configuration; import org.elasticsearch.xpack.esql.telemetry.PlanTelemetry; import java.util.BitSet; @@ -98,20 +99,20 @@ public void setEsqlConfig(EsqlConfig config) { } // testing utility - public LogicalPlan createStatement(String query) { - return createStatement(query, new QueryParams()); + public LogicalPlan createStatement(String query, Configuration configuration) { + return createStatement(query, new QueryParams(), configuration); } // testing utility - public LogicalPlan createStatement(String query, QueryParams params) { - return createStatement(query, params, new PlanTelemetry(new EsqlFunctionRegistry())); + public LogicalPlan createStatement(String query, QueryParams params, Configuration configuration) { + return createStatement(query, params, new PlanTelemetry(new EsqlFunctionRegistry()), configuration); } - public LogicalPlan createStatement(String query, QueryParams params, PlanTelemetry metrics) { + public LogicalPlan createStatement(String query, QueryParams params, PlanTelemetry metrics, Configuration configuration) { if (log.isDebugEnabled()) { log.debug("Parsing as statement: {}", query); } - return invokeParser(query, params, metrics, EsqlBaseParser::singleStatement, AstBuilder::plan); + return invokeParser(query, params, metrics, EsqlBaseParser::singleStatement, AstBuilder::plan, configuration); } private T invokeParser( @@ -119,7 +120,8 @@ private T invokeParser( QueryParams params, PlanTelemetry metrics, Function parseFunction, - BiFunction result + BiFunction result, + Configuration configuration ) { if (query.length() > MAX_LENGTH) { throw new ParsingException("ESQL statement is too large [{} characters > {}]", query.length(), MAX_LENGTH); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java index 86fde5c3d3d1f..346664d047884 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java @@ -25,11 +25,13 @@ import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute; +import org.elasticsearch.xpack.esql.core.querydsl.query.Query; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.util.Holder; import org.elasticsearch.xpack.esql.core.util.Queries; import org.elasticsearch.xpack.esql.expression.predicate.Predicates; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamWrapperQueryBuilder; import org.elasticsearch.xpack.esql.optimizer.LocalLogicalOptimizerContext; import org.elasticsearch.xpack.esql.optimizer.LocalLogicalPlanOptimizer; import org.elasticsearch.xpack.esql.optimizer.LocalPhysicalOptimizerContext; @@ -48,6 +50,7 @@ import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan; import org.elasticsearch.xpack.esql.planner.mapper.LocalMapper; import org.elasticsearch.xpack.esql.planner.mapper.Mapper; +import org.elasticsearch.xpack.esql.plugin.EsqlFlags; import org.elasticsearch.xpack.esql.session.Configuration; import org.elasticsearch.xpack.esql.stats.SearchContextStats; import org.elasticsearch.xpack.esql.stats.SearchStats; @@ -137,17 +140,26 @@ private static void forEachRelation(PhysicalPlan plan, Consumer acti } public static PhysicalPlan localPlan( + EsqlFlags flags, List searchContexts, Configuration configuration, FoldContext foldCtx, PhysicalPlan plan ) { - return localPlan(configuration, foldCtx, plan, SearchContextStats.from(searchContexts)); + return localPlan(flags, configuration, foldCtx, plan, SearchContextStats.from(searchContexts)); } - public static PhysicalPlan localPlan(Configuration configuration, FoldContext foldCtx, PhysicalPlan plan, SearchStats searchStats) { + public static PhysicalPlan localPlan( + EsqlFlags flags, + Configuration configuration, + FoldContext foldCtx, + PhysicalPlan plan, + SearchStats searchStats + ) { final var logicalOptimizer = new LocalLogicalPlanOptimizer(new LocalLogicalOptimizerContext(configuration, foldCtx, searchStats)); - var physicalOptimizer = new LocalPhysicalPlanOptimizer(new LocalPhysicalOptimizerContext(configuration, foldCtx, searchStats)); + var physicalOptimizer = new LocalPhysicalPlanOptimizer( + new LocalPhysicalOptimizerContext(flags, configuration, foldCtx, searchStats) + ); return localPlan(plan, logicalOptimizer, physicalOptimizer); } @@ -187,8 +199,13 @@ public static PhysicalPlan localPlan( /** * Extracts a filter that can be used to skip unmatched shards on the coordinator. */ - public static QueryBuilder canMatchFilter(PhysicalPlan plan) { - return detectFilter(plan, CoordinatorRewriteContext.SUPPORTED_FIELDS::contains); + public static QueryBuilder canMatchFilter( + EsqlFlags flags, + Configuration configuration, + TransportVersion minTransportVersion, + PhysicalPlan plan + ) { + return detectFilter(flags, configuration, minTransportVersion, plan, CoordinatorRewriteContext.SUPPORTED_FIELDS::contains); } /** @@ -196,11 +213,20 @@ public static QueryBuilder canMatchFilter(PhysicalPlan plan) { * We currently only use this filter for the @timestamp field, which is always a date field. Any tests that wish to use this should * take care to not use it with TEXT fields. */ - static QueryBuilder detectFilter(PhysicalPlan plan, Predicate fieldName) { + static QueryBuilder detectFilter( + EsqlFlags flags, + Configuration configuration, + TransportVersion minTransportVersion, + PhysicalPlan plan, + Predicate fieldName + ) { // first position is the REST filter, the second the query filter final List requestFilters = new ArrayList<>(); + final LucenePushdownPredicates ctx = LucenePushdownPredicates.forCanMatch(minTransportVersion, flags); plan.forEachDown(FragmentExec.class, fe -> { - requestFilters.add(fe.esFilter()); + if (fe.esFilter() != null && fe.esFilter().supportsVersion(minTransportVersion)) { + requestFilters.add(fe.esFilter()); + } // detect filter inside the query fe.fragment().forEachUp(Filter.class, f -> { // the only filter that can be pushed down is that on top of the relation @@ -224,9 +250,12 @@ && translatable(exp, LucenePushdownPredicates.DEFAULT).finish() == TranslationAw } } if (matches.isEmpty() == false) { - requestFilters.add( - TRANSLATOR_HANDLER.asQuery(LucenePushdownPredicates.DEFAULT, Predicates.combineAnd(matches)).toQueryBuilder() - ); + Query qlQuery = TRANSLATOR_HANDLER.asQuery(ctx, Predicates.combineAnd(matches)); + QueryBuilder builder = qlQuery.toQueryBuilder(); + if (qlQuery.containsPlan()) { + builder = new PlanStreamWrapperQueryBuilder(configuration, builder); + } + requestFilters.add(builder); } }); }); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ClusterComputeHandler.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ClusterComputeHandler.java index cd4ff13700515..5064b2bbd101a 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ClusterComputeHandler.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ClusterComputeHandler.java @@ -250,6 +250,7 @@ void runComputeOnRemoteCluster( final String localSessionId = clusterAlias + ":" + globalSessionId; final PhysicalPlan coordinatorPlan = ComputeService.reductionPlan(plan, true); final AtomicReference finalResponse = new AtomicReference<>(); + final EsqlFlags flags = computeService.createFlags(); final long startTimeInNanos = System.nanoTime(); final Runnable cancelQueryOnFailure = computeService.cancelQueryOnFailure(parentTask); try (var computeListener = new ComputeListener(transportService.getThreadPool(), cancelQueryOnFailure, listener.map(profiles -> { @@ -269,6 +270,7 @@ void runComputeOnRemoteCluster( localSessionId, "remote_reduce", clusterAlias, + flags, List.of(), configuration, configuration.newFoldContext(), @@ -282,6 +284,7 @@ void runComputeOnRemoteCluster( localSessionId, clusterAlias, parentTask, + flags, configuration, plan, concreteIndices, diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeContext.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeContext.java index 86af106ea7e42..d3a69bdb59a93 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeContext.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeContext.java @@ -21,6 +21,7 @@ record ComputeContext( String sessionId, String taskDescription, String clusterAlias, + EsqlFlags flags, List searchContexts, Configuration configuration, FoldContext foldCtx, diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java index a48d953850eeb..72c1aca74ca1e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java @@ -138,12 +138,14 @@ public ComputeService( public void execute( String sessionId, CancellableTask rootTask, + EsqlFlags flags, PhysicalPlan physicalPlan, Configuration configuration, FoldContext foldContext, EsqlExecutionInfo execInfo, ActionListener listener ) { + Tuple coordinatorAndDataNodePlan = PlannerUtils.breakPlanBetweenCoordinatorAndDataNode( physicalPlan, configuration @@ -175,6 +177,7 @@ public void execute( newChildSession(sessionId), "single", LOCAL_CLUSTER, + flags, List.of(), configuration, foldContext, @@ -261,6 +264,7 @@ public void execute( sessionId, "final", LOCAL_CLUSTER, + flags, List.of(), configuration, foldContext, @@ -277,6 +281,7 @@ public void execute( sessionId, LOCAL_CLUSTER, rootTask, + flags, configuration, dataNodePlan, Set.of(localConcreteIndices.indices()), @@ -424,7 +429,13 @@ public SourceProvider createSourceProvider() { LOGGER.debug("Received physical plan:\n{}", plan); - var localPlan = PlannerUtils.localPlan(context.searchExecutionContexts(), context.configuration(), context.foldCtx(), plan); + var localPlan = PlannerUtils.localPlan( + context.flags(), + context.searchExecutionContexts(), + context.configuration(), + context.foldCtx(), + plan + ); // the planner will also set the driver parallelism in LocalExecutionPlanner.LocalExecutionPlan (used down below) // it's doing this in the planning of EsQueryExec (the source of the data) // see also EsPhysicalOperationProviders.sourcePhysicalOperation @@ -497,6 +508,10 @@ CancellableTask createGroupTask(Task parentTask, Supplier description) t } } + public EsqlFlags createFlags() { + return new EsqlFlags(clusterService.getClusterSettings()); + } + private static class ComputeGroupTaskRequest extends TransportRequest { private final Supplier parentDescription; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/DataNodeComputeHandler.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/DataNodeComputeHandler.java index f6216684a971d..d0811c75e6c95 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/DataNodeComputeHandler.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/DataNodeComputeHandler.java @@ -95,6 +95,7 @@ void startComputeOnDataNodes( String sessionId, String clusterAlias, CancellableTask parentTask, + EsqlFlags flags, Configuration configuration, PhysicalPlan dataNodePlan, Set concreteIndices, @@ -111,7 +112,7 @@ void startComputeOnDataNodes( esqlExecutor, parentTask, originalIndices, - PlannerUtils.canMatchFilter(dataNodePlan), + PlannerUtils.canMatchFilter(flags, configuration, clusterService.state().getMinTransportVersion(), dataNodePlan), clusterAlias, configuration.allowPartialResults(), maxConcurrentNodesPerCluster == null ? -1 : maxConcurrentNodesPerCluster, @@ -214,6 +215,7 @@ protected void sendRequest( } private class DataNodeRequestExecutor { + private final EsqlFlags flags; private final DataNodeRequest request; private final CancellableTask parentTask; private final ExchangeSinkHandler exchangeSink; @@ -224,6 +226,7 @@ private class DataNodeRequestExecutor { private final Map shardLevelFailures; DataNodeRequestExecutor( + EsqlFlags flags, DataNodeRequest request, CancellableTask parentTask, ExchangeSinkHandler exchangeSink, @@ -232,6 +235,7 @@ private class DataNodeRequestExecutor { Map shardLevelFailures, ComputeListener computeListener ) { + this.flags = flags; this.request = request; this.parentTask = parentTask; this.exchangeSink = exchangeSink; @@ -295,6 +299,7 @@ public void onFailure(Exception e) { sessionId, "data", clusterAlias, + flags, searchContexts, configuration, configuration.newFoldContext(), @@ -415,7 +420,9 @@ private void runComputeOnDataNode( try { // run compute with target shards var internalSink = exchangeService.createSinkHandler(request.sessionId(), request.pragmas().exchangeBufferSize()); + EsqlFlags flags = computeService.createFlags(); DataNodeRequestExecutor dataNodeRequestExecutor = new DataNodeRequestExecutor( + flags, request, task, internalSink, @@ -439,6 +446,7 @@ private void runComputeOnDataNode( request.sessionId(), "node_reduce", request.clusterAlias(), + flags, List.of(), request.configuration(), new FoldContext(request.pragmas().foldLimit().getBytes()), diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlFlags.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlFlags.java new file mode 100644 index 0000000000000..09ef93d56e9ee --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlFlags.java @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.plugin; + +import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.settings.Setting; + +public class EsqlFlags { + public static final Setting ESQL_STRING_LIKE_ON_INDEX = Setting.boolSetting( + "esql.query.string_like_on_index", + true, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + + private final boolean stringLikeOnIndex; + + public EsqlFlags(boolean stringLikeOnIndex) { + this.stringLikeOnIndex = stringLikeOnIndex; + } + + public EsqlFlags(ClusterSettings settings) { + this.stringLikeOnIndex = settings.get(ESQL_STRING_LIKE_ON_INDEX); + } + + public boolean stringLikeOnIndex() { + return stringLikeOnIndex; + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java index c1988bca185e5..8b5b754b11e63 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java @@ -68,6 +68,8 @@ import org.elasticsearch.xpack.esql.enrich.LookupFromIndexOperator; import org.elasticsearch.xpack.esql.execution.PlanExecutor; import org.elasticsearch.xpack.esql.expression.ExpressionWritables; +import org.elasticsearch.xpack.esql.io.stream.ExpressionQueryBuilder; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamWrapperQueryBuilder; import org.elasticsearch.xpack.esql.plan.PlanWritables; import org.elasticsearch.xpack.esql.querydsl.query.SingleValueQuery; import org.elasticsearch.xpack.esql.querylog.EsqlQueryLog; @@ -247,9 +249,10 @@ public List> getSettings() { ESQL_QUERYLOG_THRESHOLD_DEBUG_SETTING, ESQL_QUERYLOG_THRESHOLD_INFO_SETTING, ESQL_QUERYLOG_THRESHOLD_WARN_SETTING, - ESQL_QUERYLOG_INCLUDE_USER_SETTING, + ESQL_QUERYLOG_INCLUDE_USER_SETTING, STORED_FIELDS_SEQUENTIAL_PROPORTION, - DEFAULT_DATA_PARTITIONING + DEFAULT_DATA_PARTITIONING, + EsqlFlags.ESQL_STRING_LIKE_ON_INDEX ); } @@ -307,6 +310,8 @@ public List getNamedWriteables() { entries.add(AsyncOperator.Status.ENTRY); entries.add(EnrichLookupOperator.Status.ENTRY); entries.add(LookupFromIndexOperator.Status.ENTRY); + entries.add(ExpressionQueryBuilder.ENTRY); + entries.add(PlanStreamWrapperQueryBuilder.ENTRY); entries.addAll(ExpressionWritables.getNamedWriteables()); entries.addAll(PlanWritables.getNamedWriteables()); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java index 06986c856e7dd..d19b9b5787885 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java @@ -170,6 +170,7 @@ public TransportEsqlQueryAction( defaultAllowPartialResults = EsqlPlugin.QUERY_ALLOW_PARTIAL_RESULTS.get(clusterService.getSettings()); clusterService.getClusterSettings() .addSettingsUpdateConsumer(EsqlPlugin.QUERY_ALLOW_PARTIAL_RESULTS, v -> defaultAllowPartialResults = v); + } @Override @@ -209,6 +210,7 @@ private void innerExecute(Task task, EsqlQueryRequest request, ActionListener computeService.execute( sessionId, (CancellableTask) task, + flags, plan, configuration, foldCtx, diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/EqualsSyntheticSourceDelegate.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/EqualsSyntheticSourceDelegate.java index 1db12c873a763..812e741c64d29 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/EqualsSyntheticSourceDelegate.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/EqualsSyntheticSourceDelegate.java @@ -35,6 +35,11 @@ protected String innerToString() { return fieldName + "(delegate):" + value; } + @Override + public boolean containsPlan() { + return false; + } + private class Builder extends BaseTermQueryBuilder { private Builder(String name, String value) { super(name, value); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/KnnQuery.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/KnnQuery.java index aa0e896dfc013..2946af2ac5c23 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/KnnQuery.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/KnnQuery.java @@ -81,4 +81,9 @@ public int hashCode() { public boolean scorable() { return true; } + + @Override + public boolean containsPlan() { + return false; + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/KqlQuery.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/KqlQuery.java index 7ef272f84ec9f..56229ec325d73 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/KqlQuery.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/KqlQuery.java @@ -87,4 +87,9 @@ protected String innerToString() { public boolean scorable() { return true; } + + @Override + public boolean containsPlan() { + return false; + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/MatchPhraseQuery.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/MatchPhraseQuery.java index be6f244ac4acf..6a10542687490 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/MatchPhraseQuery.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/MatchPhraseQuery.java @@ -108,4 +108,9 @@ public Map options() { public boolean scorable() { return true; } + + @Override + public boolean containsPlan() { + return false; + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/MatchQuery.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/MatchQuery.java index 57489cc930bf2..d3d9d810d78fc 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/MatchQuery.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/MatchQuery.java @@ -130,4 +130,9 @@ public Map options() { public boolean scorable() { return true; } + + @Override + public boolean containsPlan() { + return false; + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/MultiMatchQuery.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/MultiMatchQuery.java index 33042bd5efe4e..f02ed1f070a3a 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/MultiMatchQuery.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/MultiMatchQuery.java @@ -98,4 +98,9 @@ public boolean equals(Object obj) { protected String innerToString() { return fields + ":" + query; } + + @Override + public boolean containsPlan() { + return false; + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuery.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuery.java index f3418b65a504c..de83b41f96b0b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuery.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuery.java @@ -125,6 +125,11 @@ public int hashCode() { return Objects.hash(super.hashCode(), next, field, useSyntheticSourceDelegate); } + @Override + public boolean containsPlan() { + return next.containsPlan(); + } + public abstract static class AbstractBuilder extends AbstractQueryBuilder { private final QueryBuilder next; private final String field; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SpatialRelatesQuery.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SpatialRelatesQuery.java index bf0fa72d3af41..98939b5a495f6 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SpatialRelatesQuery.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SpatialRelatesQuery.java @@ -86,6 +86,11 @@ public ShapeRelation shapeRelation() { }; } + @Override + public boolean containsPlan() { + return false; + } + /** * This class is a minimal implementation of the QueryBuilder interface. * We only need the toQuery method, but ESQL makes extensive use of QueryBuilder and trimming that interface down for ESQL only would diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/TranslationAwareExpressionQuery.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/TranslationAwareExpressionQuery.java index d61101c2f594c..8106f6c6661aa 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/TranslationAwareExpressionQuery.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/TranslationAwareExpressionQuery.java @@ -40,4 +40,9 @@ public boolean scorable() { // All Full Text Functions are translated to queries using this method return true; } + + @Override + public boolean containsPlan() { + return false; + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/Configuration.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/Configuration.java index 0434d472105e0..b6fb78ff54f65 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/Configuration.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/Configuration.java @@ -209,6 +209,23 @@ public Map> tables() { return tables; } + public Configuration withoutTables() { + return new Configuration( + zoneId, + locale, + username, + clusterName, + pragmas, + resultTruncationMaxSize, + resultTruncationDefaultSize, + query, + profile, + Map.of(), + queryStartTimeNanos, + allowPartialResults + ); + } + /** * Enable profiling, sacrificing performance to return information about * what operations are taking the most time. @@ -307,4 +324,11 @@ public String toString() { + '}'; } + /** + * Reads a {@link Configuration} that doesn't contain any {@link Configuration#tables()}. + */ + public static Configuration readWithoutTables(StreamInput in) throws IOException { + BlockStreamInput blockStreamInput = new BlockStreamInput(in, null); + return new Configuration(blockStreamInput); + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java index 26da6cb14d4cd..b0d97e2e2f385 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java @@ -295,7 +295,7 @@ private LocalRelation resultToPlan(LogicalPlan plan, Result result) { } private LogicalPlan parse(String query, QueryParams params) { - var parsed = new EsqlParser().createStatement(query, params, planTelemetry); + var parsed = new EsqlParser().createStatement(query, params, planTelemetry, configuration); LOGGER.debug("Parsed logical plan:\n{}", parsed); return parsed; } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java index 3c73f95f1ec83..30e1cd804f946 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java @@ -88,6 +88,7 @@ import org.elasticsearch.xpack.esql.planner.TestPhysicalOperationProviders; import org.elasticsearch.xpack.esql.planner.mapper.Mapper; import org.elasticsearch.xpack.esql.plugin.EsqlFeatures; +import org.elasticsearch.xpack.esql.plugin.EsqlFlags; import org.elasticsearch.xpack.esql.plugin.QueryPragmas; import org.elasticsearch.xpack.esql.session.Configuration; import org.elasticsearch.xpack.esql.session.EsqlSession; @@ -533,7 +534,7 @@ private static TestPhysicalOperationProviders testOperationProviders( } private ActualResults executePlan(BigArrays bigArrays) throws Exception { - LogicalPlan parsed = parser.createStatement(testCase.query); + LogicalPlan parsed = parser.createStatement(testCase.query, EsqlTestUtils.TEST_CFG); var testDatasets = testDatasets(parsed); LogicalPlan analyzed = analyzedPlan(parsed, testDatasets); @@ -680,7 +681,7 @@ void executeSubPlan( var searchStats = new DisabledSearchStats(); var logicalTestOptimizer = new LocalLogicalPlanOptimizer(new LocalLogicalOptimizerContext(configuration, foldCtx, searchStats)); var physicalTestOptimizer = new TestLocalPhysicalPlanOptimizer( - new LocalPhysicalOptimizerContext(configuration, foldCtx, searchStats) + new LocalPhysicalOptimizerContext(new EsqlFlags(true), configuration, foldCtx, searchStats) ); var csvDataNodePhysicalPlan = PlannerUtils.localPlan(dataNodePlan, logicalTestOptimizer, physicalTestOptimizer); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTestUtils.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTestUtils.java index b7b58b05cb29b..d9891fe5ec5fc 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTestUtils.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTestUtils.java @@ -129,7 +129,7 @@ public static LogicalPlan analyze(String query, String index, String mapping) { } public static LogicalPlan analyze(String query, Analyzer analyzer) { - var plan = new EsqlParser().createStatement(query); + var plan = new EsqlParser().createStatement(query, configuration(query)); // System.out.println(plan); var analyzed = analyzer.analyze(plan); // System.out.println(analyzed); @@ -137,7 +137,7 @@ public static LogicalPlan analyze(String query, Analyzer analyzer) { } public static LogicalPlan analyze(String query, String mapping, QueryParams params) { - var plan = new EsqlParser().createStatement(query, params); + var plan = new EsqlParser().createStatement(query, params, configuration(query)); var analyzer = analyzer(loadMapping(mapping, "test"), TEST_VERIFIER, configuration(query)); return analyzer.analyze(plan); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/ParsingTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/ParsingTests.java index 5cfa311ec4f55..3412bcc2f51b9 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/ParsingTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/ParsingTests.java @@ -92,7 +92,7 @@ public void testInlineCast() throws IOException { if (EsqlDataTypeConverter.converterFunctionFactory(expectedType) == null) { continue; } - LogicalPlan plan = parser.createStatement("ROW a = 1::" + nameOrAlias); + LogicalPlan plan = parser.createStatement("ROW a = 1::" + nameOrAlias, TEST_CFG); Row row = as(plan, Row.class); assertThat(row.fields(), hasSize(1)); Function functionCall = (Function) row.fields().get(0).child(); @@ -178,7 +178,7 @@ public void testInvalidSample() { } private String error(String query) { - ParsingException e = expectThrows(ParsingException.class, () -> defaultAnalyzer.analyze(parser.createStatement(query))); + ParsingException e = expectThrows(ParsingException.class, () -> defaultAnalyzer.analyze(parser.createStatement(query, TEST_CFG))); String message = e.getMessage(); assertTrue(message.startsWith("line ")); return message.substring("line ".length()); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java index 9ec028e19d6aa..21bb083c726ee 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java @@ -36,6 +36,7 @@ import java.util.Map; import java.util.Set; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.TEST_CFG; import static org.elasticsearch.xpack.esql.EsqlTestUtils.paramAsConstant; import static org.elasticsearch.xpack.esql.EsqlTestUtils.withDefaultLimitWarning; import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.loadMapping; @@ -2197,7 +2198,7 @@ private void query(String query) { } private void query(String query, Analyzer analyzer) { - analyzer.analyze(parser.createStatement(query)); + analyzer.analyze(parser.createStatement(query, TEST_CFG)); } private String error(String query) { @@ -2228,7 +2229,7 @@ private String error(String query, Analyzer analyzer, Class Throwable e = expectThrows( exception, "Expected error for query [" + query + "] but no error was raised", - () -> analyzer.analyze(parser.createStatement(query, new QueryParams(parameters))) + () -> analyzer.analyze(parser.createStatement(query, new QueryParams(parameters), TEST_CFG)) ); assertThat(e, instanceOf(exception)); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/CheckLicenseTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/CheckLicenseTests.java index 68a6f38cdd69a..039b34252c12e 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/CheckLicenseTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/CheckLicenseTests.java @@ -79,7 +79,7 @@ public EsqlFunctionRegistry snapshotRegistry() { } }; - var plan = parser.createStatement(esql); + var plan = parser.createStatement(esql, EsqlTestUtils.TEST_CFG); plan = plan.transformDown( Limit.class, l -> Objects.equals(l.limit().fold(FoldContext.small()), 10) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithTests.java index d6a6106f9372e..0efd8daaacaa0 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithTests.java @@ -134,6 +134,6 @@ public void testLuceneQuery_NonFoldableSuffix_Translatable() { var query = function.asQuery(LucenePushdownPredicates.DEFAULT, TranslatorHandler.TRANSLATOR_HANDLER); - assertThat(query, equalTo(new WildcardQuery(Source.EMPTY, "field", "*a\\*b\\?c\\\\"))); + assertThat(query, equalTo(new WildcardQuery(Source.EMPTY, "field", "*a\\*b\\?c\\\\", false, false))); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWithTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWithTests.java index c716457dd8378..67fb9f0c41f26 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWithTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWithTests.java @@ -94,6 +94,6 @@ public void testLuceneQuery_NonFoldablePrefix_Translatable() { var query = function.asQuery(LucenePushdownPredicates.DEFAULT, TranslatorHandler.TRANSLATOR_HANDLER); - assertThat(query, equalTo(new WildcardQuery(Source.EMPTY, "field", "a\\*b\\?c\\\\*"))); + assertThat(query, equalTo(new WildcardQuery(Source.EMPTY, "field", "a\\*b\\?c\\\\*", false, false))); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLikeListTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLikeListTests.java index cac1692f1a86e..9acb12eaa48e8 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLikeListTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLikeListTests.java @@ -22,12 +22,14 @@ import org.elasticsearch.xpack.esql.expression.function.FunctionName; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.WildcardLikeList; +import org.elasticsearch.xpack.esql.plugin.EsqlFlags; import java.util.ArrayList; import java.util.List; import java.util.function.Function; import java.util.function.Supplier; +import static org.elasticsearch.TransportVersions.V_8_17_0; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.startsWith; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLikeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLikeTests.java index a5869c86a2123..6548585e7e139 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLikeTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLikeTests.java @@ -89,7 +89,7 @@ protected Expression build(Source source, List args) { return buildWildcardLike(source, args); } - static Expression buildWildcardLike(Source source, List args) { + Expression buildWildcardLike(Source source, List args) { Expression expression = args.get(0); Literal pattern = (Literal) args.get(1); Literal caseInsensitive = args.size() > 2 ? (Literal) args.get(2) : null; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/AbstractLogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/AbstractLogicalPlanOptimizerTests.java index e3c7c5aca9e51..1877a774e8f34 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/AbstractLogicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/AbstractLogicalPlanOptimizerTests.java @@ -156,7 +156,7 @@ protected LogicalPlan plan(String query) { } protected LogicalPlan plan(String query, LogicalPlanOptimizer optimizer) { - var analyzed = analyzer.analyze(parser.createStatement(query)); + var analyzed = analyzer.analyze(parser.createStatement(query, EsqlTestUtils.TEST_CFG)); // System.out.println(analyzed); var optimized = optimizer.optimize(analyzed); // System.out.println(optimized); @@ -164,7 +164,7 @@ protected LogicalPlan plan(String query, LogicalPlanOptimizer optimizer) { } protected LogicalPlan planAirports(String query) { - var analyzed = analyzerAirports.analyze(parser.createStatement(query)); + var analyzed = analyzerAirports.analyze(parser.createStatement(query, EsqlTestUtils.TEST_CFG)); // System.out.println(analyzed); var optimized = logicalOptimizer.optimize(analyzed); // System.out.println(optimized); @@ -172,7 +172,7 @@ protected LogicalPlan planAirports(String query) { } protected LogicalPlan planExtra(String query) { - var analyzed = analyzerExtra.analyze(parser.createStatement(query)); + var analyzed = analyzerExtra.analyze(parser.createStatement(query, EsqlTestUtils.TEST_CFG)); // System.out.println(analyzed); var optimized = logicalOptimizer.optimize(analyzed); // System.out.println(optimized); @@ -180,7 +180,7 @@ protected LogicalPlan planExtra(String query) { } protected LogicalPlan planTypes(String query) { - return logicalOptimizer.optimize(analyzerTypes.analyze(parser.createStatement(query))); + return logicalOptimizer.optimize(analyzerTypes.analyze(parser.createStatement(query, EsqlTestUtils.TEST_CFG))); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java index 5e6a3567f4aea..8e76986c3c404 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java @@ -508,7 +508,7 @@ public void testSparseDocument() throws Exception { TEST_VERIFIER ); - var analyzed = analyzer.analyze(parser.createStatement(query)); + var analyzed = analyzer.analyze(parser.createStatement(query, EsqlTestUtils.TEST_CFG)); var optimized = logicalOptimizer.optimize(analyzed); var localContext = new LocalLogicalOptimizerContext(EsqlTestUtils.TEST_CFG, FoldContext.small(), searchStats); var plan = new LocalLogicalPlanOptimizer(localContext).localOptimize(optimized); @@ -785,7 +785,7 @@ private LocalRelation asEmptyRelation(Object o) { } private LogicalPlan plan(String query, Analyzer analyzer) { - var analyzed = analyzer.analyze(parser.createStatement(query)); + var analyzed = analyzer.analyze(parser.createStatement(query, EsqlTestUtils.TEST_CFG)); // System.out.println(analyzed); var optimized = logicalOptimizer.optimize(analyzed); // System.out.println(optimized); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java index 859b52368ea96..2e9edec64a209 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java @@ -85,6 +85,7 @@ import org.elasticsearch.xpack.esql.plan.physical.ProjectExec; import org.elasticsearch.xpack.esql.plan.physical.TopNExec; import org.elasticsearch.xpack.esql.planner.FilterTests; +import org.elasticsearch.xpack.esql.plugin.EsqlFlags; import org.elasticsearch.xpack.esql.plugin.QueryPragmas; import org.elasticsearch.xpack.esql.querydsl.query.SingleValueQuery; import org.elasticsearch.xpack.esql.rule.Rule; @@ -1878,6 +1879,7 @@ public void testToDateNanosPushDown() { assertThat(expected.toString(), is(esQuery.query().toString())); } + private boolean isMultiTypeEsField(Expression e) { return e instanceof FieldAttribute fa && fa.field() instanceof MultiTypeEsField; } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java index 9f990c301293e..670494c9f7f4c 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java @@ -5209,20 +5209,26 @@ public void testEmptyMappingIndex() { TEST_VERIFIER ); - var plan = logicalOptimizer.optimize(analyzer.analyze(parser.createStatement("from empty_test"))); + var plan = logicalOptimizer.optimize(analyzer.analyze(parser.createStatement("from empty_test", EsqlTestUtils.TEST_CFG))); as(plan, LocalRelation.class); assertThat(plan.output(), equalTo(NO_FIELDS)); - plan = logicalOptimizer.optimize(analyzer.analyze(parser.createStatement("from empty_test metadata _id | eval x = 1"))); + plan = logicalOptimizer.optimize( + analyzer.analyze(parser.createStatement("from empty_test metadata _id | eval x = 1", EsqlTestUtils.TEST_CFG)) + ); as(plan, LocalRelation.class); assertThat(Expressions.names(plan.output()), contains("_id", "x")); - plan = logicalOptimizer.optimize(analyzer.analyze(parser.createStatement("from empty_test metadata _id, _version | limit 5"))); + plan = logicalOptimizer.optimize( + analyzer.analyze(parser.createStatement("from empty_test metadata _id, _version | limit 5", EsqlTestUtils.TEST_CFG)) + ); as(plan, LocalRelation.class); assertThat(Expressions.names(plan.output()), contains("_id", "_version")); plan = logicalOptimizer.optimize( - analyzer.analyze(parser.createStatement("from empty_test | eval x = \"abc\" | enrich languages_idx on x")) + analyzer.analyze( + parser.createStatement("from empty_test | eval x = \"abc\" | enrich languages_idx on x", EsqlTestUtils.TEST_CFG) + ) ); LocalRelation local = as(plan, LocalRelation.class); assertThat(Expressions.names(local.output()), contains(NO_FIELDS.get(0).name(), "x", "language_code", "language_name")); @@ -5917,7 +5923,7 @@ private void doTestSimplifyComparisonArithmetics( private void assertSemanticMatching(String expected, String provided) { BinaryComparison bc = extractPlannedBinaryComparison(provided); - LogicalPlan exp = analyzerTypes.analyze(parser.createStatement("FROM types | WHERE " + expected)); + LogicalPlan exp = analyzerTypes.analyze(parser.createStatement("FROM types | WHERE " + expected, EsqlTestUtils.TEST_CFG)); assertSemanticMatching(bc, extractPlannedBinaryComparison(exp)); } @@ -5945,7 +5951,7 @@ private Expression getComparisonFromLogicalPlan(LogicalPlan plan) { private void assertNotSimplified(String comparison) { String query = "FROM types | WHERE " + comparison; Expression optimized = getComparisonFromLogicalPlan(planTypes(query)); - Expression raw = getComparisonFromLogicalPlan(analyzerTypes.analyze(parser.createStatement(query))); + Expression raw = getComparisonFromLogicalPlan(analyzerTypes.analyze(parser.createStatement(query, EsqlTestUtils.TEST_CFG))); assertTrue(raw.semanticEquals(optimized)); } @@ -6095,7 +6101,7 @@ public void testReplaceStringCasingWithInsensitiveEqualsLowerTrue() { public void testReplaceStringCasingWithInsensitiveEqualsEquals() { for (var fn : List.of("TO_LOWER", "TO_UPPER")) { var value = fn.equals("TO_LOWER") ? fn.toLowerCase(Locale.ROOT) : fn.toUpperCase(Locale.ROOT); - value += "🐔✈🔥🎉"; // these should not cause folding, they're not in the upper/lower char class + value += "�✈��"; // these should not cause folding, they're not in the upper/lower char class var plan = optimizedPlan("FROM test | WHERE " + fn + "(first_name) == \"" + value + "\""); var limit = as(plan, Limit.class); var filter = as(limit.child(), Filter.class); @@ -6110,7 +6116,7 @@ public void testReplaceStringCasingWithInsensitiveEqualsEquals() { public void testReplaceStringCasingWithInsensitiveEqualsNotEquals() { for (var fn : List.of("TO_LOWER", "TO_UPPER")) { var value = fn.equals("TO_LOWER") ? fn.toLowerCase(Locale.ROOT) : fn.toUpperCase(Locale.ROOT); - value += "🐔✈🔥🎉"; // these should not cause folding, they're not in the upper/lower char class + value += "�✈��"; // these should not cause folding, they're not in the upper/lower char class var plan = optimizedPlan("FROM test | WHERE " + fn + "(first_name) != \"" + value + "\""); var limit = as(plan, Limit.class); var filter = as(limit.child(), Filter.class); @@ -6642,7 +6648,7 @@ public void testMultipleLookupShadowing() { public void testTranslateMetricsWithoutGrouping() { assumeTrue("requires snapshot builds", Build.current().isSnapshot()); var query = "METRICS k8s max(rate(network.total_bytes_in))"; - var plan = logicalOptimizer.optimize(metricsAnalyzer.analyze(parser.createStatement(query))); + var plan = logicalOptimizer.optimize(metricsAnalyzer.analyze(parser.createStatement(query, EsqlTestUtils.TEST_CFG))); Limit limit = as(plan, Limit.class); Aggregate finalAggs = as(limit.child(), Aggregate.class); Aggregate aggsByTsid = as(finalAggs.child(), Aggregate.class); @@ -6663,7 +6669,7 @@ public void testTranslateMetricsWithoutGrouping() { public void testTranslateMixedAggsWithoutGrouping() { assumeTrue("requires snapshot builds", Build.current().isSnapshot()); var query = "METRICS k8s max(rate(network.total_bytes_in)), max(network.cost)"; - var plan = logicalOptimizer.optimize(metricsAnalyzer.analyze(parser.createStatement(query))); + var plan = logicalOptimizer.optimize(metricsAnalyzer.analyze(parser.createStatement(query, EsqlTestUtils.TEST_CFG))); Limit limit = as(plan, Limit.class); Aggregate finalAggs = as(limit.child(), Aggregate.class); Aggregate aggsByTsid = as(finalAggs.child(), Aggregate.class); @@ -6688,7 +6694,7 @@ public void testTranslateMixedAggsWithoutGrouping() { public void testTranslateMixedAggsWithMathWithoutGrouping() { assumeTrue("requires snapshot builds", Build.current().isSnapshot()); var query = "METRICS k8s max(rate(network.total_bytes_in)), max(network.cost + 0.2) * 1.1"; - var plan = logicalOptimizer.optimize(metricsAnalyzer.analyze(parser.createStatement(query))); + var plan = logicalOptimizer.optimize(metricsAnalyzer.analyze(parser.createStatement(query, EsqlTestUtils.TEST_CFG))); Project project = as(plan, Project.class); Eval mulEval = as(project.child(), Eval.class); assertThat(mulEval.fields(), hasSize(1)); @@ -6725,7 +6731,7 @@ public void testTranslateMixedAggsWithMathWithoutGrouping() { public void testTranslateMetricsGroupedByOneDimension() { assumeTrue("requires snapshot builds", Build.current().isSnapshot()); var query = "METRICS k8s sum(rate(network.total_bytes_in)) BY cluster | SORT cluster | LIMIT 10"; - var plan = logicalOptimizer.optimize(metricsAnalyzer.analyze(parser.createStatement(query))); + var plan = logicalOptimizer.optimize(metricsAnalyzer.analyze(parser.createStatement(query, EsqlTestUtils.TEST_CFG))); TopN topN = as(plan, TopN.class); Aggregate aggsByCluster = as(topN.child(), Aggregate.class); assertThat(aggsByCluster.aggregates(), hasSize(2)); @@ -6749,7 +6755,7 @@ public void testTranslateMetricsGroupedByOneDimension() { public void testTranslateMetricsGroupedByTwoDimension() { assumeTrue("requires snapshot builds", Build.current().isSnapshot()); var query = "METRICS k8s avg(rate(network.total_bytes_in)) BY cluster, pod"; - var plan = logicalOptimizer.optimize(metricsAnalyzer.analyze(parser.createStatement(query))); + var plan = logicalOptimizer.optimize(metricsAnalyzer.analyze(parser.createStatement(query, EsqlTestUtils.TEST_CFG))); Project project = as(plan, Project.class); Eval eval = as(project.child(), Eval.class); assertThat(eval.fields(), hasSize(1)); @@ -6788,7 +6794,7 @@ public void testTranslateMetricsGroupedByTwoDimension() { public void testTranslateMetricsGroupedByTimeBucket() { assumeTrue("requires snapshot builds", Build.current().isSnapshot()); var query = "METRICS k8s sum(rate(network.total_bytes_in)) BY bucket(@timestamp, 1h)"; - var plan = logicalOptimizer.optimize(metricsAnalyzer.analyze(parser.createStatement(query))); + var plan = logicalOptimizer.optimize(metricsAnalyzer.analyze(parser.createStatement(query, EsqlTestUtils.TEST_CFG))); Limit limit = as(plan, Limit.class); Aggregate finalAgg = as(limit.child(), Aggregate.class); assertThat(finalAgg.aggregates(), hasSize(2)); @@ -6819,7 +6825,7 @@ METRICS k8s avg(rate(network.total_bytes_in)) BY pod, bucket(@timestamp, 5 minut | SORT cluster | LIMIT 10 """; - var plan = logicalOptimizer.optimize(metricsAnalyzer.analyze(parser.createStatement(query))); + var plan = logicalOptimizer.optimize(metricsAnalyzer.analyze(parser.createStatement(query, EsqlTestUtils.TEST_CFG))); Project project = as(plan, Project.class); TopN topN = as(project.child(), TopN.class); Eval eval = as(topN.child(), Eval.class); @@ -6858,7 +6864,7 @@ METRICS k8s avg(rate(network.total_bytes_in)), avg(network.cost) BY bucket(@time | SORT cluster | LIMIT 10 """; - var plan = logicalOptimizer.optimize(metricsAnalyzer.analyze(parser.createStatement(query))); + var plan = logicalOptimizer.optimize(metricsAnalyzer.analyze(parser.createStatement(query, EsqlTestUtils.TEST_CFG))); Project project = as(plan, Project.class); TopN topN = as(project.child(), TopN.class); Eval eval = as(topN.child(), Eval.class); @@ -6907,7 +6913,7 @@ METRICS k8s avg(round(1.05 * rate(network.total_bytes_in))) BY bucket(@timestamp | SORT cluster | LIMIT 10 """; - var plan = logicalOptimizer.optimize(metricsAnalyzer.analyze(parser.createStatement(query))); + var plan = logicalOptimizer.optimize(metricsAnalyzer.analyze(parser.createStatement(query, EsqlTestUtils.TEST_CFG))); Project project = as(plan, Project.class); TopN topN = as(project.child(), TopN.class); Eval evalDiv = as(topN.child(), Eval.class); @@ -6970,7 +6976,7 @@ METRICS k8s count(to_long(network.total_bytes_in)) BY bucket(@timestamp, 1 minut """); List plans = new ArrayList<>(); for (String query : queries) { - var plan = logicalOptimizer.optimize(metricsAnalyzer.analyze(parser.createStatement(query))); + var plan = logicalOptimizer.optimize(metricsAnalyzer.analyze(parser.createStatement(query, EsqlTestUtils.TEST_CFG))); plans.add(plan); } for (LogicalPlan plan : plans) { @@ -7560,7 +7566,7 @@ public void testPruneRedundantOrderBy() { | mv_expand x | sort y """; - LogicalPlan analyzed = analyzer.analyze(parser.createStatement(query)); + LogicalPlan analyzed = analyzer.analyze(parser.createStatement(query, EsqlTestUtils.TEST_CFG)); LogicalPlan optimized = rule.apply(analyzed); // check that all the redundant SORTs are removed in a single run diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/OptimizerRulesTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/OptimizerRulesTests.java index b36cb3f6c6a42..fe4fd96120a01 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/OptimizerRulesTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/OptimizerRulesTests.java @@ -8,6 +8,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.esql.EsqlTestUtils; import org.elasticsearch.xpack.esql.core.expression.Alias; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; @@ -128,7 +129,7 @@ protected Expression rule(Expression e, LogicalOptimizerContext ctx) { }; rule.apply( - new EsqlParser().createStatement("FROM index | EVAL x=f1+1 | KEEP x, f2 | LIMIT 1"), + new EsqlParser().createStatement("FROM index | EVAL x=f1+1 | KEEP x, f2 | LIMIT 1", EsqlTestUtils.TEST_CFG), new LogicalOptimizerContext(null, FoldContext.small()) ); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java index d5bd5314bb095..f734685cd36f9 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java @@ -136,6 +136,7 @@ import org.elasticsearch.xpack.esql.planner.LocalExecutionPlanner; import org.elasticsearch.xpack.esql.planner.PlannerUtils; import org.elasticsearch.xpack.esql.planner.mapper.Mapper; +import org.elasticsearch.xpack.esql.plugin.EsqlFlags; import org.elasticsearch.xpack.esql.plugin.QueryPragmas; import org.elasticsearch.xpack.esql.querydsl.query.EqualsSyntheticSourceDelegate; import org.elasticsearch.xpack.esql.querydsl.query.SingleValueQuery; @@ -7837,7 +7838,7 @@ private LocalExecutionPlanner.LocalExecutionPlan physicalOperationsFromPhysicalP // The TopN needs an estimated row size for the planner to work var plans = PlannerUtils.breakPlanBetweenCoordinatorAndDataNode(EstimatesRowSize.estimateRowSize(0, plan), config); plan = useDataNodePlan ? plans.v2() : plans.v1(); - plan = PlannerUtils.localPlan(config, FoldContext.small(), plan, TEST_SEARCH_STATS); + plan = PlannerUtils.localPlan(new EsqlFlags(true), config, FoldContext.small(), plan, TEST_SEARCH_STATS); ExchangeSinkHandler exchangeSinkHandler = new ExchangeSinkHandler(null, 10, () -> 10); LocalExecutionPlanner planner = new LocalExecutionPlanner( "test", @@ -8198,7 +8199,7 @@ private PhysicalPlan optimizedPlan(PhysicalPlan plan, SearchStats searchStats) { // individually hence why here the plan is kept as is var l = p.transformUp(FragmentExec.class, fragment -> { - var localPlan = PlannerUtils.localPlan(config, FoldContext.small(), fragment, searchStats); + var localPlan = PlannerUtils.localPlan(new EsqlFlags(true), config, FoldContext.small(), fragment, searchStats); return EstimatesRowSize.estimateRowSize(fragment.estimatedRowSize(), localPlan); }); @@ -8236,7 +8237,7 @@ private PhysicalPlan physicalPlan(String query, TestDataSource dataSource) { } private PhysicalPlan physicalPlan(String query, TestDataSource dataSource, boolean assertSerialization) { - var logical = logicalOptimizer.optimize(dataSource.analyzer.analyze(parser.createStatement(query))); + var logical = logicalOptimizer.optimize(dataSource.analyzer.analyze(parser.createStatement(query, config))); // System.out.println("Logical\n" + logical); var physical = mapper.map(logical); // System.out.println("Physical\n" + physical); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/TestPlannerOptimizer.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/TestPlannerOptimizer.java index e6a7d110f8c09..761902bebe19e 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/TestPlannerOptimizer.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/TestPlannerOptimizer.java @@ -15,6 +15,7 @@ import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan; import org.elasticsearch.xpack.esql.planner.PlannerUtils; import org.elasticsearch.xpack.esql.planner.mapper.Mapper; +import org.elasticsearch.xpack.esql.plugin.EsqlFlags; import org.elasticsearch.xpack.esql.session.Configuration; import org.elasticsearch.xpack.esql.stats.SearchStats; @@ -66,7 +67,7 @@ private PhysicalPlan optimizedPlan(PhysicalPlan plan, SearchStats searchStats) { new LocalLogicalOptimizerContext(config, FoldContext.small(), searchStats) ); var physicalTestOptimizer = new TestLocalPhysicalPlanOptimizer( - new LocalPhysicalOptimizerContext(config, FoldContext.small(), searchStats), + new LocalPhysicalOptimizerContext(new EsqlFlags(true), config, FoldContext.small(), searchStats), true ); var l = PlannerUtils.localPlan(physicalPlan, logicalTestOptimizer, physicalTestOptimizer); @@ -79,7 +80,7 @@ private PhysicalPlan optimizedPlan(PhysicalPlan plan, SearchStats searchStats) { } private PhysicalPlan physicalPlan(String query, Analyzer analyzer) { - var logical = logicalOptimizer.optimize(analyzer.analyze(parser.createStatement(query))); + var logical = logicalOptimizer.optimize(analyzer.analyze(parser.createStatement(query, EsqlTestUtils.TEST_CFG))); // System.out.println("Logical\n" + logical); var physical = mapper.map(logical); return physical; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushTopNToSourceTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushTopNToSourceTests.java index cacf6e422882b..00f8dfb9aaacc 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushTopNToSourceTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushTopNToSourceTests.java @@ -35,6 +35,7 @@ import org.elasticsearch.xpack.esql.plan.physical.EvalExec; import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan; import org.elasticsearch.xpack.esql.plan.physical.TopNExec; +import org.elasticsearch.xpack.esql.plugin.EsqlFlags; import org.elasticsearch.xpack.esql.stats.SearchStats; import java.io.IOException; @@ -416,7 +417,7 @@ private static void assertNoPushdownSort(TestPhysicalPlanBuilder builder, String private static PhysicalPlan pushTopNToSource(TopNExec topNExec) { var configuration = EsqlTestUtils.configuration("from test"); - var ctx = new LocalPhysicalOptimizerContext(configuration, FoldContext.small(), SearchStats.EMPTY); + var ctx = new LocalPhysicalOptimizerContext(new EsqlFlags(true), configuration, FoldContext.small(), SearchStats.EMPTY); var pushTopNToSource = new PushTopNToSource(); return pushTopNToSource.rule(topNExec, ctx); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/AbstractStatementParserTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/AbstractStatementParserTests.java index c97cbb460d1f5..1cd4f38b4ae20 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/AbstractStatementParserTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/AbstractStatementParserTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.index.IndexMode; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.esql.EsqlTestUtils; import org.elasticsearch.xpack.esql.VerificationException; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Literal; @@ -57,11 +58,11 @@ LogicalPlan statement(String e) { } LogicalPlan statement(String e, QueryParams params) { - return parser.createStatement(e, params); + return parser.createStatement(e, params, EsqlTestUtils.TEST_CFG); } LogicalPlan processingCommand(String e) { - return parser.createStatement("row a = 1 | " + e); + return parser.createStatement("row a = 1 | " + e, EsqlTestUtils.TEST_CFG); } static UnresolvedAttribute attribute(String name) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/ExpressionTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/ExpressionTests.java index ff49cd5eed00a..44dc263483935 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/ExpressionTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/ExpressionTests.java @@ -9,6 +9,7 @@ import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.esql.EsqlTestUtils; import org.elasticsearch.xpack.esql.core.expression.Alias; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FoldContext; @@ -657,7 +658,7 @@ private Project projectExpression(String e) { } private LogicalPlan parse(String s) { - return parser.createStatement(s); + return parser.createStatement(s, EsqlTestUtils.TEST_CFG); } private Literal l(Object value, DataType type) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/GrammarInDevelopmentParsingTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/GrammarInDevelopmentParsingTests.java index dea1d62743b66..f8051bc38fc52 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/GrammarInDevelopmentParsingTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/GrammarInDevelopmentParsingTests.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.esql.parser; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.esql.EsqlTestUtils; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.not; @@ -35,7 +36,7 @@ public void testDevelopmentRerank() { } void parse(String query, String errorMessage) { - ParsingException pe = expectThrows(ParsingException.class, () -> parser().createStatement(query)); + ParsingException pe = expectThrows(ParsingException.class, () -> parser().createStatement(query, EsqlTestUtils.TEST_CFG)); assertThat(pe.getMessage(), containsString("mismatched input '" + errorMessage + "'")); // check the parser eliminated the DEV_ tokens from the message assertThat(pe.getMessage(), not(containsString("DEV_"))); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java index 055d899ceb674..026954843bf06 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java @@ -13,6 +13,7 @@ import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.IndexMode; +import org.elasticsearch.xpack.esql.EsqlTestUtils; import org.elasticsearch.xpack.esql.action.EsqlCapabilities; import org.elasticsearch.xpack.esql.core.capabilities.UnresolvedException; import org.elasticsearch.xpack.esql.core.expression.Alias; @@ -3646,7 +3647,7 @@ public void testRerankWithPositionalParameters() { List.of(paramAsConstant(null, "query text"), paramAsConstant(null, "reranker"), paramAsConstant(null, "rerank_score")) ); var rerank = as( - parser.createStatement("row a = 1 | RERANK ? ON title WITH inferenceId=?, scoreColumn=? ", queryParams), + parser.createStatement("row a = 1 | RERANK ? ON title WITH inferenceId=?, scoreColumn=? ", queryParams, EsqlTestUtils.TEST_CFG), Rerank.class ); @@ -3669,7 +3670,8 @@ public void testRerankWithNamedParameters() { var rerank = as( parser.createStatement( "row a = 1 | RERANK ?queryText ON title WITH inferenceId=?inferenceId, scoreColumn=?scoreColumnName", - queryParams + queryParams, + EsqlTestUtils.TEST_CFG ), Rerank.class ); @@ -3717,7 +3719,10 @@ public void testCompletionDefaultFieldName() { public void testCompletionWithPositionalParameters() { var queryParams = new QueryParams(List.of(paramAsConstant(null, "inferenceId"))); - var plan = as(parser.createStatement("row a = 1 | COMPLETION prompt_field WITH ?", queryParams), Completion.class); + var plan = as( + parser.createStatement("row a = 1 | COMPLETION prompt_field WITH ?", queryParams, EsqlTestUtils.TEST_CFG), + Completion.class + ); assertThat(plan.prompt(), equalTo(attribute("prompt_field"))); assertThat(plan.inferenceId(), equalTo(literalString("inferenceId"))); @@ -3726,7 +3731,10 @@ public void testCompletionWithPositionalParameters() { public void testCompletionWithNamedParameters() { var queryParams = new QueryParams(List.of(paramAsConstant("inferenceId", "myInference"))); - var plan = as(parser.createStatement("row a = 1 | COMPLETION prompt_field WITH ?inferenceId", queryParams), Completion.class); + var plan = as( + parser.createStatement("row a = 1 | COMPLETION prompt_field WITH ?inferenceId", queryParams, EsqlTestUtils.TEST_CFG), + Completion.class + ); assertThat(plan.prompt(), equalTo(attribute("prompt_field"))); assertThat(plan.inferenceId(), equalTo(literalString("myInference"))); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/FilterTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/FilterTests.java index 04d2cac31af55..d6727cc62e790 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/FilterTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/FilterTests.java @@ -28,8 +28,10 @@ import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; import org.elasticsearch.xpack.esql.index.EsIndex; import org.elasticsearch.xpack.esql.index.IndexResolution; +import org.elasticsearch.xpack.esql.io.stream.ExpressionQueryBuilder; import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamWrapperQueryBuilder; import org.elasticsearch.xpack.esql.optimizer.LogicalPlanOptimizer; import org.elasticsearch.xpack.esql.optimizer.PhysicalOptimizerContext; import org.elasticsearch.xpack.esql.optimizer.PhysicalPlanOptimizer; @@ -37,6 +39,7 @@ import org.elasticsearch.xpack.esql.plan.physical.FragmentExec; import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan; import org.elasticsearch.xpack.esql.planner.mapper.Mapper; +import org.elasticsearch.xpack.esql.plugin.EsqlFlags; import org.elasticsearch.xpack.esql.querydsl.query.SingleValueQuery; import org.elasticsearch.xpack.esql.session.Configuration; import org.junit.BeforeClass; @@ -47,6 +50,7 @@ import java.util.Map; import static java.util.Arrays.asList; +import static org.elasticsearch.TransportVersions.V_8_17_0; import static org.elasticsearch.index.query.QueryBuilders.rangeQuery; import static org.elasticsearch.xpack.esql.ConfigurationTestUtils.randomConfiguration; import static org.elasticsearch.xpack.esql.EsqlTestUtils.TEST_VERIFIER; @@ -314,6 +318,7 @@ public void testTimestampAsFunctionArgumentInsideExpression() { assertThat(filter, nullValue()); } + /** * Ugly hack to create a QueryBuilder for SingleValueQuery. * For some reason however the queryName is set to null on range queries when deserializing. @@ -342,7 +347,7 @@ public static QueryBuilder singleValueQuery(String query, QueryBuilder inner, St } private PhysicalPlan plan(String query, QueryBuilder restFilter) { - var logical = logicalOptimizer.optimize(analyzer.analyze(parser.createStatement(query))); + var logical = logicalOptimizer.optimize(analyzer.analyze(parser.createStatement(query, EsqlTestUtils.TEST_CFG))); // System.out.println("Logical\n" + logical); var physical = mapper.map(logical); // System.out.println("physical\n" + physical); @@ -361,7 +366,7 @@ private QueryBuilder restFilterQuery(String field) { } private QueryBuilder filterQueryForTransportNodes(PhysicalPlan plan) { - return PlannerUtils.detectFilter(plan, EMP_NO::equals); + return PlannerUtils.detectFilter(new EsqlFlags(true), null, minTransportVersion, plan, Set.of(EMP_NO, LAST_NAME)::contains); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/ClusterRequestTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/ClusterRequestTests.java index 9b7615d0cc37e..cba3c3a7556e0 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/ClusterRequestTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/ClusterRequestTests.java @@ -201,7 +201,7 @@ static LogicalPlan parse(String query) { ), TEST_VERIFIER ); - return logicalOptimizer.optimize(analyzer.analyze(new EsqlParser().createStatement(query))); + return logicalOptimizer.optimize(analyzer.analyze(new EsqlParser().createStatement(query, EsqlTestUtils.TEST_CFG))); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/DataNodeRequestSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/DataNodeRequestSerializationTests.java index abf9b527f008d..1a1d981ca0ba1 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/DataNodeRequestSerializationTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/DataNodeRequestSerializationTests.java @@ -302,7 +302,7 @@ static LogicalPlan parse(String query) { ), TEST_VERIFIER ); - return logicalOptimizer.optimize(analyzer.analyze(new EsqlParser().createStatement(query))); + return logicalOptimizer.optimize(analyzer.analyze(new EsqlParser().createStatement(query, EsqlTestUtils.TEST_CFG))); } static PhysicalPlan mapAndMaybeOptimize(LogicalPlan logicalPlan) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java index a10bc7bd73116..dca1d997ef93b 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java @@ -9,6 +9,7 @@ import org.elasticsearch.Build; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.esql.EsqlTestUtils; import org.elasticsearch.xpack.esql.action.EsqlCapabilities; import org.elasticsearch.xpack.esql.parser.EsqlParser; import org.elasticsearch.xpack.esql.parser.ParsingException; @@ -1467,7 +1468,7 @@ public void testEnrichOnDefaultField() { public void testMetrics() { var query = "METRICS k8s bytes=sum(rate(network.total_bytes_in)), sum(rate(network.total_cost)) BY cluster"; if (Build.current().isSnapshot() == false) { - var e = expectThrows(ParsingException.class, () -> parser.createStatement(query)); + var e = expectThrows(ParsingException.class, () -> parser.createStatement(query, EsqlTestUtils.TEST_CFG)); assertThat(e.getMessage(), containsString("line 1:1: mismatched input 'METRICS' expecting {")); return; } @@ -1861,7 +1862,8 @@ public void testDropWildcardFieldsAfterKeepAndLookupJoins2() { private Set fieldNames(String query, Set enrichPolicyMatchFields) { var preAnalysisResult = new EsqlSession.PreAnalysisResult(null); - return EsqlSession.fieldNames(parser.createStatement(query), enrichPolicyMatchFields, preAnalysisResult).fieldNames(); + return EsqlSession.fieldNames(parser.createStatement(query, EsqlTestUtils.TEST_CFG), enrichPolicyMatchFields, preAnalysisResult) + .fieldNames(); } private void assertFieldNames(String query, Set expected) { @@ -1870,7 +1872,11 @@ private void assertFieldNames(String query, Set expected) { } private void assertFieldNames(String query, Set expected, Set wildCardIndices) { - var preAnalysisResult = EsqlSession.fieldNames(parser.createStatement(query), Set.of(), new EsqlSession.PreAnalysisResult(null)); + var preAnalysisResult = EsqlSession.fieldNames( + parser.createStatement(query, EsqlTestUtils.TEST_CFG), + Set.of(), + new EsqlSession.PreAnalysisResult(null) + ); assertThat("Query-wide field names", preAnalysisResult.fieldNames(), equalTo(expected)); assertThat("Lookup Indices that expect wildcard lookups", preAnalysisResult.wildcardJoinIndices(), equalTo(wildCardIndices)); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/telemetry/VerifierMetricsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/telemetry/VerifierMetricsTests.java index de377fe78588c..32e9fbba4c129 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/telemetry/VerifierMetricsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/telemetry/VerifierMetricsTests.java @@ -10,6 +10,7 @@ import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.core.watcher.common.stats.Counters; +import org.elasticsearch.xpack.esql.EsqlTestUtils; import org.elasticsearch.xpack.esql.analysis.Verifier; import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; import org.elasticsearch.xpack.esql.expression.function.FunctionDefinition; @@ -529,7 +530,7 @@ private Counters esql(String esql, Verifier v) { metrics = new Metrics(new EsqlFunctionRegistry()); verifier = new Verifier(metrics, new XPackLicenseState(() -> 0L)); } - analyzer(verifier).analyze(parser.createStatement(esql)); + analyzer(verifier).analyze(parser.createStatement(esql, EsqlTestUtils.TEST_CFG)); return metrics == null ? null : metrics.stats(); } diff --git a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java index 4c04458af503d..b80a0ec01086a 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java +++ b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java @@ -211,6 +211,11 @@ protected boolean matches(String pattern, boolean caseInsensitive, QueryRewriteC return Regex.simpleMatch(pattern, value, caseInsensitive); } + @Override + public String getConstantFieldValue(SearchExecutionContext context) { + return value; + } + @Override public Query existsQuery(SearchExecutionContext context) { return value != null ? new MatchAllDocsQuery() : new MatchNoDocsQuery();