diff --git a/docs/changelog/129150.yaml b/docs/changelog/129150.yaml new file mode 100644 index 0000000000000..5e53f6f6a9171 --- /dev/null +++ b/docs/changelog/129150.yaml @@ -0,0 +1,6 @@ +pr: 129150 +summary: Add `none` chunking strategy to disable automatic chunking for inference + endpoints +area: Machine Learning +type: feature +issues: [] diff --git a/docs/reference/elasticsearch/mapping-reference/semantic-text.md b/docs/reference/elasticsearch/mapping-reference/semantic-text.md index 47a3439ce5b94..2799c91d466ba 100644 --- a/docs/reference/elasticsearch/mapping-reference/semantic-text.md +++ b/docs/reference/elasticsearch/mapping-reference/semantic-text.md @@ -117,15 +117,16 @@ If specified, these will override the chunking settings set in the {{infer-cap}} endpoint associated with `inference_id`. If chunking settings are updated, they will not be applied to existing documents until they are reindexed. +To completely disable chunking, use the `none` chunking strategy. **Valid values for `chunking_settings`**: `type` - : Indicates the type of chunking strategy to use. Valid values are `word` or + : Indicates the type of chunking strategy to use. Valid values are `none`, `word` or `sentence`. Required. `max_chunk_size` - : The maximum number of works in a chunk. Required. + : The maximum number of words in a chunk. Required for `word` and `sentence` strategies. `overlap` : The number of overlapping words allowed in chunks. This cannot be defined as @@ -136,6 +137,12 @@ until they are reindexed. : The number of overlapping sentences allowed in chunks. Valid values are `0` or `1`. Required for `sentence` type chunking settings +::::{warning} +If the input exceeds the maximum token limit of the underlying model, some services (such as OpenAI) may return an +error. In contrast, the `elastic` and `elasticsearch` services will automatically truncate the input to fit within the +model's limit. +:::: + ## {{infer-cap}} endpoint validation [infer-endpoint-validation] The `inference_id` will not be validated when the mapping is created, but when @@ -166,10 +173,49 @@ For more details on chunking and how to configure chunking settings, see [Configuring chunking](https://www.elastic.co/docs/api/doc/elasticsearch/group/endpoint-inference) in the Inference API documentation. +You can pre-chunk the input by sending it to Elasticsearch as an array of strings. +Example: + +```console +PUT test-index +{ + "mappings": { + "properties": { + "my_semantic_field": { + "type": "semantic_text", + "chunking_settings": { + "strategy": "none" <1> + } + } + } + } +} +``` + +1. Disable chunking on `my_semantic_field`. + +```console +PUT test-index/_doc/1 +{ + "my_semantic_field": ["my first chunk", "my second chunk", ...] <1> + ... +} +``` + +1. The text is pre-chunked and provided as an array of strings. + Each element in the array represents a single chunk that will be sent directly to the inference service without further chunking. + +**Important considerations**: + +* When providing pre-chunked input, ensure that you set the chunking strategy to `none` to avoid additional processing. +* Each chunk should be sized carefully, staying within the token limit of the inference service and the underlying model. +* If a chunk exceeds the model's token limit, the behavior depends on the service: + * Some services (such as OpenAI) will return an error. + * Others (such as `elastic` and `elasticsearch`) will automatically truncate the input. + Refer to [this tutorial](docs-content://solutions/search/semantic-search/semantic-search-semantic-text.md) -to learn more about semantic search using `semantic_text` and the `semantic` -query. +to learn more about semantic search using `semantic_text`. ## Extracting Relevant Fragments from Semantic Text [semantic-text-highlighting] diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index 6e942cdc263e7..d008b885a40a8 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -194,6 +194,7 @@ static TransportVersion def(int id) { public static final TransportVersion SEARCH_SOURCE_EXCLUDE_VECTORS_PARAM_8_19 = def(8_841_0_46); public static final TransportVersion ML_INFERENCE_MISTRAL_CHAT_COMPLETION_ADDED_8_19 = def(8_841_0_47); public static final TransportVersion ML_INFERENCE_ELASTIC_RERANK_ADDED_8_19 = def(8_841_0_48); + public static final TransportVersion NONE_CHUNKING_STRATEGY_8_19 = def(8_841_0_49); public static final TransportVersion V_9_0_0 = def(9_000_0_09); public static final TransportVersion INITIAL_ELASTICSEARCH_9_0_1 = def(9_000_0_10); public static final TransportVersion INITIAL_ELASTICSEARCH_9_0_2 = def(9_000_0_11); @@ -294,6 +295,7 @@ static TransportVersion def(int id) { public static final TransportVersion ML_INFERENCE_ELASTIC_RERANK = def(9_094_0_00); public static final TransportVersion SEARCH_LOAD_PER_INDEX_STATS = def(9_095_0_00); public static final TransportVersion HEAP_USAGE_IN_CLUSTER_INFO = def(9_096_0_00); + public static final TransportVersion NONE_CHUNKING_STRATEGY = def(9_097_0_00); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/inference/ChunkingStrategy.java b/server/src/main/java/org/elasticsearch/inference/ChunkingStrategy.java index bb5e0254834a3..78404c1b409ee 100644 --- a/server/src/main/java/org/elasticsearch/inference/ChunkingStrategy.java +++ b/server/src/main/java/org/elasticsearch/inference/ChunkingStrategy.java @@ -15,7 +15,8 @@ public enum ChunkingStrategy { WORD("word"), - SENTENCE("sentence"); + SENTENCE("sentence"), + NONE("none"); private final String chunkingStrategy; diff --git a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/AbstractTestInferenceService.java b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/AbstractTestInferenceService.java index 34e2af8034527..44c9d0463cd05 100644 --- a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/AbstractTestInferenceService.java +++ b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/AbstractTestInferenceService.java @@ -25,6 +25,7 @@ import org.elasticsearch.inference.TaskSettings; import org.elasticsearch.inference.TaskType; import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xpack.inference.chunking.NoopChunker; import org.elasticsearch.xpack.inference.chunking.WordBoundaryChunker; import org.elasticsearch.xpack.inference.chunking.WordBoundaryChunkingSettings; @@ -126,7 +127,14 @@ protected List chunkInputs(ChunkInferenceInput input) { } List chunkedInputs = new ArrayList<>(); - if (chunkingSettings.getChunkingStrategy() == ChunkingStrategy.WORD) { + if (chunkingSettings.getChunkingStrategy() == ChunkingStrategy.NONE) { + var offsets = NoopChunker.INSTANCE.chunk(input.input(), chunkingSettings); + List ret = new ArrayList<>(); + for (var offset : offsets) { + ret.add(new ChunkedInput(inputText.substring(offset.start(), offset.end()), offset.start(), offset.end())); + } + return ret; + } else if (chunkingSettings.getChunkingStrategy() == ChunkingStrategy.WORD) { WordBoundaryChunker chunker = new WordBoundaryChunker(); WordBoundaryChunkingSettings wordBoundaryChunkingSettings = (WordBoundaryChunkingSettings) chunkingSettings; List offsets = chunker.chunk( diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java index 3fd5b06450a73..a495f2af93eb2 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java @@ -26,6 +26,7 @@ import org.elasticsearch.xpack.core.inference.results.TextEmbeddingByteResults; import org.elasticsearch.xpack.core.inference.results.TextEmbeddingFloatResults; import org.elasticsearch.xpack.inference.action.task.StreamingTaskManager; +import org.elasticsearch.xpack.inference.chunking.NoneChunkingSettings; import org.elasticsearch.xpack.inference.chunking.SentenceBoundaryChunkingSettings; import org.elasticsearch.xpack.inference.chunking.WordBoundaryChunkingSettings; import org.elasticsearch.xpack.inference.common.amazon.AwsSecretSettings; @@ -553,6 +554,9 @@ private static void addInternalNamedWriteables(List namedWriteables) { + namedWriteables.add( + new NamedWriteableRegistry.Entry(ChunkingSettings.class, NoneChunkingSettings.NAME, in -> NoneChunkingSettings.INSTANCE) + ); namedWriteables.add( new NamedWriteableRegistry.Entry(ChunkingSettings.class, WordBoundaryChunkingSettings.NAME, WordBoundaryChunkingSettings::new) ); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/ChunkerBuilder.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/ChunkerBuilder.java index 830f1579348f6..f10748c2fec97 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/ChunkerBuilder.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/ChunkerBuilder.java @@ -16,6 +16,7 @@ public static Chunker fromChunkingStrategy(ChunkingStrategy chunkingStrategy) { } return switch (chunkingStrategy) { + case NONE -> NoopChunker.INSTANCE; case WORD -> new WordBoundaryChunker(); case SENTENCE -> new SentenceBoundaryChunker(); }; diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/ChunkingSettingsBuilder.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/ChunkingSettingsBuilder.java index 25553a4c760f0..b1bc5987eaa99 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/ChunkingSettingsBuilder.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/ChunkingSettingsBuilder.java @@ -45,6 +45,7 @@ public static ChunkingSettings fromMap(Map settings, boolean ret settings.get(ChunkingSettingsOptions.STRATEGY.toString()).toString() ); return switch (chunkingStrategy) { + case NONE -> NoneChunkingSettings.INSTANCE; case WORD -> WordBoundaryChunkingSettings.fromMap(new HashMap<>(settings)); case SENTENCE -> SentenceBoundaryChunkingSettings.fromMap(new HashMap<>(settings)); }; diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/NoneChunkingSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/NoneChunkingSettings.java new file mode 100644 index 0000000000000..bd3da61d81063 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/NoneChunkingSettings.java @@ -0,0 +1,110 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.chunking; + +import org.elasticsearch.TransportVersion; +import org.elasticsearch.TransportVersions; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.ValidationException; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.inference.ChunkingSettings; +import org.elasticsearch.inference.ChunkingStrategy; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +public class NoneChunkingSettings implements ChunkingSettings { + public static final String NAME = "NoneChunkingSettings"; + public static NoneChunkingSettings INSTANCE = new NoneChunkingSettings(); + + private static final ChunkingStrategy STRATEGY = ChunkingStrategy.NONE; + private static final Set VALID_KEYS = Set.of(ChunkingSettingsOptions.STRATEGY.toString()); + + private NoneChunkingSettings() {} + + @Override + public ChunkingStrategy getChunkingStrategy() { + return STRATEGY; + } + + @Override + public String getWriteableName() { + return NAME; + } + + @Override + public TransportVersion getMinimalSupportedVersion() { + throw new IllegalStateException("not used"); + } + + @Override + public boolean supportsVersion(TransportVersion version) { + return version.isPatchFrom(TransportVersions.NONE_CHUNKING_STRATEGY_8_19) + || version.onOrAfter(TransportVersions.NONE_CHUNKING_STRATEGY); + } + + @Override + public void writeTo(StreamOutput out) throws IOException {} + + @Override + public Map asMap() { + return Map.of(ChunkingSettingsOptions.STRATEGY.toString(), STRATEGY.toString().toLowerCase(Locale.ROOT)); + } + + public static NoneChunkingSettings fromMap(Map map) { + ValidationException validationException = new ValidationException(); + + var invalidSettings = map.keySet().stream().filter(key -> VALID_KEYS.contains(key) == false).toArray(); + if (invalidSettings.length > 0) { + validationException.addValidationError( + Strings.format( + "When chunking is disabled (none), settings can not have the following: %s", + Arrays.toString(invalidSettings) + ) + ); + } + + if (validationException.validationErrors().isEmpty() == false) { + throw validationException; + } + + return NoneChunkingSettings.INSTANCE; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + { + builder.field(ChunkingSettingsOptions.STRATEGY.toString(), STRATEGY); + } + builder.endObject(); + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + return true; + } + + @Override + public int hashCode() { + return Objects.hash(getClass()); + } + + @Override + public String toString() { + return Strings.toString(this); + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/NoopChunker.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/NoopChunker.java new file mode 100644 index 0000000000000..4698275de7c59 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/NoopChunker.java @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.chunking; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.inference.ChunkingSettings; +import org.elasticsearch.xpack.inference.services.openai.embeddings.OpenAiEmbeddingsModel; + +import java.util.List; + +/** + * A {@link Chunker} implementation that returns the input unchanged (no chunking is performed). + * + *

WARNINGIf the input exceeds the maximum token limit, some services (such as {@link OpenAiEmbeddingsModel}) + * may return an error. + *

+ */ +public class NoopChunker implements Chunker { + public static final NoopChunker INSTANCE = new NoopChunker(); + + private NoopChunker() {} + + @Override + public List chunk(String input, ChunkingSettings chunkingSettings) { + if (chunkingSettings instanceof NoneChunkingSettings) { + return List.of(new ChunkOffset(0, input.length())); + } else { + throw new IllegalArgumentException( + Strings.format("NoopChunker can't use ChunkingSettings with strategy [%s]", chunkingSettings.getChunkingStrategy()) + ); + } + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/ChunkerBuilderTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/ChunkerBuilderTests.java index d2aea45d4603c..471b5400991e2 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/ChunkerBuilderTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/ChunkerBuilderTests.java @@ -27,6 +27,13 @@ public void testValidChunkingStrategy() { } private Map> chunkingStrategyToExpectedChunkerClassMap() { - return Map.of(ChunkingStrategy.WORD, WordBoundaryChunker.class, ChunkingStrategy.SENTENCE, SentenceBoundaryChunker.class); + return Map.of( + ChunkingStrategy.NONE, + NoopChunker.class, + ChunkingStrategy.WORD, + WordBoundaryChunker.class, + ChunkingStrategy.SENTENCE, + SentenceBoundaryChunker.class + ); } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/ChunkingSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/ChunkingSettingsTests.java index 2832c2f64e0e6..9dfa417c3c477 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/ChunkingSettingsTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/ChunkingSettingsTests.java @@ -20,6 +20,9 @@ public static ChunkingSettings createRandomChunkingSettings() { ChunkingStrategy randomStrategy = randomFrom(ChunkingStrategy.values()); switch (randomStrategy) { + case NONE -> { + return NoneChunkingSettings.INSTANCE; + } case WORD -> { var maxChunkSize = randomIntBetween(10, 300); return new WordBoundaryChunkingSettings(maxChunkSize, randomIntBetween(1, maxChunkSize / 2)); @@ -37,15 +40,15 @@ public static Map createRandomChunkingSettingsMap() { chunkingSettingsMap.put(ChunkingSettingsOptions.STRATEGY.toString(), randomStrategy.toString()); switch (randomStrategy) { + case NONE -> { + } case WORD -> { var maxChunkSize = randomIntBetween(10, 300); chunkingSettingsMap.put(ChunkingSettingsOptions.MAX_CHUNK_SIZE.toString(), maxChunkSize); chunkingSettingsMap.put(ChunkingSettingsOptions.OVERLAP.toString(), randomIntBetween(1, maxChunkSize / 2)); } - case SENTENCE -> { - chunkingSettingsMap.put(ChunkingSettingsOptions.MAX_CHUNK_SIZE.toString(), randomIntBetween(20, 300)); - } + case SENTENCE -> chunkingSettingsMap.put(ChunkingSettingsOptions.MAX_CHUNK_SIZE.toString(), randomIntBetween(20, 300)); default -> { } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/EmbeddingRequestChunkerTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/EmbeddingRequestChunkerTests.java index a9cf741815d51..411d992adfa3d 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/EmbeddingRequestChunkerTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/EmbeddingRequestChunkerTests.java @@ -46,6 +46,22 @@ public void testEmptyInput_SentenceChunker() { assertThat(batches, empty()); } + public void testEmptyInput_NoopChunker() { + var batches = new EmbeddingRequestChunker<>(List.of(), 10, NoneChunkingSettings.INSTANCE).batchRequestsWithListeners( + testListener() + ); + assertThat(batches, empty()); + } + + public void testAnyInput_NoopChunker() { + var randomInput = randomAlphaOfLengthBetween(100, 1000); + var batches = new EmbeddingRequestChunker<>(List.of(new ChunkInferenceInput(randomInput)), 10, NoneChunkingSettings.INSTANCE) + .batchRequestsWithListeners(testListener()); + assertThat(batches, hasSize(1)); + assertThat(batches.get(0).batch().inputs().get(), hasSize(1)); + assertThat(batches.get(0).batch().inputs().get().get(0), Matchers.is(randomInput)); + } + public void testWhitespaceInput_SentenceChunker() { var batches = new EmbeddingRequestChunker<>( List.of(new ChunkInferenceInput(" ")), diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/NoneChunkingSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/NoneChunkingSettingsTests.java new file mode 100644 index 0000000000000..660b223d0cab0 --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/NoneChunkingSettingsTests.java @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.chunking; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractWireSerializingTestCase; + +import java.io.IOException; + +public class NoneChunkingSettingsTests extends AbstractWireSerializingTestCase { + @Override + protected Writeable.Reader instanceReader() { + return in -> NoneChunkingSettings.INSTANCE; + } + + @Override + protected NoneChunkingSettings createTestInstance() { + return NoneChunkingSettings.INSTANCE; + } + + @Override + protected boolean shouldBeSame(NoneChunkingSettings newInstance) { + return true; + } + + @Override + protected NoneChunkingSettings mutateInstance(NoneChunkingSettings instance) throws IOException { + return null; + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/WordBoundaryChunkerTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/WordBoundaryChunkerTests.java index a4228d50eae4b..65d9645daca4f 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/WordBoundaryChunkerTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/WordBoundaryChunkerTests.java @@ -147,8 +147,11 @@ public void testNumberOfChunksWithWordBoundaryChunkingSettings() { } public void testInvalidChunkingSettingsProvided() { - ChunkingSettings chunkingSettings = new SentenceBoundaryChunkingSettings(randomIntBetween(20, 300), 0); - assertThrows(IllegalArgumentException.class, () -> { new WordBoundaryChunker().chunk(TEST_TEXT, chunkingSettings); }); + ChunkingSettings chunkingSettings1 = new SentenceBoundaryChunkingSettings(randomIntBetween(20, 300), 0); + assertThrows(IllegalArgumentException.class, () -> { new WordBoundaryChunker().chunk(TEST_TEXT, chunkingSettings1); }); + + ChunkingSettings chunkingSettings2 = NoneChunkingSettings.INSTANCE; + assertThrows(IllegalArgumentException.class, () -> { new WordBoundaryChunker().chunk(TEST_TEXT, chunkingSettings2); }); } public void testWindowSpanningWithOverlapNumWordsInOverlapSection() { diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldTests.java index 6b2f9d387071b..d1499f4009d0a 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldTests.java @@ -30,6 +30,7 @@ import org.elasticsearch.xpack.core.inference.results.TextEmbeddingByteResults; import org.elasticsearch.xpack.core.inference.results.TextEmbeddingFloatResults; import org.elasticsearch.xpack.core.utils.FloatConversionUtils; +import org.elasticsearch.xpack.inference.chunking.NoneChunkingSettings; import org.elasticsearch.xpack.inference.chunking.SentenceBoundaryChunkingSettings; import org.elasticsearch.xpack.inference.chunking.WordBoundaryChunkingSettings; import org.elasticsearch.xpack.inference.model.TestModel; @@ -342,9 +343,12 @@ public static ChunkingSettings generateRandomChunkingSettings(boolean allowNull) if (allowNull && randomBoolean()) { return null; // Use model defaults } - return randomBoolean() - ? new WordBoundaryChunkingSettings(randomIntBetween(20, 100), randomIntBetween(1, 10)) - : new SentenceBoundaryChunkingSettings(randomIntBetween(20, 100), randomIntBetween(0, 1)); + return switch (randomIntBetween(0, 2)) { + case 0 -> NoneChunkingSettings.INSTANCE; + case 1 -> new WordBoundaryChunkingSettings(randomIntBetween(20, 100), randomIntBetween(1, 10)); + case 2 -> new SentenceBoundaryChunkingSettings(randomIntBetween(20, 100), randomIntBetween(0, 1)); + default -> throw new IllegalStateException("Illegal state while generating random chunking settings"); + }; } public static ChunkingSettings generateRandomChunkingSettingsOtherThan(ChunkingSettings chunkingSettings) { diff --git a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/25_semantic_text_field_mapping_chunking.yml b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/25_semantic_text_field_mapping_chunking.yml index a6ff307f0ef4a..f3167e69e113f 100644 --- a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/25_semantic_text_field_mapping_chunking.yml +++ b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/25_semantic_text_field_mapping_chunking.yml @@ -91,6 +91,20 @@ setup: max_chunk_size: 10 overlap: 1 + - do: + indices.create: + index: none-chunking-dense + body: + mappings: + properties: + keyword_field: + type: keyword + inference_field: + type: semantic_text + inference_id: dense-inference-id + chunking_settings: + strategy: none + - do: index: index: default-chunking-sparse @@ -127,6 +141,15 @@ setup: inference_field: "Elasticsearch is an open source, distributed, RESTful, search engine which is built on top of Lucene internally and enjoys all the features it provides." refresh: true + - do: + index: + index: none-chunking-dense + id: doc_5 + body: + keyword_field: "none chunking" + inference_field: "Elasticsearch is an open source, distributed, RESTful, search engine which is built on top of Lucene internally and enjoys all the features it provides." + refresh: true + --- "We return chunking configurations with mappings": @@ -158,6 +181,11 @@ setup: - match: { "custom-chunking-dense.mappings.properties.inference_field.chunking_settings.max_chunk_size": 10 } - match: { "custom-chunking-dense.mappings.properties.inference_field.chunking_settings.overlap": 1 } + - do: + indices.get_mapping: + index: none-chunking-dense + + - match: { "none-chunking-dense.mappings.properties.inference_field.chunking_settings.strategy": "none" } --- "We do not set custom chunking settings for null or empty specified chunking settings": @@ -283,6 +311,25 @@ setup: - match: { hits.hits.0.highlight.inference_field.1: " which is built on top of Lucene internally and enjoys" } - match: { hits.hits.0.highlight.inference_field.2: " enjoys all the features it provides." } + - do: + search: + index: none-chunking-dense + body: + query: + semantic: + field: "inference_field" + query: "What is Elasticsearch?" + highlight: + fields: + inference_field: + type: "semantic" + number_of_fragments: 2 + + - match: { hits.total.value: 1 } + - match: { hits.hits.0._id: "doc_5" } + - length: { hits.hits.0.highlight.inference_field: 1 } + - match: { hits.hits.0.highlight.inference_field.0: "Elasticsearch is an open source, distributed, RESTful, search engine which is built on top of Lucene internally and enjoys all the features it provides." } + --- "We respect multiple semantic_text fields with different chunking configurations": diff --git a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/25_semantic_text_field_mapping_chunking_bwc.yml b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/25_semantic_text_field_mapping_chunking_bwc.yml index f189d5535bb77..9bf49741bca64 100644 --- a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/25_semantic_text_field_mapping_chunking_bwc.yml +++ b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/25_semantic_text_field_mapping_chunking_bwc.yml @@ -99,6 +99,22 @@ setup: max_chunk_size: 10 overlap: 1 + - do: + indices.create: + index: none-chunking-dense + body: + settings: + index.mapping.semantic_text.use_legacy_format: true + mappings: + properties: + keyword_field: + type: keyword + inference_field: + type: semantic_text + inference_id: dense-inference-id + chunking_settings: + strategy: none + - do: index: index: default-chunking-sparse @@ -135,6 +151,15 @@ setup: inference_field: "Elasticsearch is an open source, distributed, RESTful, search engine which is built on top of Lucene internally and enjoys all the features it provides." refresh: true + - do: + index: + index: none-chunking-dense + id: doc_5 + body: + keyword_field: "none chunking" + inference_field: "Elasticsearch is an open source, distributed, RESTful, search engine which is built on top of Lucene internally and enjoys all the features it provides." + refresh: true + --- "We return chunking configurations with mappings": @@ -166,6 +191,11 @@ setup: - match: { "custom-chunking-dense.mappings.properties.inference_field.chunking_settings.max_chunk_size": 10 } - match: { "custom-chunking-dense.mappings.properties.inference_field.chunking_settings.overlap": 1 } + - do: + indices.get_mapping: + index: none-chunking-dense + + - match: { "none-chunking-dense.mappings.properties.inference_field.chunking_settings.strategy": "none" } --- "We do not set custom chunking settings for null or empty specified chunking settings": @@ -295,6 +325,25 @@ setup: - match: { hits.hits.0.highlight.inference_field.1: " which is built on top of Lucene internally and enjoys" } - match: { hits.hits.0.highlight.inference_field.2: " enjoys all the features it provides." } + - do: + search: + index: none-chunking-dense + body: + query: + semantic: + field: "inference_field" + query: "What is Elasticsearch?" + highlight: + fields: + inference_field: + type: "semantic" + number_of_fragments: 2 + + - match: { hits.total.value: 1 } + - match: { hits.hits.0._id: "doc_5" } + - length: { hits.hits.0.highlight.inference_field: 1 } + - match: { hits.hits.0.highlight.inference_field.0: "Elasticsearch is an open source, distributed, RESTful, search engine which is built on top of Lucene internally and enjoys all the features it provides." } + --- "We respect multiple semantic_text fields with different chunking configurations":