From c557fb39c108fa7253aa9276bbf0ffaf1d963d72 Mon Sep 17 00:00:00 2001 From: Benjamin Trent <4357155+benwtrent@users.noreply.github.com> Date: Fri, 2 May 2025 16:22:36 -0400 Subject: [PATCH 1/4] Adding versions for bwc for #125599 --- .../java/org/elasticsearch/TransportVersions.java | 1 + .../org/elasticsearch/index/IndexVersions.java | 1 + .../mapper/vectors/DenseVectorFieldMapper.java | 14 ++++++++++---- .../search/vectors/RescoreVectorBuilder.java | 6 +++++- .../vectors/DenseVectorFieldMapperTests.java | 8 ++++++-- 5 files changed, 23 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index d94827b02ef86..881568f77f1eb 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -165,6 +165,7 @@ static TransportVersion def(int id) { public static final TransportVersion ML_INFERENCE_SAGEMAKER_8_19 = def(8_841_0_21); public static final TransportVersion ESQL_REPORT_ORIGINAL_TYPES_BACKPORT_8_19 = def(8_841_0_22); public static final TransportVersion INTRODUCE_FAILURES_LIFECYCLE_BACKPORT_8_19 = def(8_841_0_23); + public static final TransportVersion RESCORE_VECTOR_ALLOW_ZERO_BACKPORT_8_19 = def(8_841_0_25); public static final TransportVersion V_9_0_0 = def(9_000_0_09); public static final TransportVersion INITIAL_ELASTICSEARCH_9_0_1 = def(9_000_0_10); public static final TransportVersion COHERE_BIT_EMBEDDING_TYPE_SUPPORT_ADDED = def(9_001_0_00); diff --git a/server/src/main/java/org/elasticsearch/index/IndexVersions.java b/server/src/main/java/org/elasticsearch/index/IndexVersions.java index 53d2ad0d19707..1dd12dbe0322a 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexVersions.java +++ b/server/src/main/java/org/elasticsearch/index/IndexVersions.java @@ -138,6 +138,7 @@ private static Version parseUnchecked(String version) { public static final IndexVersion USE_SYNTHETIC_SOURCE_FOR_RECOVERY_BY_DEFAULT_BACKPORT = def(8_526_0_00, parseUnchecked("9.12.1")); public static final IndexVersion SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_BACKPORT_8_X = def(8_527_0_00, Version.LUCENE_9_12_1); public static final IndexVersion ADD_RESCORE_PARAMS_TO_QUANTIZED_VECTORS_BACKPORT_8_X = def(8_528_0_00, Version.LUCENE_9_12_1); + public static final IndexVersion RESCORE_PARAMS_ALLOW_ZERO_TO_QUANTIZED_VECTORS_BACKPORT_8_X = def(8_529_0_00, Version.LUCENE_9_12_1); public static final IndexVersion UPGRADE_TO_LUCENE_10_0_0 = def(9_000_0_00, Version.LUCENE_10_0_0); public static final IndexVersion LOGSDB_DEFAULT_IGNORE_DYNAMIC_BEYOND_LIMIT = def(9_001_0_00, Version.LUCENE_10_0_0); public static final IndexVersion TIME_BASED_K_ORDERED_DOC_ID = def(9_002_0_00, Version.LUCENE_10_0_0); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index 532cdfc08666c..48413d7216e02 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -96,6 +96,7 @@ import static org.elasticsearch.common.Strings.format; import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; import static org.elasticsearch.index.IndexVersions.DEFAULT_DENSE_VECTOR_TO_INT8_HNSW; +import static org.elasticsearch.index.IndexVersions.RESCORE_PARAMS_ALLOW_ZERO_TO_QUANTIZED_VECTORS; /** * A {@link FieldMapper} for indexing a dense vector of floats. @@ -114,14 +115,19 @@ private static boolean hasRescoreIndexVersion(IndexVersion version) { || version.between(IndexVersions.ADD_RESCORE_PARAMS_TO_QUANTIZED_VECTORS_BACKPORT_8_X, IndexVersions.UPGRADE_TO_LUCENE_10_0_0); } + private static boolean allowsZeroRescore(IndexVersion version) { + return version.onOrAfter(RESCORE_PARAMS_ALLOW_ZERO_TO_QUANTIZED_VECTORS) + || version.between( + IndexVersions.RESCORE_PARAMS_ALLOW_ZERO_TO_QUANTIZED_VECTORS_BACKPORT_8_X, + IndexVersions.UPGRADE_TO_LUCENE_10_0_0 + ); + } + public static final IndexVersion MAGNITUDE_STORED_INDEX_VERSION = IndexVersions.V_7_5_0; public static final IndexVersion INDEXED_BY_DEFAULT_INDEX_VERSION = IndexVersions.FIRST_DETACHED_INDEX_VERSION; public static final IndexVersion NORMALIZE_COSINE = IndexVersions.NORMALIZED_VECTOR_COSINE; public static final IndexVersion DEFAULT_TO_INT8 = DEFAULT_DENSE_VECTOR_TO_INT8_HNSW; public static final IndexVersion LITTLE_ENDIAN_FLOAT_STORED_INDEX_VERSION = IndexVersions.V_8_9_0; - public static final IndexVersion ADD_RESCORE_PARAMS_TO_QUANTIZED_VECTORS = IndexVersions.ADD_RESCORE_PARAMS_TO_QUANTIZED_VECTORS; - public static final IndexVersion RESCORE_PARAMS_ALLOW_ZERO_TO_QUANTIZED_VECTORS = - IndexVersions.RESCORE_PARAMS_ALLOW_ZERO_TO_QUANTIZED_VECTORS; public static final IndexVersion DEFAULT_OVERSAMPLE_VALUE_FOR_BBQ = IndexVersions.DEFAULT_OVERSAMPLE_VALUE_FOR_BBQ; public static final NodeFeature RESCORE_VECTOR_QUANTIZED_VECTOR_MAPPING = new NodeFeature("mapper.dense_vector.rescore_vector"); @@ -2050,7 +2056,7 @@ static RescoreVector fromIndexOptions(Map indexOptionsMap, IndexVersi throw new IllegalArgumentException("Invalid rescore_vector value. Missing required field " + OVERSAMPLE); } float oversampleValue = (float) XContentMapValues.nodeDoubleValue(oversampleNode); - if (oversampleValue == 0 && indexVersion.before(RESCORE_PARAMS_ALLOW_ZERO_TO_QUANTIZED_VECTORS)) { + if (oversampleValue == 0 && allowsZeroRescore(indexVersion)) { throw new IllegalArgumentException("oversample must be greater than 1"); } if (oversampleValue < 1 && oversampleValue != 0) { diff --git a/server/src/main/java/org/elasticsearch/search/vectors/RescoreVectorBuilder.java b/server/src/main/java/org/elasticsearch/search/vectors/RescoreVectorBuilder.java index 51cc87d58fcc2..326e41cf5abc3 100644 --- a/server/src/main/java/org/elasticsearch/search/vectors/RescoreVectorBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/vectors/RescoreVectorBuilder.java @@ -24,6 +24,8 @@ import java.util.Objects; import static org.elasticsearch.TransportVersions.RESCORE_VECTOR_ALLOW_ZERO; +import static org.elasticsearch.TransportVersions.RESCORE_VECTOR_ALLOW_ZERO_BACKPORT_8_19; +import static org.elasticsearch.TransportVersions.V_9_0_0; public class RescoreVectorBuilder implements Writeable, ToXContentObject { @@ -57,7 +59,9 @@ public RescoreVectorBuilder(StreamInput in) throws IOException { @Override public void writeTo(StreamOutput out) throws IOException { // We don't want to serialize a `0` oversample to a node that doesn't know what to do with it. - if (oversample == NO_OVERSAMPLE && out.getTransportVersion().before(RESCORE_VECTOR_ALLOW_ZERO)) { + if (oversample == NO_OVERSAMPLE + && out.getTransportVersion().before(RESCORE_VECTOR_ALLOW_ZERO) + && (out.getTransportVersion().between(RESCORE_VECTOR_ALLOW_ZERO_BACKPORT_8_19, V_9_0_0) == false)) { throw new ElasticsearchStatusException( "[rescore_vector] does not support a 0 for [" + OVERSAMPLE_FIELD.getPreferredName() diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java index d57acf786d715..9fbadcbf37dfb 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java @@ -934,8 +934,12 @@ public void testRescoreVectorOldIndexVersion() { public void testRescoreZeroVectorOldIndexVersion() { IndexVersion incompatibleVersion = IndexVersionUtils.randomVersionBetween( random(), - IndexVersionUtils.getLowestReadCompatibleVersion(), - IndexVersionUtils.getPreviousVersion(DenseVectorFieldMapper.RESCORE_PARAMS_ALLOW_ZERO_TO_QUANTIZED_VECTORS) + IndexVersionUtils.randomVersionBetween( + random(), + IndexVersionUtils.getLowestReadCompatibleVersion(), + IndexVersionUtils.getPreviousVersion(IndexVersions.RESCORE_PARAMS_ALLOW_ZERO_TO_QUANTIZED_VECTORS_BACKPORT_8_X) + ), + IndexVersionUtils.getPreviousVersion(IndexVersions.RESCORE_PARAMS_ALLOW_ZERO_TO_QUANTIZED_VECTORS) ); for (String indexType : List.of("int8_hnsw", "int8_flat", "int4_hnsw", "int4_flat", "bbq_hnsw", "bbq_flat")) { expectThrows( From 7e74cf566a6277cf997028beb13cb1a888af9a06 Mon Sep 17 00:00:00 2001 From: Benjamin Trent <4357155+benwtrent@users.noreply.github.com> Date: Mon, 5 May 2025 07:55:26 -0400 Subject: [PATCH 2/4] fix serialization and test --- .../search/vectors/RescoreVectorBuilder.java | 3 +-- .../mapper/vectors/DenseVectorFieldMapperTests.java | 9 ++++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/vectors/RescoreVectorBuilder.java b/server/src/main/java/org/elasticsearch/search/vectors/RescoreVectorBuilder.java index 326e41cf5abc3..fd939820ab37b 100644 --- a/server/src/main/java/org/elasticsearch/search/vectors/RescoreVectorBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/vectors/RescoreVectorBuilder.java @@ -25,7 +25,6 @@ import static org.elasticsearch.TransportVersions.RESCORE_VECTOR_ALLOW_ZERO; import static org.elasticsearch.TransportVersions.RESCORE_VECTOR_ALLOW_ZERO_BACKPORT_8_19; -import static org.elasticsearch.TransportVersions.V_9_0_0; public class RescoreVectorBuilder implements Writeable, ToXContentObject { @@ -61,7 +60,7 @@ public void writeTo(StreamOutput out) throws IOException { // We don't want to serialize a `0` oversample to a node that doesn't know what to do with it. if (oversample == NO_OVERSAMPLE && out.getTransportVersion().before(RESCORE_VECTOR_ALLOW_ZERO) - && (out.getTransportVersion().between(RESCORE_VECTOR_ALLOW_ZERO_BACKPORT_8_19, V_9_0_0) == false)) { + && (out.getTransportVersion().isPatchFrom(RESCORE_VECTOR_ALLOW_ZERO_BACKPORT_8_19) == false)) { throw new ElasticsearchStatusException( "[rescore_vector] does not support a 0 for [" + OVERSAMPLE_FIELD.getPreferredName() diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java index 9fbadcbf37dfb..19d78a5237269 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java @@ -932,14 +932,17 @@ public void testRescoreVectorOldIndexVersion() { } public void testRescoreZeroVectorOldIndexVersion() { - IndexVersion incompatibleVersion = IndexVersionUtils.randomVersionBetween( - random(), + IndexVersion incompatibleVersion = randomFrom( IndexVersionUtils.randomVersionBetween( random(), IndexVersionUtils.getLowestReadCompatibleVersion(), IndexVersionUtils.getPreviousVersion(IndexVersions.RESCORE_PARAMS_ALLOW_ZERO_TO_QUANTIZED_VECTORS_BACKPORT_8_X) ), - IndexVersionUtils.getPreviousVersion(IndexVersions.RESCORE_PARAMS_ALLOW_ZERO_TO_QUANTIZED_VECTORS) + IndexVersionUtils.randomVersionBetween( + random(), + IndexVersions.UPGRADE_TO_LUCENE_10_0_0, + IndexVersionUtils.getPreviousVersion(IndexVersions.RESCORE_PARAMS_ALLOW_ZERO_TO_QUANTIZED_VECTORS) + ) ); for (String indexType : List.of("int8_hnsw", "int8_flat", "int4_hnsw", "int4_flat", "bbq_hnsw", "bbq_flat")) { expectThrows( From 65944f96e9913ec4e43be53894f81a5fd6c369e4 Mon Sep 17 00:00:00 2001 From: Benjamin Trent <4357155+benwtrent@users.noreply.github.com> Date: Mon, 5 May 2025 09:07:35 -0400 Subject: [PATCH 3/4] fixing transport validation --- .../elasticsearch/search/vectors/RescoreVectorBuilder.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/vectors/RescoreVectorBuilder.java b/server/src/main/java/org/elasticsearch/search/vectors/RescoreVectorBuilder.java index fd939820ab37b..90d0f707700c2 100644 --- a/server/src/main/java/org/elasticsearch/search/vectors/RescoreVectorBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/vectors/RescoreVectorBuilder.java @@ -59,8 +59,8 @@ public RescoreVectorBuilder(StreamInput in) throws IOException { public void writeTo(StreamOutput out) throws IOException { // We don't want to serialize a `0` oversample to a node that doesn't know what to do with it. if (oversample == NO_OVERSAMPLE - && out.getTransportVersion().before(RESCORE_VECTOR_ALLOW_ZERO) - && (out.getTransportVersion().isPatchFrom(RESCORE_VECTOR_ALLOW_ZERO_BACKPORT_8_19) == false)) { + && (out.getTransportVersion().before(RESCORE_VECTOR_ALLOW_ZERO) + || (out.getTransportVersion().isPatchFrom(RESCORE_VECTOR_ALLOW_ZERO_BACKPORT_8_19) == false))) { throw new ElasticsearchStatusException( "[rescore_vector] does not support a 0 for [" + OVERSAMPLE_FIELD.getPreferredName() From 9596c7f45e87afd185ef7eed15a5e968be635cda Mon Sep 17 00:00:00 2001 From: Benjamin Trent <4357155+benwtrent@users.noreply.github.com> Date: Mon, 5 May 2025 09:40:45 -0400 Subject: [PATCH 4/4] fixing version checks --- .../index/mapper/vectors/DenseVectorFieldMapper.java | 2 +- .../elasticsearch/search/vectors/RescoreVectorBuilder.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index 48413d7216e02..7198338daf7e2 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -2056,7 +2056,7 @@ static RescoreVector fromIndexOptions(Map indexOptionsMap, IndexVersi throw new IllegalArgumentException("Invalid rescore_vector value. Missing required field " + OVERSAMPLE); } float oversampleValue = (float) XContentMapValues.nodeDoubleValue(oversampleNode); - if (oversampleValue == 0 && allowsZeroRescore(indexVersion)) { + if (oversampleValue == 0 && allowsZeroRescore(indexVersion) == false) { throw new IllegalArgumentException("oversample must be greater than 1"); } if (oversampleValue < 1 && oversampleValue != 0) { diff --git a/server/src/main/java/org/elasticsearch/search/vectors/RescoreVectorBuilder.java b/server/src/main/java/org/elasticsearch/search/vectors/RescoreVectorBuilder.java index 90d0f707700c2..ac5938b047804 100644 --- a/server/src/main/java/org/elasticsearch/search/vectors/RescoreVectorBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/vectors/RescoreVectorBuilder.java @@ -59,8 +59,8 @@ public RescoreVectorBuilder(StreamInput in) throws IOException { public void writeTo(StreamOutput out) throws IOException { // We don't want to serialize a `0` oversample to a node that doesn't know what to do with it. if (oversample == NO_OVERSAMPLE - && (out.getTransportVersion().before(RESCORE_VECTOR_ALLOW_ZERO) - || (out.getTransportVersion().isPatchFrom(RESCORE_VECTOR_ALLOW_ZERO_BACKPORT_8_19) == false))) { + && out.getTransportVersion().before(RESCORE_VECTOR_ALLOW_ZERO) + && out.getTransportVersion().isPatchFrom(RESCORE_VECTOR_ALLOW_ZERO_BACKPORT_8_19) == false) { throw new ElasticsearchStatusException( "[rescore_vector] does not support a 0 for [" + OVERSAMPLE_FIELD.getPreferredName()