Skip to content

Commit e6a0186

Browse files
authored
Add support for INT vector types (#3616)
* Add int vector types * Skip tests on older versions
1 parent e81f4fe commit e6a0186

3 files changed

Lines changed: 104 additions & 1 deletion

File tree

src/main/java/io/lettuce/core/search/arguments/VectorFieldArgs.java

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,15 @@ public enum VectorType {
102102
/**
103103
* 64-bit floating point.
104104
*/
105-
FLOAT64
105+
FLOAT64,
106+
/**
107+
* 8-bit signed integer.
108+
*/
109+
INT8,
110+
/**
111+
* 8-bit unsigned integer.
112+
*/
113+
UINT8
106114
}
107115

108116
/**

src/test/java/io/lettuce/core/search/RediSearchVectorIntegrationTests.java

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,8 @@
3333
import org.junit.jupiter.api.Disabled;
3434
import org.junit.jupiter.api.Tag;
3535
import org.junit.jupiter.api.Test;
36+
import org.junit.jupiter.params.ParameterizedTest;
37+
import org.junit.jupiter.params.provider.MethodSource;
3638

3739
import java.nio.ByteBuffer;
3840
import java.nio.ByteOrder;
@@ -45,6 +47,7 @@
4547
import java.util.List;
4648
import java.util.Map;
4749
import java.util.Set;
50+
import java.util.stream.Stream;
4851

4952
import static io.lettuce.TestTags.INTEGRATION_TEST;
5053
import static org.assertj.core.api.Assertions.assertThat;
@@ -1186,4 +1189,94 @@ public ByteBuffer encodeValue(Object value) {
11861189
}
11871190
}
11881191

1192+
static Stream<VectorFieldArgs.VectorType> quantizedVectorTypes() {
1193+
return Stream.of(VectorFieldArgs.VectorType.INT8, VectorFieldArgs.VectorType.UINT8);
1194+
}
1195+
1196+
@ParameterizedTest
1197+
@MethodSource("quantizedVectorTypes")
1198+
void testQuantizedVectorTypes(VectorFieldArgs.VectorType vectorType) {
1199+
assumeTrue(RedisConditions.of(redis).hasVersionGreaterOrEqualsTo("8.0"));
1200+
1201+
String typeName = vectorType.name();
1202+
String indexName = typeName.toLowerCase() + "-idx";
1203+
String prefix = typeName.toLowerCase() + ":";
1204+
String fieldName = "embedding_" + typeName.toLowerCase();
1205+
1206+
// Create vector field with appropriate algorithm based on type
1207+
FieldArgs<String> vectorField;
1208+
if (vectorType == VectorFieldArgs.VectorType.INT8) {
1209+
// INT8 with FLAT algorithm and L2 distance
1210+
vectorField = VectorFieldArgs.<String> builder().name(fieldName).flat().type(vectorType).dimensions(4)
1211+
.distanceMetric(VectorFieldArgs.DistanceMetric.L2).build();
1212+
} else {
1213+
// UINT8 with HNSW algorithm and COSINE distance
1214+
vectorField = VectorFieldArgs.<String> builder().name(fieldName).hnsw().type(vectorType).dimensions(4)
1215+
.distanceMetric(VectorFieldArgs.DistanceMetric.COSINE).attribute("M", 16).attribute("EF_CONSTRUCTION", 200)
1216+
.build();
1217+
}
1218+
1219+
FieldArgs<String> nameField = TextFieldArgs.<String> builder().name("name").build();
1220+
1221+
CreateArgs<String, String> createArgs = CreateArgs.<String, String> builder().withPrefix(prefix)
1222+
.on(CreateArgs.TargetType.HASH).build();
1223+
1224+
redis.ftCreate(indexName, createArgs, Arrays.asList(vectorField, nameField));
1225+
1226+
// Add vectors based on type
1227+
byte[] vector1, vector2, vector3, queryVector;
1228+
if (vectorType == VectorFieldArgs.VectorType.INT8) {
1229+
// INT8 vectors (signed 8-bit: -128 to 127)
1230+
vector1 = new byte[] { 10, 20, 30, 40 };
1231+
vector2 = new byte[] { -50, 60, -70, 80 };
1232+
vector3 = new byte[] { 15, 25, 35, 45 };
1233+
queryVector = new byte[] { 12, 22, 32, 42 };
1234+
} else {
1235+
// UINT8 vectors (unsigned 8-bit: 0 to 255, stored as signed bytes in Java)
1236+
vector1 = new byte[] { (byte) 100, (byte) 150, (byte) 200, (byte) 250 };
1237+
vector2 = new byte[] { (byte) 50, (byte) 100, (byte) 150, (byte) 200 };
1238+
vector3 = new byte[] { (byte) 110, (byte) 160, (byte) 210, (byte) 240 };
1239+
queryVector = new byte[] { (byte) 105, (byte) 155, (byte) 205, (byte) 245 };
1240+
}
1241+
1242+
// Store documents
1243+
Map<String, Object> doc1 = new HashMap<>();
1244+
doc1.put("name", typeName + " Vector 1");
1245+
doc1.put(fieldName, vector1);
1246+
storeHashDocument(prefix + "1", doc1);
1247+
1248+
Map<String, Object> doc2 = new HashMap<>();
1249+
doc2.put("name", typeName + " Vector 2");
1250+
doc2.put(fieldName, vector2);
1251+
storeHashDocument(prefix + "2", doc2);
1252+
1253+
Map<String, Object> doc3 = new HashMap<>();
1254+
doc3.put("name", typeName + " Vector 3");
1255+
doc3.put(fieldName, vector3);
1256+
storeHashDocument(prefix + "3", doc3);
1257+
1258+
// Test KNN search
1259+
ByteBuffer queryBuffer = ByteBuffer.wrap(queryVector);
1260+
ByteBuffer blobKey = ByteBuffer.wrap("BLOB".getBytes(StandardCharsets.UTF_8));
1261+
SearchArgs<ByteBuffer, ByteBuffer> searchArgs = SearchArgs.<ByteBuffer, ByteBuffer> builder()
1262+
.param(blobKey, queryBuffer).limit(0, 2).build();
1263+
1264+
ByteBuffer queryString = ByteBuffer
1265+
.wrap(("*=>[KNN 2 @" + fieldName + " $BLOB AS distance]").getBytes(StandardCharsets.UTF_8));
1266+
SearchReply<ByteBuffer, ByteBuffer> results = redisBinary.ftSearch(indexName, queryString, searchArgs);
1267+
1268+
assertThat(results.getCount()).isEqualTo(2);
1269+
assertThat(results.getResults()).hasSize(2);
1270+
1271+
// Verify that the search worked with the quantized vectors
1272+
ByteBuffer nameKey = ByteBuffer.wrap("name".getBytes(StandardCharsets.UTF_8));
1273+
for (SearchReply.SearchResult<ByteBuffer, ByteBuffer> result : results.getResults()) {
1274+
String name = new String(result.getFields().get(nameKey).array(), StandardCharsets.UTF_8);
1275+
assertThat(name).contains(typeName + " Vector");
1276+
}
1277+
1278+
// Cleanup
1279+
redis.ftDropindex(indexName);
1280+
}
1281+
11891282
}

src/test/java/io/lettuce/core/search/arguments/VectorFieldArgsTest.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,8 @@ void testVectorFieldArgsWithAllHnswOptions() {
139139
void testVectorTypeEnum() {
140140
assertThat(VectorFieldArgs.VectorType.FLOAT32.name()).isEqualTo("FLOAT32");
141141
assertThat(VectorFieldArgs.VectorType.FLOAT64.name()).isEqualTo("FLOAT64");
142+
assertThat(VectorFieldArgs.VectorType.INT8.name()).isEqualTo("INT8");
143+
assertThat(VectorFieldArgs.VectorType.UINT8.name()).isEqualTo("UINT8");
142144
}
143145

144146
@Test

0 commit comments

Comments
 (0)