feat: Add index-level STOPWORDS configuration support

Add support for configuring stopwords at index creation time via IndexInfo.stopwords field.

- Add stopwords field to IndexInfo class (None/[]/custom list)
- Update SearchIndex.create() and AsyncSearchIndex.create() to pass stopwords
- Update convert_index_info_to_schema() to parse stopwords from FT.INFO
- Update cluster_create_index() functions to accept stopwords parameter
- Add warning when using query-time stopwords with index-level STOPWORDS 0
- Add comprehensive documentation in 11_advanced_queries.ipynb
- Create stopwords_interaction_guide.md explaining best practices

Author: nkanu17

docs/user_guide/11_advanced_queries.ipynb

@@ -79,6 +79,7 @@
     BaseVectorQuery,
     CountQuery,
     FilterQuery,
+    TextQuery,
 )
 from redisvl.query.filter import FilterExpression
 from redisvl.redis.connection import (
@@ -248,6 +249,30 @@ def _validate_query(self, query: BaseQuery) -> None:
                     "Vector field using 'flat' algorithm does not support EF_RUNTIME query parameter."
                 )
 
+        # Warn if using query-time stopwords with index-level STOPWORDS 0
+        if isinstance(query, TextQuery):
+            index_stopwords = self.schema.index.stopwords
+            query_stopwords = query.stopwords
+
+            # Check if index has STOPWORDS 0 (empty list) and query has stopwords configured
+            # Note: query.stopwords is a set, and when stopwords=None is passed to TextQuery,
+            # it becomes an empty set. So we check if the set is non-empty.
+            if (
+                index_stopwords is not None
+                and len(index_stopwords) == 0
+                and len(query_stopwords) > 0
+            ):
+                warnings.warn(
+                    "Query-time stopwords are configured but the index has STOPWORDS 0 (stopwords = []). "
+                    "This is counterproductive: all words including common words like 'of', 'the', 'a' are indexed, "
+                    "but your query-time stopwords will filter them from the search query. "
+                    "This makes your search less precise than it could be. "
+                    "Consider setting stopwords=None in TextQuery to search for all indexed words. "
+                    "See docs/stopwords_interaction_guide.md for more information.",
+                    UserWarning,
+                    stacklevel=3,
+                )
+
     @property
     def name(self) -> str:
         """The name of the Redis search index."""
@@ -601,17 +626,22 @@ def create(self, overwrite: bool = False, drop: bool = False) -> None:
             definition = IndexDefinition(
                 prefix=[self.schema.index.prefix], index_type=self._storage.type
             )
+            # Extract stopwords from schema
+            stopwords = self.schema.index.stopwords
+
             if isinstance(self._redis_client, RedisCluster):
                 cluster_create_index(
                     index_name=self.name,
                     client=self._redis_client,
                     fields=redis_fields,
                     definition=definition,
+                    stopwords=stopwords,
                 )
             else:
                 self._redis_client.ft(self.name).create_index(
                     fields=redis_fields,
                     definition=definition,
+                    stopwords=stopwords,
                 )
         except redis.exceptions.RedisError as e:
             raise RedisSearchError(
@@ -1384,17 +1414,22 @@ async def create(self, overwrite: bool = False, drop: bool = False) -> None:
             definition = IndexDefinition(
                 prefix=[self.schema.index.prefix], index_type=self._storage.type
             )
+            # Extract stopwords from schema
+            stopwords = self.schema.index.stopwords
+
             if isinstance(client, AsyncRedisCluster):
                 await async_cluster_create_index(
                     index_name=self.schema.index.name,
                     client=client,
                     fields=redis_fields,
                     definition=definition,
+                    stopwords=stopwords,
                 )
             else:
                 await client.ft(self.schema.index.name).create_index(
                     fields=redis_fields,
                     definition=definition,
+                    stopwords=stopwords,
                 )
         except redis.exceptions.RedisError as e:
             raise RedisSearchError(

redisvl/index/index.py

@@ -204,6 +204,17 @@ def convert_index_info_to_schema(index_info: Dict[str, Any]) -> Dict[str, Any]:
         prefixes = prefixes[0]
     storage_type = index_info["index_definition"][1].lower()
 
+    # Parse stopwords if present in FT.INFO output
+    # stopwords_list is only present when explicitly set (STOPWORDS 0 or custom list)
+    # If not present, we use None to indicate default Redis behavior
+    stopwords = None
+    if "stopwords_list" in index_info:
+        # Convert bytes to strings if needed
+        stopwords_list = index_info["stopwords_list"]
+        stopwords = [
+            sw.decode("utf-8") if isinstance(sw, bytes) else sw for sw in stopwords_list
+        ]
+
     index_fields = index_info["attributes"]
 
     def parse_vector_attrs(attrs):
@@ -411,8 +422,12 @@ def parse_attrs(attrs, field_type=None):
         # append field
         schema_fields.append(field)
 
+    index_dict = {"name": index_name, "prefix": prefixes, "storage_type": storage_type}
+    if stopwords is not None:
+        index_dict["stopwords"] = stopwords
+
     return {
-        "index": {"name": index_name, "prefix": prefixes, "storage_type": storage_type},
+        "index": index_dict,
         "fields": schema_fields,
     }
 

redisvl/redis/connection.py

@@ -1,8 +1,7 @@
-import re
 from collections.abc import Mapping, Sequence
 from enum import Enum
 from pathlib import Path
-from typing import Any, Dict, List, Literal, Union
+from typing import Any, Dict, List, Literal, Optional, Union
 
 import yaml
 from pydantic import BaseModel, Field, model_validator
@@ -31,7 +30,7 @@ class StorageType(Enum):
 
 class IndexInfo(BaseModel):
     """Index info includes the essential details regarding index settings,
-    such as its name, prefix, key separator, and storage type in Redis.
+    such as its name, prefix, key separator, storage type, and stopwords in Redis.
 
     In yaml format, the index info section looks like:
 
@@ -42,6 +41,7 @@ class IndexInfo(BaseModel):
             prefix: user
             key_separtor: ':'
             storage_type: json
+            stopwords: []  # Disable stopwords (STOPWORDS 0)
 
     In dict format, the index info section looks like:
 
@@ -51,7 +51,8 @@ class IndexInfo(BaseModel):
             "name": "user-index",
             "prefix": "user",
             "key_separator": ":",
-            "storage_type": "json"
+            "storage_type": "json",
+            "stopwords": ["the", "a", "an"]  # Custom stopwords
         }}
 
     """
@@ -64,6 +65,9 @@ class IndexInfo(BaseModel):
     """The separator character used in designing Redis keys."""
     storage_type: StorageType = StorageType.HASH
     """The storage type used in Redis (e.g., 'hash' or 'json')."""
+    stopwords: Optional[List[str]] = None
+    """Index-level stopwords configuration. None (default) uses Redis default stopwords,
+    empty list [] disables stopwords (STOPWORDS 0), or provide a custom list of stopwords."""
 
 
 class IndexSchema(BaseModel):