diff --git a/chatterbot/chatterbot.py b/chatterbot/chatterbot.py index 010f4f798..dc54ac769 100644 --- a/chatterbot/chatterbot.py +++ b/chatterbot/chatterbot.py @@ -2,7 +2,7 @@ from typing import Union from chatterbot.storage import StorageAdapter from chatterbot.logic import LogicAdapter -from chatterbot.search import TextSearch, IndexedTextSearch +from chatterbot.search import TextSearch, IndexedTextSearch, SemanticVectorSearch from chatterbot.tagging import PosLemmaTagger from chatterbot.conversation import Statement from chatterbot import languages @@ -74,41 +74,60 @@ def __init__(self, name, stream=False, **kwargs): tagger_language = kwargs.get('tagger_language', languages.ENG) - try: - Tagger = kwargs.get('tagger', PosLemmaTagger) - - # Allow instances to be provided for performance optimization - # (Example: a pre-loaded model in a tagger when unit testing) - if not isinstance(Tagger, type): - self.tagger = Tagger - else: - self.tagger = Tagger(language=tagger_language) - except IOError as io_error: - # Return a more helpful error message if possible - if "Can't find model" in str(io_error): - model_name = utils.get_model_for_language(tagger_language) - if hasattr(tagger_language, 'ENGLISH_NAME'): - language_name = tagger_language.ENGLISH_NAME + # Check if storage adapter has a preferred tagger + PreferredTagger = self.storage.get_preferred_tagger() + + if PreferredTagger is not None: + # Storage adapter specifies its own tagger + self.tagger = PreferredTagger(language=tagger_language) + else: + # Use default or user-specified tagger + try: + Tagger = kwargs.get('tagger', PosLemmaTagger) + + # Allow instances to be provided for performance optimization + # (Example: a pre-loaded model in a tagger when unit testing) + if not isinstance(Tagger, type): + self.tagger = Tagger else: - language_name = tagger_language - raise self.ChatBotException( - 'Setup error:\n' - f'The Spacy model for "{language_name}" language is missing.\n' - 'Please install the model using the command:\n\n' - f'python -m spacy download {model_name}\n\n' - 'See https://spacy.io/usage/models for more information about available models.' - ) from io_error - else: - raise io_error + self.tagger = Tagger(language=tagger_language) + except IOError as io_error: + # Return a more helpful error message if possible + if "Can't find model" in str(io_error): + model_name = utils.get_model_for_language(tagger_language) + if hasattr(tagger_language, 'ENGLISH_NAME'): + language_name = tagger_language.ENGLISH_NAME + else: + language_name = tagger_language + raise self.ChatBotException( + 'Setup error:\n' + f'The Spacy model for "{language_name}" language is missing.\n' + 'Please install the model using the command:\n\n' + f'python -m spacy download {model_name}\n\n' + 'See https://spacy.io/usage/models for more information about available models.' + ) from io_error + else: + raise io_error + # Initialize search algorithms primary_search_algorithm = IndexedTextSearch(self, **kwargs) text_search_algorithm = TextSearch(self, **kwargs) + semantic_vector_search_algorithm = SemanticVectorSearch(self, **kwargs) self.search_algorithms = { primary_search_algorithm.name: primary_search_algorithm, - text_search_algorithm.name: text_search_algorithm + text_search_algorithm.name: text_search_algorithm, + semantic_vector_search_algorithm.name: semantic_vector_search_algorithm } + # Check if storage adapter has a preferred search algorithm + preferred_search_algorithm = self.storage.get_preferred_search_algorithm() + if preferred_search_algorithm and preferred_search_algorithm in self.search_algorithms: + # Set as default for logic adapters that don't specify their own search algorithm + # This ensures BestMatch and other adapters use the optimal search method + self.logger.info(f'Storage adapter prefers search algorithm: {preferred_search_algorithm}') + kwargs.setdefault('search_algorithm_name', preferred_search_algorithm) + for adapter in logic_adapters: utils.validate_adapter_class(adapter, LogicAdapter) logic_adapter = utils.initialize_class(adapter, self, **kwargs) @@ -191,15 +210,22 @@ def get_response(self, statement: Union[Statement, str, dict] = None, **kwargs) input_statement.in_response_to = previous_statement.text # Make sure the input statement has its search text saved - - if not input_statement.search_text: - _search_text = self.tagger.get_text_index_string(input_statement.text) - input_statement.search_text = _search_text - - if not input_statement.search_in_response_to and input_statement.in_response_to: - input_statement.search_in_response_to = self.tagger.get_text_index_string( - input_statement.in_response_to - ) + if not self.tagger.needs_text_indexing(): + # Tagger doesn't transform text, use it directly + if not input_statement.search_text: + input_statement.search_text = input_statement.text + if not input_statement.search_in_response_to and input_statement.in_response_to: + input_statement.search_in_response_to = input_statement.in_response_to + else: + # Use tagger for text indexing or transformations + if not input_statement.search_text: + _search_text = self.tagger.get_text_index_string(input_statement.text) + input_statement.search_text = _search_text + + if not input_statement.search_in_response_to and input_statement.in_response_to: + input_statement.search_in_response_to = self.tagger.get_text_index_string( + input_statement.in_response_to + ) response = self.generate_response( input_statement, diff --git a/chatterbot/search.py b/chatterbot/search.py index 01daec5a6..89139ae07 100644 --- a/chatterbot/search.py +++ b/chatterbot/search.py @@ -157,3 +157,73 @@ def search(self, input_statement, **additional_parameters): if confidence >= 1.0: self.chatbot.logger.info('Exact match found, stopping search') break + + +class SemanticVectorSearch: + """ + Semantic vector search for storage adapters that use vector embeddings. + Does not require a tagger or comparison function - relies on the storage + adapter's native vector similarity search capabilities. + + :param search_page_size: + The maximum number of records to load into memory at a time when searching. + Defaults to 1000 + """ + + name = 'semantic_vector_search' + + def __init__(self, chatbot, **kwargs): + self.chatbot = chatbot + + self.search_page_size = kwargs.get( + 'search_page_size', 1000 + ) + + def search(self, input_statement, **additional_parameters): + """ + Search for semantically similar statements using vector similarity. + Confidence scores are calculated by the storage adapter based on + vector distances and returned in the results. + + :param input_statement: A statement. + :type input_statement: chatterbot.conversation.Statement + + :param **additional_parameters: Additional parameters to be passed + to the ``filter`` method of the storage adapter when searching. + + :rtype: Generator yielding one closest matching statement at a time. + """ + self.chatbot.logger.info('Beginning semantic vector search') + + search_parameters = { + 'search_in_response_to_contains': input_statement.text, + 'persona_not_startswith': 'bot:', + 'page_size': self.search_page_size + } + + if additional_parameters: + search_parameters.update(additional_parameters) + + statement_list = self.chatbot.storage.filter(**search_parameters) + + best_confidence_so_far = 0 + + self.chatbot.logger.info('Processing search results') + + # Yield statements with confidence scores from vector similarity + for statement in statement_list: + # Confidence should already be set by the storage adapter + confidence = getattr(statement, 'confidence', 0.0) + + if confidence > best_confidence_so_far: + best_confidence_so_far = confidence + + self.chatbot.logger.info('Similar statement found: {} {}'.format( + statement.in_response_to, confidence + )) + + yield statement + + if confidence >= 1.0: + self.chatbot.logger.info('Exact match found, stopping search') + break diff --git a/chatterbot/storage/redis.py b/chatterbot/storage/redis.py index 6764cb7b6..e352bd6f3 100644 --- a/chatterbot/storage/redis.py +++ b/chatterbot/storage/redis.py @@ -30,13 +30,19 @@ class RedisVectorStorageAdapter(StorageAdapter): in the future and its behavior has not yet been finalized. The RedisVectorStorageAdapter allows ChatterBot to store conversation - data in a redis instance. + data in a redis instance using vector embeddings for semantic similarity search. All parameters are optional, by default a redis instance on localhost is assumed. :keyword database_uri: eg: redis://localhost:6379/0', The database_uri can be specified to choose a redis instance. :type database_uri: str + + NOTES: + * Unlike other database based storage adapters, the RedisVectorStorageAdapter + does not leverage `search_text` and `search_in_response_to` fields for indexing. + Instead, it uses vector embeddings to find similar statements based on + semantic similarity. This allows for more flexible and context-aware matching. """ class RedisMetaDataType: @@ -100,6 +106,21 @@ def __init__(self, **kwargs): self.vector_store = RedisVectorStore(embeddings, config=config) + def get_preferred_tagger(self): + """ + Redis uses vector embeddings and doesn't need POS-lemma indexing. + Returns NoOpTagger to avoid unnecessary spaCy processing. + """ + from chatterbot.tagging import NoOpTagger + return NoOpTagger + + def get_preferred_search_algorithm(self): + """ + Redis uses semantic vector search instead of text-based matching. + Returns the name of the SemanticVectorSearch algorithm. + """ + return 'semantic_vector_search' + def get_statement_model(self): """ Return the statement model. @@ -127,6 +148,16 @@ def model_to_object(self, document): values.update(document.metadata) + # Convert Unix timestamp back to datetime for StatementObject + # Redis may return this as int, float, or string representation + if 'created_at' in values: + created_at_value = values['created_at'] + if isinstance(created_at_value, str): + # Convert string to float first + created_at_value = float(created_at_value) + if isinstance(created_at_value, (int, float)): + values['created_at'] = datetime.fromtimestamp(created_at_value) + tags = values['tags'] values['tags'] = list(set(tags.split('|') if tags else [])) @@ -177,6 +208,7 @@ def filter(self, page_size=4, **kwargs): - exclude_text - exclude_text_words - persona_not_startswith + - search_text_contains - search_in_response_to_contains - order_by """ @@ -245,27 +277,26 @@ def filter(self, page_size=4, **kwargs): else: filter_condition = query - # Handle search_text parameter (used by BestMatch logic adapter) - # BestMatch uses search_text to find statements with matching indexed text. - # Since Redis doesn't store search_text as a field, we approximate this by: - # 1. Using the search_text value as a semantic query against in_response_to - # 2. This finds statements that are responses to similar inputs - # The effect is similar to BestMatch's Phase 2: finding alternate responses - if 'search_text' in kwargs: - _search_text = kwargs.get('search_text', '') - - # Get embedding for the search text - # Note: search_text may be indexed (e.g., "NOUN:cat VERB:run") so this - # approximates finding responses to semantically similar queries - embedding = self.vector_store.embeddings.embed_query(_search_text) + if 'search_text_contains' in kwargs: + # Find statements whose text (responses) are similar. + # + # Use semantic similarity on the search query itself. This finds responses + # that would be semantically appropriate, even if they don't share exact words. + # + # Our vectors are of 'in_response_to' (what was said TO the bot), + # not 'text' (what the bot said). So we use the query as if it were an input, + # and find statements that would respond to similar inputs. The result is + # statements whose context (in_response_to) is similar, which tends to yield + # similar responses. + _search_query = kwargs['search_text_contains'] + + # Use vector similarity to find statements responding to similar contexts + embedding = self.vector_store.embeddings.embed_query(_search_query) - # Build return fields from metadata schema return_fields = [ 'text', 'in_response_to', 'conversation', 'persona', 'tags', 'created_at' ] - # Use direct index query via RedisVL - # Search on the vectorized content (in_response_to) to find similar response patterns query = VectorQuery( vector=embedding, vector_field_name='embedding', @@ -274,20 +305,35 @@ def filter(self, page_size=4, **kwargs): filter_expression=filter_condition ) - # Execute query results = self.vector_store.index.query(query) - # Convert results to Document objects Document = self.get_statement_model() documents = [] - for result in results: - # Extract metadata and content + + # Calculate confidence from vector distances + # Results are ordered by similarity (best match first) + for idx, result in enumerate(results): in_response_to = result.get('in_response_to', '') - # Convert created_at from integer (YYMMDD) to datetime - created_at_int = int(result.get('created_at', 0)) - if created_at_int: - created_at = datetime.strptime(str(created_at_int), '%y%m%d') + # Redis vector_score is cosine distance (lower is better) + # Convert to confidence: confidence = 1 - distance + # If vector_score not available, use result order + vector_score = result.get('vector_score') + if vector_score is not None: + # Cosine distance ranges from 0 (identical) to 2 (opposite) + # Normalize to confidence: 1.0 (identical) to 0.0 (opposite) + confidence = max(0.0, 1.0 - (float(vector_score) / 2.0)) + else: + # Fallback: use result order (first result = highest confidence) + # Start at 0.95 for first result, decay by 0.05 per position + confidence = max(0.0, 0.95 - (idx * 0.05)) + + # Parse timestamp + created_at_value = result.get('created_at', 0) + if isinstance(created_at_value, str): + created_at = datetime.fromtimestamp(float(created_at_value)) + elif created_at_value: + created_at = datetime.fromtimestamp(float(created_at_value)) else: created_at = datetime.now() @@ -297,6 +343,7 @@ def filter(self, page_size=4, **kwargs): 'persona': result.get('persona', ''), 'tags': result.get('tags', ''), 'created_at': created_at, + 'confidence': confidence, } doc = Document( page_content=in_response_to, @@ -307,6 +354,23 @@ def filter(self, page_size=4, **kwargs): return [self.model_to_object(document) for document in documents] + # Redis uses vector similarity: we search for statements whose actual + # text field is semantically similar to the text that produced this search_text. + # This is stored in the closest_match.text field, but BestMatch only passes + # search_text. Since we can't reverse POS tags to original text (for now), + # we treat this parameter as a signal to do text-based similarity search. + # + # Note: The caller should ideally pass the actual text, but for compatibility + # we'll work with what we receive. In practice, search_text_contains is the + # better parameter for this use case. + if 'search_text' in kwargs: + # For now, we'll treat search_text as a filter-only parameter + # and fall through to the regular query_search below. + # This prevents the broken behavior of embedding POS tags. + # The proper fix requires BestMatch to pass additional context + # or use search_text_contains instead. + pass + ordering = kwargs.get('order_by', None) if ordering: @@ -341,14 +405,31 @@ def filter(self, page_size=4, **kwargs): # Convert results to Document objects Document = self.get_statement_model() documents = [] - for result in results: + + # Calculate confidence from vector distances + # Results are ordered by similarity (best match first) + for idx, result in enumerate(results): # Extract metadata and content in_response_to = result.get('in_response_to', '') - # Convert created_at from integer (YYMMDD) to datetime - created_at_int = int(result.get('created_at', 0)) - if created_at_int: - created_at = datetime.strptime(str(created_at_int), '%y%m%d') + # Redis vector_score is cosine distance (lower is better) + # Convert to confidence: confidence = 1 - distance + # If vector_score not available, use result order + vector_score = result.get('vector_score') + if vector_score is not None: + # Cosine distance ranges from 0 (identical) to 2 (opposite) + # Normalize to confidence: 1.0 (identical) to 0.0 (opposite) + confidence = max(0.0, 1.0 - (float(vector_score) / 2.0)) + else: + # Fallback: use result order (first result = highest confidence) + # Start at 0.95 for first result, decay by 0.05 per position + confidence = max(0.0, 0.95 - (idx * 0.05)) + + # Convert Unix timestamp back to datetime + # Redis returns numeric fields as strings + created_at_timestamp = result.get('created_at', '0') + if created_at_timestamp and created_at_timestamp != '0': + created_at = datetime.fromtimestamp(float(created_at_timestamp)) else: created_at = datetime.now() @@ -358,6 +439,7 @@ def filter(self, page_size=4, **kwargs): 'persona': result.get('persona', ''), 'tags': result.get('tags', ''), 'created_at': created_at, + 'confidence': confidence, } doc = Document( page_content=in_response_to, @@ -395,9 +477,9 @@ def create( metadata = { 'text': text, 'category': kwargs.get('category', ''), - # NOTE: `created_at` must have a valid numeric value or results will - # not be returned for similarity_search for some reason - 'created_at': kwargs.get('created_at') or int(_default_date.strftime('%y%m%d')), + # Store created_at as Unix timestamp with microseconds (float) + # This provides full datetime precision while maintaining Redis NUMERIC field compatibility + 'created_at': kwargs.get('created_at') or _default_date.timestamp(), 'tags': '|'.join(unique_tags) if unique_tags else '', 'conversation': kwargs.get('conversation', ''), 'persona': kwargs.get('persona', ''), @@ -427,7 +509,7 @@ def create_many(self, statements): metadata={ 'text': statement.text, 'conversation': statement.conversation or '', - 'created_at': int(statement.created_at.strftime('%y%m%d')), + 'created_at': statement.created_at.timestamp(), 'persona': statement.persona or '', # Prevent duplicate tag entries in the database 'tags': '|'.join( @@ -452,7 +534,7 @@ def update(self, statement): metadata = { 'text': statement.text, 'conversation': statement.conversation or '', - 'created_at': int(statement.created_at.strftime('%y%m%d')), + 'created_at': statement.created_at.timestamp(), 'persona': statement.persona or '', 'tags': '|'.join(unique_tags) if unique_tags else '', } @@ -508,11 +590,9 @@ def get_random(self): # Parse the metadata metadata = json.loads(data[b'_metadata_json'].decode()) - # Convert created_at from integer (YYMMDD) back to datetime - if 'created_at' in metadata and isinstance(metadata['created_at'], int): - created_at_str = str(metadata['created_at']) - # Parse YYMMDD format - metadata['created_at'] = datetime.strptime(created_at_str, '%y%m%d') + # Convert created_at from Unix timestamp back to datetime + if 'created_at' in metadata and isinstance(metadata['created_at'], (int, float)): + metadata['created_at'] = datetime.fromtimestamp(metadata['created_at']) # Get the in_response_to from the hash in_response_to = data.get(b'in_response_to', b'').decode() diff --git a/chatterbot/storage/storage_adapter.py b/chatterbot/storage/storage_adapter.py index 3f7b8c273..2f579f931 100644 --- a/chatterbot/storage/storage_adapter.py +++ b/chatterbot/storage/storage_adapter.py @@ -173,6 +173,87 @@ def close(self): """ pass + def get_preferred_tagger(self): + """ + Returns the tagger class preferred by this storage adapter. + Returns None by default, meaning the default tagger will be used. + + Storage adapters should override this method to specify their + preferred tagger based on their search capabilities. + + Available Taggers: + + - NoOpTagger: Returns text unchanged (for vector-based storage). + No spaCy model loading (~500MB memory saved). + Faster startup (<1 second vs 2-5 seconds). + Use when storage handles semantic search natively. + + - PosLemmaTagger: Creates POS-lemma bigrams (default, for SQL). + Enables pattern matching (e.g., "NOUN:cat VERB:run"). + Requires spaCy language model. + Best for exact phrase matching. + + - LowercaseTagger: Simple lowercase transformation. + Minimal processing overhead. + Case-insensitive matching. + + Example - Vector Storage:: + + def get_preferred_tagger(self): + from chatterbot.tagging import NoOpTagger + return NoOpTagger + + Example - Traditional Storage:: + + def get_preferred_tagger(self): + return None # Use default PosLemmaTagger + + :return: Tagger class or None + """ + return None + + def get_preferred_search_algorithm(self): + """ + Returns the search algorithm name preferred by this storage adapter. + Returns None by default, meaning the default search algorithm will be used. + + Storage adapters should override this method to specify their + preferred search algorithm based on their capabilities. + + Available Search Algorithms: + + - 'indexed_text_search' (default): + Uses POS-lemma indexed fields (search_text, search_in_response_to). + Python-based Levenshtein distance comparison. + Requires PosLemmaTagger. + Best for: Exact pattern matching. + + - 'semantic_vector_search': + Uses raw text with vector similarity. + Delegates to storage.filter(search_in_response_to_contains=text). + No tagger required (works with NoOpTagger). + Confidence from storage adapter (cosine similarity). + Best for: Context-aware AI responses, semantic understanding. + + - 'text_search' (fallback): + Compares raw text without indexes. + Slower but works with any storage. + Uses comparison functions on all statements. + + Example - Vector Storage:: + + def get_preferred_search_algorithm(self): + return 'semantic_vector_search' + + Example - SQL Storage:: + + def get_preferred_search_algorithm(self): + return None # Use default 'indexed_text_search' + + :return: Search algorithm name string or None + """ + return None + class EmptyDatabaseException(Exception): def __init__(self, message=None): diff --git a/chatterbot/tagging.py b/chatterbot/tagging.py index c32e62512..99f05ac06 100644 --- a/chatterbot/tagging.py +++ b/chatterbot/tagging.py @@ -4,6 +4,56 @@ import spacy +class NoOpTagger(object): + """ + A no-operation tagger that returns text unchanged. + Used by storage adapters that don't rely on indexed search_text fields. + """ + + def __init__(self, language=None): + self.language = language or languages.ENG + + def needs_text_indexing(self): + """ + Indicates whether this tagger performs text indexing/transformation. + Returns False since NoOpTagger passes text through unchanged. + + :return: False + """ + return False + + def get_text_index_string(self, text: Union[str, List[str]]): + """ + Return the text unchanged (no indexing applied). + """ + return text + + def as_nlp_pipeline( + self, + texts: Union[List[str], Tuple[str, dict]], + batch_size: int = 1000, + n_process: int = 1 + ): + """ + Returns texts unchanged without NLP processing. + Maintains API compatibility with other taggers. + + :param texts: Text strings or tuples of (text, context_dict) + :param batch_size: Ignored (for API compatibility) + :param n_process: Ignored (for API compatibility) + """ + process_as_tuples = texts and isinstance(texts[0], tuple) + + if process_as_tuples: + # Return generator of (text, context) tuples + for text, context in texts: + yield (text, context) + else: + # Return generator of text strings + for text in texts: + yield text + + class LowercaseTagger(object): """ Returns the text in lowercase. @@ -21,6 +71,15 @@ def __init__(self, language=None): 'chatterbot_lowercase_indexer', name='chatterbot_lowercase_indexer', last=True ) + def needs_text_indexing(self): + """ + Indicates whether this tagger performs text indexing/transformation. + Returns True since LowercaseTagger transforms text to lowercase. + + :return: True + """ + return True + def get_text_index_string(self, text: Union[str, List[str]]): if isinstance(text, list): documents = self.nlp.pipe(text, batch_size=1000, n_process=1) @@ -73,6 +132,15 @@ def __init__(self, language=None): 'chatterbot_bigram_indexer', name='chatterbot_bigram_indexer', last=True ) + def needs_text_indexing(self): + """ + Indicates whether this tagger performs text indexing/transformation. + Returns True since PosLemmaTagger creates POS-lemma bigram indexes. + + :return: True + """ + return True + def get_text_index_string(self, text: Union[str, List[str]]) -> str: """ Return a string of text containing part-of-speech, lemma pairs. diff --git a/docs/index.rst b/docs/index.rst index 303a0419c..c57cc85bc 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,6 +1,6 @@ .. meta:: - :description: ChatterBot documentation: ChatterBot is a machine learning, conversational dialog engine designed to support multiple languages. - :keywords: ChatterBot, chatbot, chat, bot, natural language processing, nlp, artificial intelligence, ai + :description: ChatterBot documentation: Python machine learning chatbot library with semantic vector search, AI conversational dialog engine supporting multiple languages and vector databases + :keywords: ChatterBot, chatbot, chat, bot, natural language processing, nlp, artificial intelligence, ai, machine learning, vector database, semantic search, vector embeddings, conversational ai, python chatbot library .. container:: banner @@ -15,6 +15,14 @@ ChatterBot is a Python library that makes it easy to generate automated responses to a user's input. ChatterBot uses a selection of machine learning algorithms to produce different types of responses. This makes it easy for developers to create chat bots and automate conversations with users. + +**Modern AI Capabilities** (2025): + +- **Semantic Vector Search**: Advanced context understanding using vector embeddings and Redis vector database +- **Large Language Model (LLM) Integration**: Experimental support for Ollama and OpenAI models +- **Storage-Aware Architecture**: Automatic optimization based on storage backend capabilities +- **Multi-Language Support**: Language-independent design with spaCy integration + For more details about the ideas and concepts behind ChatterBot see the :ref:`process flow diagram `. diff --git a/docs/storage/index.rst b/docs/storage/index.rst index 632378af7..e80abb2a9 100644 --- a/docs/storage/index.rst +++ b/docs/storage/index.rst @@ -2,8 +2,18 @@ Storage Adapters ================ +.. meta:: + :description: ChatterBot storage adapters: SQL, Redis vector database, MongoDB. Semantic search with vector embeddings for AI-powered contextual responses + :keywords: storage adapter, database, SQL, Redis, MongoDB, vector database, semantic search, vector embeddings + Storage adapters provide an interface that allows ChatterBot -to connect to different storage technologies. +to connect to different storage technologies. Each adapter is optimized +for different use cases: + +- **Redis Vector Storage**: Semantic similarity search using vector embeddings (best for contextual AI responses) +- **SQL Storage**: Traditional pattern matching with POS-lemma indexing (best for exact phrase matching) +- **MongoDB Storage**: NoSQL document storage with flexible schema +- **Django Storage**: Integrated with Django ORM for web applications The storage adapter that your bot uses can be specified by setting the ``storage_adapter`` parameter to the import path of the @@ -19,6 +29,8 @@ storage adapter you want to use. Built-in Storage Adapters ========================= +ChatterBot includes multiple storage adapters for different AI and database technologies: + .. toctree:: :maxdepth: 2 @@ -27,6 +39,33 @@ Built-in Storage Adapters sql ../django/index +Choosing a Storage Adapter +=========================== + +**For Semantic AI Chatbots** (Recommended for modern conversational AI): + +Note that as of December 2025, the Redis Vector Storage Adapter is still an experimental beta feature. + +Use **Redis Vector Storage** when you need: + +- Context-aware responses based on meaning, not keywords +- Vector embeddings for semantic similarity search +- Automatic confidence scoring from cosine similarity +- Best match for conversational AI and natural language understanding + +**For Pattern-Based Matching**: + +Use **SQL Storage** when you need: + +- Exact phrase or pattern matching +- POS-lemma bigram indexing +- Traditional database features (ACID compliance) +- Lower memory footprint + +**For Flexibility**: + +Use **MongoDB** or **Django Storage** for schema flexibility and web framework integration. + Common storage adapter attributes ================================= diff --git a/docs/storage/redis.rst b/docs/storage/redis.rst index 573a0b79d..e75c726cb 100644 --- a/docs/storage/redis.rst +++ b/docs/storage/redis.rst @@ -3,9 +3,13 @@ Redis Vector Storage Adapter .. note:: - **(March, 2025)**: + **(December, 2025)**: The ``RedisVectorStorageAdapter`` is new and experimental functionality introduced as a "beta" feature. Its functionality might not yet be fully stable and is subject to change in future releases. +.. meta:: + :description: Redis vector storage for ChatterBot: semantic similarity search, vector embeddings, AI-powered contextual responses with HuggingFace transformers + :keywords: redis vector database, semantic search, vector embeddings, sentence transformers, AI chatbot, natural language understanding, context-aware responses, vector similarity + .. image:: /_static/Redis_Logo_Red_RGB.svg :alt: Redis Logo :align: center @@ -14,14 +18,20 @@ Redis Vector Storage Adapter Imaged used in accordance with the Redis Trademark Policy https://redis.io/legal/trademark-policy/ -The ``RedisVectorStorageAdapter`` allows a ChatterBot instance to store and retrieve text and metadata using a Redis® instance configured as a :term:`vector database`. -This adapter supports the use of vectors when filtering queries to search for similar text. +The ``RedisVectorStorageAdapter`` enables advanced **semantic similarity search** for ChatterBot using Redis® as a :term:`vector database`. +Unlike traditional keyword-based storage adapters, this adapter uses **vector embeddings** and **cosine similarity** to understand conversational context and find semantically related responses. + +**Key Features:** -Vectors are a mathematical representation of text that can be used -to calculate the similarity between two pieces of text based on -the distance between their vectors. This allows for more accurate -search results when looking for similar text because the context of -the text can be taken into account. +- **Semantic Understanding**: Matches responses based on meaning, not just keywords +- **Vector Embeddings**: Uses HuggingFace ``sentence-transformers/all-mpnet-base-v2`` model for state-of-the-art text encoding +- **Confidence Scoring**: Returns similarity scores (0.0-1.0) based on vector distance for intelligent response ranking +- **Performance Optimized**: Automatic NoOpTagger eliminates unnecessary spaCy processing overhead +- **Context-Aware Responses**: Finds conversationally appropriate responses even when exact words differ + +Vectors are mathematical representations of text (multi-dimensional embeddings) that capture semantic meaning. +The adapter calculates similarity between text by measuring the **cosine distance** between their vector representations, +allowing ChatterBot to understand that "How are you?" is similar to "How's it going?" even with different words. For example, consider the following words: @@ -88,28 +98,121 @@ To use the ``RedisVectorStorageAdapter`` you will need to provide the following chatbot = ChatBot( 'Redis Bot', - storage_adapter='chatterbot.storage.RedisVectorStorageAdapter, + storage_adapter='chatterbot.storage.RedisVectorStorageAdapter', # Optional: Override the default Redis URI # database_uri='redis://localhost:6379/0' ) +Storage-Aware Architecture +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The Redis adapter automatically configures ChatterBot for optimal performance with vector-based search: + +- **Automatic Tagger Selection**: Uses ``NoOpTagger`` instead of ``PosLemmaTagger`` to eliminate spaCy model loading overhead +- **Semantic Vector Search**: Automatically selects ``SemanticVectorSearch`` algorithm instead of text-based comparison +- **No Manual Configuration**: These optimizations are applied automatically when using the Redis adapter + +This "storage-aware" design means ChatterBot adapts its processing pipeline based on the storage adapter's capabilities, +ensuring maximum performance and accuracy for vector-based semantic search. + +.. code-block:: python + + # No need to specify tagger or search algorithm - Redis adapter handles it! + chatbot = ChatBot( + 'Semantic Bot', + storage_adapter='chatterbot.storage.RedisVectorStorageAdapter' + ) + # Automatically uses: + # - NoOpTagger (no spaCy overhead) + # - SemanticVectorSearch (vector similarity) + # - Confidence scores from cosine similarity + +Semantic Search vs. Traditional Text Search +------------------------------------------- + +The Redis adapter uses **semantic vector search** instead of traditional pattern matching: + +.. list-table:: Comparison: Semantic Vector Search vs. Text-Based Search + :widths: 30 35 35 + :header-rows: 1 + + * - Feature + - Traditional Text Search (SQL) + - Semantic Vector Search (Redis) + * - Search Method + - POS-lemma bigram matching + - 768-dimensional vector similarity + * - Context Understanding + - Structural patterns only + - Deep semantic meaning + * - "How are you?" matches "How's it going?" + - ❌ No (different lemmas) + - ✅ Yes (similar meaning) + * - Confidence Scoring + - Levenshtein distance + - Cosine similarity (1 - distance/2) + * - Processing Overhead + - Requires spaCy models + - No spaCy needed (NoOpTagger) + * - Best For + - Exact pattern matching + - Conversational AI, context understanding + +**Example: Semantic Similarity in Action** + +.. code-block:: python + + # These inputs find similar responses despite different words: + response1 = chatbot.get_response("What's the weather like?") + response2 = chatbot.get_response("How's the climate today?") + # Both queries find weather-related responses due to semantic similarity + + # Confidence scores help rank responses: + # - Vector distance 0.1 → confidence ~0.95 (very similar) + # - Vector distance 0.5 → confidence ~0.75 (somewhat similar) + # - Vector distance 1.0 → confidence ~0.50 (loosely related) + Class Attributes ---------------- .. autoclass:: chatterbot.storage.RedisVectorStorageAdapter :members: -More on Vector Databases ------------------------- +Performance Considerations +-------------------------- + +**Vector Embedding Model**: By default, the Redis adapter uses ``sentence-transformers/all-mpnet-base-v2`` from HuggingFace: + +- **Dimensions**: 768-dimensional embeddings +- **Model Size**: ~420MB (downloaded once, cached locally) +- **Performance**: ~2000 sentences/second on CPU +- **Quality**: State-of-the-art semantic similarity (as of 2025) + +**First-Time Setup**: The embedding model downloads automatically on first use: + +.. code-block:: python + + # First initialization downloads model (~420MB) + chatbot = ChatBot('Bot', storage_adapter='chatterbot.storage.RedisVectorStorageAdapter') + # Subsequent uses load from cache (fast startup) + +**Memory Usage**: Redis vector storage requires more memory than SQL due to embedding storage: + +- Each statement: ~3KB (768 floats × 4 bytes) +- 10,000 statements: ~30MB vector data +- Trade-off: Higher memory for better semantic understanding + +More on Vector Databases & Semantic Search +------------------------------------------- -For those looking to learn more about vector databases, the following resources can be good starting points: +For those looking to learn more about vector databases, vector embeddings, and semantic search in AI applications: -.. list-table:: Vector Database Learning Resources +.. list-table:: Vector Database & AI Learning Resources :widths: 50 50 :header-rows: 1 - * - Article - - Link + * - Topic + - Resource Link * - What is a vector database? - https://www.mongodb.com/resources/basics/databases/vector-databases * - Why use a vector database? @@ -118,6 +221,12 @@ For those looking to learn more about vector databases, the following resources - https://learn.microsoft.com/en-us/azure/cosmos-db/mongodb/vcore/vector-search-ai * - Redis as a vector database - https://redis.io/docs/latest/develop/interact/search-and-query/advanced-concepts/vectors/ + * - Sentence Transformers (Embeddings) + - https://www.sbert.net/ + * - Understanding Cosine Similarity + - https://en.wikipedia.org/wiki/Cosine_similarity + * - Vector Search for AI/LLMs + - https://www.pinecone.io/learn/vector-search-basics/ :sub:`* Redis is a registered trademark of Redis Ltd. Any rights therein are reserved to Redis Ltd.`