added warning messages when OverflowError occurs.

Author: ParticularMiner

setup.py

@@ -26,6 +26,6 @@
                       , 'scikit-learn'
                       , 'numpy'
                       , 'sparse_dot_topn_for_blocks>=0.3.1'
-                      , 'topn>=0.0.6'
+                      , 'topn>=0.0.7'
                       ]
 )

string_grouper/string_grouper.py

@@ -2,6 +2,7 @@
 import numpy as np
 import re
 import multiprocessing
+import warnings
 from sklearn.feature_extraction.text import TfidfVectorizer
 from scipy.sparse import vstack
 from scipy.sparse.csr import csr_matrix
@@ -409,13 +410,19 @@ def divide_by(n, series):
             hblocks = []
             for right_block in block_ranges_right:
                 right_matrix = self._get_right_tf_idf_matrix(right_block)
-
-                # Calculate the matches using the cosine similarity
-                # Note: awesome_cossim_topn will sort each row only when
-                # _max_n_matches < size of right_block or sort=True
-                matches, block_true_max_n_matches = self._build_matches(
-                    left_matrix, right_matrix, nnz_rows, sort=(len(block_ranges_right) == 1)
-                )
+                try:
+                    # Calculate the matches using the cosine similarity
+                    # Note: awesome_cossim_topn will sort each row only when
+                    # _max_n_matches < size of right_block or sort=True
+                    matches, block_true_max_n_matches = self._build_matches(
+                        left_matrix, right_matrix, nnz_rows, sort=(len(block_ranges_right) == 1)
+                    )
+                except OverflowError as oe:
+                    import sys
+                    raise (type(oe)(f"{str(oe)} Use the n_blocks parameter to split-up "
+                                    f"the data into smaller chunks.  The current values"
+                                    f"(n_blocks = {n_blocks}) are too small.")
+                           .with_traceback(sys.exc_info()[2]))
                 hblocks.append(matches)
                 # end of inner loop
 
@@ -478,6 +485,12 @@ def end(partition, left=True):
                 left_matrix, right_matrix, nnz_rows[slice(*left_partition)],
                 sort=sort)
         except OverflowError:
+            warnings.warn("An OverflowError occurred but is being "
+                          "handled.  The input data will be automatically "
+                          "split-up into smaller chunks which will then be "
+                          "processed one chunk at a time.  To prevent "
+                          "OverflowError, use the n_blocks parameter to split-up "
+                          "the data manually into small enough chunks.")
             # Matrices too big!  Try splitting:
             del left_matrix, right_matrix
 

string_grouper/test/test_string_grouper.py

@@ -122,7 +122,7 @@ def test_auto_blocking_single_DataFrame(self):
         # This function will force an OverflowError to occur when
         # the input Series have a combined length above a given number:
         # OverflowThreshold.  This will in turn trigger automatic splitting
-        # of the Series/matrices into smaller blocks
+        # of the Series/matrices into smaller blocks when n_blocks = None
 
         sort_cols = ['right_index', 'left_index']
 
@@ -143,7 +143,7 @@ def fix_row_order(df):
         # Create a custom wrapper for this StringGrouper instance's
         # _build_matches() method which will later be used to
         # mock _build_matches().
-        # Note that we have to define the mock function here because
+        # Note that we have  to  define  the  wrapper  here  because
         # _build_matches() is a non-static function of StringGrouper
         # and needs access to the specific StringGrouper instance sg
         # created here.
@@ -240,6 +240,48 @@ def fix_row_order(df):
             match_strings(df1, n_blocks=(3, 2), min_similarity=0.1))
         pd.testing.assert_frame_equal(matches11, matches32)
 
+        # Create a custom wrapper for this StringGrouper instance's
+        # _build_matches() method which will later be used to
+        # mock _build_matches().
+        # Note that we have  to  define  the  wrapper  here  because
+        # _build_matches() is a non-static function of StringGrouper
+        # and needs access to the specific StringGrouper instance sg
+        # created here.
+        sg = StringGrouper(df1, min_similarity=0.1)
+
+        def mock_build_matches(OverflowThreshold,
+                               real_build_matches=sg._build_matches):
+            def wrapper(left_matrix,
+                        right_matrix,
+                        nnz_rows=None,
+                        sort=True):
+                if (left_matrix.shape[0] + right_matrix.shape[0]) > \
+                        OverflowThreshold:
+                    raise OverflowError
+                return real_build_matches(left_matrix, right_matrix, nnz_rows, sort)
+            return wrapper
+
+        def test_overflow_error_with(OverflowThreshold, n_blocks):
+            nonlocal sg
+            sg._build_matches = Mock(side_effect=mock_build_matches(OverflowThreshold))
+            sg.clear_data()
+            max_left_block_size = (len(df1)//n_blocks[0]
+                                   + (1 if len(df1) % n_blocks[0] > 0 else 0))
+            max_right_block_size = (len(df1)//n_blocks[1]
+                                    + (1 if len(df1) % n_blocks[1] > 0 else 0))
+            if (max_left_block_size + max_right_block_size) > OverflowThreshold:
+                with self.assertRaises(Exception):
+                    _ = sg.match_strings(df1, n_blocks=n_blocks)
+            else:
+                matches_manual = fix_row_order(sg.match_strings(df1, n_blocks=n_blocks))
+                pd.testing.assert_frame_equal(matches11, matches_manual)
+
+        test_overflow_error_with(OverflowThreshold=100, n_blocks=(1, 1))
+        test_overflow_error_with(OverflowThreshold=10, n_blocks=(1, 1))
+        test_overflow_error_with(OverflowThreshold=10, n_blocks=(2, 1))
+        test_overflow_error_with(OverflowThreshold=10, n_blocks=(1, 2))
+        test_overflow_error_with(OverflowThreshold=10, n_blocks=(4, 4))
+
     def test_n_blocks_both_DataFrames(self):
         """tests whether manual blocking yields consistent results"""
         sort_cols = ['right_index', 'left_index']