up

Author: ArthurZucker

bindings/python/py_src/tokenizers/processors/__init__.pyi

@@ -138,6 +138,11 @@ class ByteLevel(PostProcessor):
     Args:
         trim_offsets (:obj:`bool`):
             Whether to trim the whitespaces from the produced offsets.
+
+        add_prefix_space (:obj:`bool`, `optional`, defaults to :obj:`True`):
+            If :obj:`True`, keeps the first token's offset as is. If :obj:`False`, increments
+            the start of the first token's offset by 1. Only has an effect if :obj:`trim_offsets`
+            is set to :obj:`True`.
     """
     def __init__(self, add_prefix_space=None, trim_offsets=None, use_regex=None):
         pass