add new config for limits

Author: Patralos

README.rst

@@ -7,6 +7,8 @@ This fork has some added features:
   - support for nested data structures
   - support for set as datatype
   - new http handler with support for ssl and basic auth
+  - New config for limiting strings, containers and stacktraces
+  - Version 0 Logging has been removed
 
 http://logstash.net/
 

example1.py

@@ -6,8 +6,9 @@
 
 test_logger = logging.getLogger('python-logstash-logger')
 test_logger.setLevel(logging.INFO)
-test_logger.addHandler(logstash.LogstashHandler(host, 5959, version=1))
-# test_logger.addHandler(logstash.TCPLogstashHandler(host, 5959, version=1))
+# test_logger.addHandler(logstash.LogstashHandler(host, 5959, version=1))
+test_logger.addHandler(logstash.TCPLogstashHandler(host, 5959, limit_containers=6, limit_stacktrace=10000,
+                                                   limit_string_fields=1000))
 
 test_logger.error('python-logstash: test logstash error message.')
 test_logger.info('python-logstash: test logstash info message.')
@@ -20,6 +21,18 @@
     'test_dict': {'a': 1, 'b': set(['a'])},
     'test_float': 1.23,
     'test_integer': 123,
-    'test_list': [1, 2, 3],
+    'test_list': [1, 2, 3, 4, 5, 6, 7, 8 ],
 }
 test_logger.info('python-logstash: test extra fields', extra=extra)
+
+extra['test_string'] = "A" * 2000
+extra['test_dict'].update({
+    'c': 3,
+    'd': 4,
+    'e': 5,
+    'f': 6,
+    'g': 7,
+    'h': 8
+})
+test_logger.info('python-logstash: test extra fields 2', extra=extra)
+

example2.py

@@ -12,8 +12,7 @@
 test_logger.setLevel(logging.INFO)
 
 # add the handler
-test_logger.addHandler(logstash.AMQPLogstashHandler(version=1,
-                                                    host=host,
+test_logger.addHandler(logstash.AMQPLogstashHandler(host=host,
                                                     durable=True,
                                                     username=username,
                                                     password=password,

example3.py

@@ -6,7 +6,7 @@
 
 test_logger = logging.getLogger('python-logstash-logger')
 test_logger.setLevel(logging.INFO)
-test_logger.addHandler(logstash.HTTPLogstashHandler(host, 1337, version=1, ssl=True, verify=True, username="user", password="pw"))
+test_logger.addHandler(logstash.HTTPLogstashHandler(host, 1337, ssl=True, verify=True, username="user", password="pw"))
 # test_logger.addHandler(logstash.TCPLogstashHandler(host, 5959, version=1))
 
 test_logger.error('python-logstash: test logstash error message.')

logstash/__init__.py

@@ -1,5 +1,5 @@
 
-from logstash.formatter import LogstashFormatterVersion0, LogstashFormatterVersion1
+from logstash.formatter import LogstashFormatter
 
 from logstash.handler_tcp import TCPLogstashHandler
 from logstash.handler_udp import UDPLogstashHandler, LogstashHandler

logstash/formatter.py

@@ -11,15 +11,25 @@
 
 class LogstashFormatterBase(logging.Formatter):
 
-    def __init__(self, message_type='Logstash', tags=None, fqdn=False):
+    limit_stacktrace = 0
+
+    def __init__(self, message_type='Logstash', tags=None, fqdn=False, limit_stacktrace=0, limit_string_fields=0, limit_containers=0):
         self.message_type = message_type
         self.tags = tags if tags is not None else []
+        LogstashFormatterBase.limit_stacktrace = limit_stacktrace
+        self.limit_string_fields = limit_string_fields
+        self.limit_containers = limit_containers
 
         if fqdn:
             self.host = socket.getfqdn()
         else:
             self.host = socket.gethostname()
 
+    def limit_string_field(self, value):
+        if self.limit_string_fields == 0:
+            return value
+        return value[:self.limit_string_fields] + (value[self.limit_string_fields:] and '...Truncated...')
+
     def get_extra_fields(self, record, first=True):
         # The list contains all the attributes listed in
         # http://docs.python.org/library/logging.html#logrecord-attributes
@@ -31,40 +41,54 @@ def get_extra_fields(self, record, first=True):
             'auth_token', 'password')
 
         if sys.version_info < (3, 0):
-            easy_types = (basestring, bool, dict, float, int, long, list, type(None))
+            easy_types = (basestring, bool, float, int, long, type(None))
         else:
-            easy_types = (str, bool, dict, float, int, list, type(None))
+            easy_types = (str, bool, float, int, type(None))
         containers = (dict, set, list)
         fields = {}
         if first:
             for key, value in record.__dict__.items():
                 if key not in skip_list or not first:
                     if isinstance(value, containers):
                         fields[key] = self.get_extra_fields(value, first=False)
+                    elif type(value) == str:
+                        fields[key] = self.limit_string_field(value)
                     elif isinstance(value, easy_types):
                         fields[key] = value
                     else:
-                        fields[key] = repr(value)
+                        fields[key] = self.limit_string_field(repr(value))
         elif type(record) == dict:
+            counter = 0
             for key, value in record.items():
+                counter += 1
+                if self.limit_containers != 0 and counter > self.limit_containers:
+                    fields['WARNING'] = "...Truncated..."
+                    break
                 if isinstance(value, containers):
                     fields[key] = self.get_extra_fields(value, first=False)
+                elif type(value) == str:
+                    fields[key] = self.limit_string_field(value)
                 elif isinstance(value, easy_types):
                     fields[key] = value
                 else:
-                    fields[key] = repr(value)
+                    fields[key] = self.limit_string_field(repr(value))
         elif type(record) == list or type(record) == set:
             tmp = []
+            counter = 0
             for value in record:
+                counter += 1
+                if self.limit_containers != 0 and counter > self.limit_containers:
+                    tmp.append("...Truncated...")
+                    break
                 tmp.append(self.get_extra_fields(value, first=False))
             return tmp
         else:
+            if type(record) == str:
+                return self.limit_string_field(record)
             if isinstance(record, easy_types):
                 return record
             else:
-                return repr(record)
-
-
+                return self.limit_string_field(repr(record))
 
         return fields
 
@@ -97,7 +121,10 @@ def format_timestamp(cls, time):
 
     @classmethod
     def format_exception(cls, exc_info):
-        return ''.join(traceback.format_exception(*exc_info)) if exc_info else ''
+        stacktrace = ''.join(traceback.format_exception(*exc_info)) if exc_info else ''
+        if cls.limit_stacktrace == 0:
+            return stacktrace
+        return stacktrace[:cls.limit_stacktrace] + (stacktrace[cls.limit_stacktrace:] and '..')
 
     @classmethod
     def serialize(cls, message):

logstash/handler_amqp.py

@@ -40,13 +40,17 @@ class AMQPLogstashHandler(SocketHandler, object):
         host (socket.getfqdn()).
     :param facility: Replace facility with specified value. If specified,
         record.name will be passed as `logger` parameter.
+    :param limit_stacktrace: limit characters for stacktraces
+    :param limit_string_fields: limit characters for string fields
+    :param limit_containers: limit length of containers (dict, list, set)
     """
 
     def __init__(self, host='localhost', port=5672, username='guest',
                  password='guest', exchange='logstash', exchange_type='fanout',
                  virtual_host='/', message_type='logstash', tags=None,
                  durable=False, passive=False, extra_fields=True,
-                 fqdn=False, facility=None, exchange_routing_key=''):
+                 fqdn=False, facility=None, exchange_routing_key='',
+                 limit_stacktrace=0, limit_string_fields=0, limit_containers=0):
 
 
         # AMQP parameters
@@ -65,7 +69,9 @@ def __init__(self, host='localhost', port=5672, username='guest',
 
         # Extract Logstash paramaters
         self.tags = tags or []
-        self.formatter = formatter.LogstashFormatter(message_type, tags, fqdn)
+        self.formatter = formatter.LogstashFormatter(message_type, tags, fqdn, limit_stacktrace=limit_stacktrace,
+                                                     limit_string_fields=limit_string_fields,
+                                                     limit_containers=limit_containers)
 
         # Standard logging parameters
         self.extra_fields = extra_fields

logstash/handler_http.py

@@ -18,12 +18,18 @@ class HTTPLogstashHandler(NullHandler, object):
     :param verify: verify ssl (default is True)
     :param username: basic_auth user (default is None)
     :param password: basic_auth user (default is None)
+    :param limit_stacktrace: limit characters for stacktraces
+    :param limit_string_fields: limit characters for string fields
+    :param limit_containers: limit length of containers (dict, list, set)
     """
 
-    def __init__(self, host, port=80, ssl=False, message_type='logstash', tags=None, fqdn=False, verify=True, username=None, password=None):
+    def __init__(self, host, port=80, ssl=False, message_type='logstash', tags=None, fqdn=False, verify=True,
+                 username=None, password=None, limit_stacktrace=0, limit_string_fields=0, limit_containers=0):
         super(NullHandler, self).__init__()
 
-        self.formatter = formatter.LogstashFormatter(message_type, tags, fqdn)
+        self.formatter = formatter.LogstashFormatter(message_type, tags, fqdn, limit_stacktrace=limit_stacktrace,
+                                                     limit_string_fields=limit_string_fields,
+                                                     limit_containers=limit_containers)
 
         if username and password:
             self.auth = HTTPBasicAuth(username, password)

logstash/handler_tcp.py

@@ -11,12 +11,18 @@ class TCPLogstashHandler(SocketHandler, object):
     :param message_type: The type of the message (default logstash).
     :param fqdn; Indicates whether to show fully qualified domain name or not (default False).
     :param tags: list of tags for a logger (default is None).
+    :param limit_stacktrace: limit characters for stacktraces
+    :param limit_string_fields: limit characters for string fields
+    :param limit_containers: limit length of containers (dict, list, set)
     """
 
-    def __init__(self, host, port=5959, message_type='logstash', tags=None, fqdn=False):
+    def __init__(self, host, port=5959, message_type='logstash', tags=None, fqdn=False,
+                 limit_stacktrace=0, limit_string_fields=0, limit_containers=0):
         super(TCPLogstashHandler, self).__init__(host, port)
 
-        self.formatter = formatter.LogstashFormatter(message_type, tags, fqdn)
+        self.formatter = formatter.LogstashFormatter(message_type, tags, fqdn, limit_stacktrace=limit_stacktrace,
+                                                     limit_string_fields=limit_string_fields,
+                                                     limit_containers=limit_containers)
 
     def makePickle(self, record):
         return self.formatter.format(record) + b'\n'

logstash/handler_udp.py

@@ -10,6 +10,9 @@ class UDPLogstashHandler(TCPLogstashHandler, DatagramHandler):
     :param message_type: The type of the message (default logstash).
     :param fqdn; Indicates whether to show fully qualified domain name or not (default False).
     :param tags: list of tags for a logger (default is None).
+    :param limit_stacktrace: limit characters for stacktraces
+    :param limit_string_fields: limit characters for string fields
+    :param limit_containers: limit length of containers (dict, list, set)
     """
 
     def makePickle(self, record):