feat: itoa integration (experimental)

add

wip

wip

wip

wip

Update docker-compose.yml

.gitignore

@@ -386,3 +386,4 @@ tests/test_plugin_*.py
 !package/etc/conf.d/local
 replay
 .addon/
+itoa_cache.sqlite

Pipfile

@@ -0,0 +1,11 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+
+[dev-packages]
+
+[requires]
+python_version = "3.7"

package/Dockerfile

@@ -15,7 +15,7 @@
 #You should have received a copy of the CC0 legalcode along with this
 #work.  If not, see <http://creativecommons.org/publicdomain/zero/1.0/>.
 FROM registry.access.redhat.com/ubi8:8.4-206.1626828523
-
+ARG COPR=czanik/syslog-ng334
 COPY --from=hairyhenderson/gomplate:v3.5.0 /gomplate /usr/local/bin/gomplate
 
 RUN curl -fsSL https://goss.rocks/install | GOSS_VER=v0.3.16 sh
@@ -24,8 +24,8 @@ RUN cd /tmp ;\
     dnf install 'dnf-command(copr)' -y ;\
     dnf install http://mirror.centos.org/centos/8/AppStream/x86_64/os/Packages/libnet-1.1.6-15.el8.x86_64.rpm -y; \
     dnf install https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm -y ;\
-    dnf copr enable czanik/syslog-ng334  -y ;\
-    dnf install tzdata curl wget nc socat syslog-ng syslog-ng-python syslog-ng-http syslog-ng-kafka syslog-ng-afsnmp python38-pip gcc python38-devel procps-ng net-tools less net-snmp -y ;\
+    dnf copr enable ${COPR} -y ;\
+    dnf install tzdata curl wget nc socat syslog-ng syslog-ng-python syslog-ng-http syslog-ng-kafka syslog-ng-afsnmp python39-pip procps-ng net-tools less net-snmp -y ;\
     dnf update -y ;\
     dnf clean all
 
@@ -53,7 +53,7 @@ COPY package/etc/goss.yaml /etc/syslog-ng/goss.yaml
 COPY pyproject.toml /
 COPY poetry.lock /
 RUN pip3 install poetry
-RUN poetry export --format requirements.txt | pip3 install --user -r /dev/stdin
+RUN poetry export --format requirements.txt | pip3 install -r /dev/stdin
 
 COPY package/etc/syslog-ng.conf /etc/syslog-ng/syslog-ng.conf
 COPY package/etc/conf.d /etc/syslog-ng/conf.d
@@ -62,6 +62,7 @@ COPY package/etc/context_templates /etc/syslog-ng/context_templates
 COPY package/etc/local_config /etc/syslog-ng/local_config
 COPY package/etc/local_config /etc/syslog-ng/local_config
 COPY package/sbin/entrypoint.sh /
+COPY package/etc/python /etc/syslog-ng/python
 
 COPY package/snmp/snmptrapd.conf /etc/snmp/
 

package/etc/conf.d/conflib/_splunk/itoa.conf

@@ -0,0 +1,5 @@
+parser p_itoa {
+    python(
+        class("itoa.entities")
+    );
+};

package/etc/conf.d/sc4slib/source_syslog/plugin.jinja

@@ -138,6 +138,9 @@ source s_{{ port_id }} {
         };        
 
         parser(vendor_product_by_source);
+        {%- if use_itoa == True %}
+        parser(p_itoa);
+        {%- endif %}
         if {
             parser { 
                 app-parser(topic({{ topic }}-network-source)); 
@@ -278,6 +281,9 @@ source s_{{ port_id }} {
             set('$(lowercase "$HOST")' value(HOST));
         };
         parser(vendor_product_by_source);
+        {%- if use_itoa == True %}
+        parser(p_itoa);
+        {%- endif %}
         if {
             parser { 
                 app-parser(topic({{ topic }}-network-source)); 

package/etc/conf.d/sc4slib/source_syslog/plugin.py

@@ -43,6 +43,17 @@
 else:
     use_reverse_dns = False
 
+if os.getenv(f"SC4S_ITOA_ENABLE", "no").lower() in [
+    "true",
+    "1",
+    "t",
+    "y",
+    "yes",
+]:
+    use_itoa = True
+else:
+    use_itoa = False
+
 if os.getenv(f"SC4S_SOURCE_TLS_ENABLE", "no").lower() in [
     "true",
     "1",
@@ -127,5 +138,6 @@
             f"SC4S_SOURCE_RFC5425_CIPHER_SUITE",
             "HIGH:!aNULL:!eNULL:!kECDH:!aDH:!RC4:!3DES:!CAMELLIA:!MD5:!PSK:!SRP:!KRB5:@STRENGTH",
         ),
+        use_itoa=use_itoa,
     )
     print(outputText)

package/etc/python/itoa.py

@@ -0,0 +1,133 @@
+from pyrate_limiter import RedisBucket, RequestRate, Duration
+from requests import Session
+from requests_cache import CacheMixin, RedisCache
+from requests_ratelimiter import LimiterMixin
+
+import json
+import logging
+import sys, traceback
+import os
+import urllib3
+
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+
+try:
+    import syslogng
+
+    logger = syslogng.Logger()
+except ImportError:
+    log_format = logging.Formatter("[%(asctime)s] [%(levelname)s] - %(message)s")
+    logger = logging.getLogger(__name__)
+    logger.setLevel(logging.DEBUG)
+    handler = logging.StreamHandler(sys.stdout)
+    handler.setLevel(logging.DEBUG)
+    handler.setFormatter(log_format)
+    logger.addHandler(handler)
+
+
+class CachedLimiterSession(CacheMixin, LimiterMixin, Session):
+    """Session class with caching and rate-limiting behavior. Accepts arguments for both
+    LimiterSession and CachedSession.
+    """
+
+
+class entities:
+    def init(self, options):
+        # 'host', "`SC4S_ITOA_HOST`",
+        #     'port', "`SC4S_ITOA_PORT`",
+        #     'verify', "`SC4S_ITOA_TLS_VERIFY`",
+        #     'token', "`SC4S_ITOA_AUTH_TOKEN`"
+        host = os.getenv(f"SC4S_ITOA_HOST", "")
+        if host == "":
+            raise Exception("ITOA Invalid Host")
+
+        port = os.getenv(f"SC4S_ITOA_PORT", "8089")
+
+        token = os.getenv(f"SC4S_ITOA_AUTH_TOKEN", "")
+        if token == "":
+            raise Exception("ITOA Invalid Token")
+
+        logger.debug("Init itoa")
+        self.session = CachedLimiterSession(
+            per_second=int(os.getenv(f"SC4S_ITOA_LIMIT_PER_SEC", "60")),
+            cache_name="itoa_cache",
+            backend="sqlite",
+            expire_after=int(os.getenv(f"SC4S_ITOA_TTL", "600")),
+            logger=logger,
+            match_headers=False,
+            stale_if_error=True,
+        )
+
+        if os.getenv(f"SC4S_ITOA_TLS_VERIFY", "yes") in [
+            "true",
+            "1",
+            "t",
+            "y",
+            "yes",
+        ]:
+            self.verify = True
+        else:
+            self.verify = False
+
+        self.url = (
+            f"https://{host}:{port}/servicesNS/nobody/SA-ITOA/itoa_interface/entity"
+        )
+        self.headers = {
+            "Authorization": f"Bearer {token}",
+            "user-agent": "sc4s/1.0 (itoa)",
+        }
+        return True
+
+    def deinit(self):
+        pass
+
+    def parse(self, log_message):
+        # try to resolve the IP address
+        try:
+            ip = log_message["SOURCEIP"].decode("utf-8")
+            host = log_message["HOST"].decode("utf-8")
+            logger.debug(f"checking for {host},{ip}")
+            response = self.session.request(
+                "GET",
+                self.url,
+                timeout=10,
+                params={
+                    "fields": "title,entity_type_ids,sc4s_vendor_product,host",
+                    "filter": f'{{"$or": [{{"host": "{host}"}},{{"host": "{ip}"}}]}}',
+                },
+                headers=self.headers,
+                verify=self.verify,
+            )
+
+            logger.debug(f"result={response.text}")
+
+            entities = json.loads(response.text)
+            if len(entities) > 0:
+                entity = entities[0]
+                log_message["HOST"] = entity["title"]
+                if "sc4s_vendor_product" in entity:
+                    log_message[".netsource.sc4s_vendor_product"] = entity[
+                        "sc4s_vendor_product"
+                    ][0]
+
+        except:
+            exc_type, exc_value, exc_traceback = sys.exc_info()
+            lines = traceback.format_exception(exc_type, exc_value, exc_traceback)
+            er = "".join("!! " + line for line in lines)  # Log it or whatever here
+            logger.error(f"ITOA exception\n{er}")
+        # return True, other way message is dropped
+        return True
+
+
+if __name__ == "__main__":
+    # execute only if run as a script
+    options = {}
+    ef = entities()
+    ef.init(options)
+    lm = {}
+    lm["HOST"] = "vserver".encode("utf-8")
+    lm["SOURCEIP"] = "10.136.153.117".encode("utf-8")
+    ef.parse(lm)
+    print(lm)
+    ef.deinit()

package/sbin/entrypoint.sh

@@ -1,6 +1,8 @@
 #!/usr/bin/env bash
 function join_by { local d=$1; shift; local f=$1; shift; printf %s "$f" "${@/#/$d}"; }
 
+export PYTHONPATH="/etc/syslog-ng/python:/usr/local/lib/python3.8/site-packages"
+
 # These path variables allow for a single entrypoint script to be utilized for both Container and BYOE runtimes
 export SC4S_LISTEN_DEFAULT_TCP_PORT=${SC4S_LISTEN_DEFAULT_TCP_PORT:=514}
 export SC4S_LISTEN_DEFAULT_UDP_PORT=${SC4S_LISTEN_DEFAULT_UDP_PORT:=514}