logit filters: improve handling of alb_access logs

Author: risicle

config/logit/filters.d/30_various_timestamps.conf

@@ -116,3 +116,8 @@ date {
   match => [ "[vxlan_policy_agent][timestamp]", "dd/MMMM/yyyy:HH:mm:ss Z", "dd/MMM/yyyy:HH:mm:ss Z", "ISO8601", "UNIX" ]
   target => "@timestamp"
 }
+# alb_access
+date {
+  match => [ "[event][end]", "dd/MMMM/yyyy:HH:mm:ss Z", "dd/MMM/yyyy:HH:mm:ss Z", "ISO8601", "UNIX" ]
+  target => "@timestamp"
+}

config/logit/filters.d/99_clean_alb_access.conf

@@ -2,13 +2,42 @@
 # Fix up the datapoints for the alb_access log type
 #
 if [@input] == "alb_access" {
+    grok {
+        match => ["[file][path]", "^(?<deploy_env>[^/]+)"]
+        tag_on_failure => ["fail/deploy_env/_grokparsefailure"]
+    }
+
     ## remove extraneous fields
     mutate {
-        remove_field => [ '[@shipper][name]', '[@source][component]', '[@source][type]' ]
+        remove_field => [
+            '[source][geo][city_name]',
+            '[source][geo][continent_code]',
+            '[source][geo][country_code2]',
+            '[source][geo][country_code3]',
+            '[source][geo][country_name]',
+            '[source][geo][ip]',
+            '[source][geo][latitude]',
+            '[source][geo][location][lat]',
+            '[source][geo][location][lon]',
+            '[source][geo][longitude]',
+            '[source][geo][postal_code]',
+            '[source][geo][region_code]',
+            '[source][geo][region_name]',
+            '[source][geo][timezone]',
+            '[device]',
+            '[name]',
+            '[os]',
+            '[os_name]',
+            '[@shipper][name]',
+            '[@source][type]'
+        ]
     }
 
     ## set @type
     mutate {
-        replace => { "@type" => "alb_access" }
+        replace => {
+            "@type" => "alb_access"
+            "[@source][component]" => "alb_access"
+        }
     }
 }

config/logit/output/generated_logit_filters.conf

@@ -1331,18 +1331,52 @@ filter {
     match => [ "[vxlan_policy_agent][timestamp]", "dd/MMMM/yyyy:HH:mm:ss Z", "dd/MMM/yyyy:HH:mm:ss Z", "ISO8601", "UNIX" ]
     target => "@timestamp"
   }
+  # alb_access
+  date {
+    match => [ "[event][end]", "dd/MMMM/yyyy:HH:mm:ss Z", "dd/MMM/yyyy:HH:mm:ss Z", "ISO8601", "UNIX" ]
+    target => "@timestamp"
+  }
   #
   # Fix up the datapoints for the alb_access log type
   #
   if [@input] == "alb_access" {
+      grok {
+          match => ["[file][path]", "^(?<deploy_env>[^/]+)"]
+          tag_on_failure => ["fail/deploy_env/_grokparsefailure"]
+      }
+
       ## remove extraneous fields
       mutate {
-          remove_field => [ '[@shipper][name]', '[@source][component]', '[@source][type]' ]
+          remove_field => [
+              '[source][geo][city_name]',
+              '[source][geo][continent_code]',
+              '[source][geo][country_code2]',
+              '[source][geo][country_code3]',
+              '[source][geo][country_name]',
+              '[source][geo][ip]',
+              '[source][geo][latitude]',
+              '[source][geo][location][lat]',
+              '[source][geo][location][lon]',
+              '[source][geo][longitude]',
+              '[source][geo][postal_code]',
+              '[source][geo][region_code]',
+              '[source][geo][region_name]',
+              '[source][geo][timezone]',
+              '[device]',
+              '[name]',
+              '[os]',
+              '[os_name]',
+              '[@shipper][name]',
+              '[@source][type]'
+          ]
       }
 
       ## set @type
       mutate {
-          replace => { "@type" => "alb_access" }
+          replace => {
+              "@type" => "alb_access"
+              "[@source][component]" => "alb_access"
+          }
       }
   }
 }