From 6a5eaaff5524a1edbacbdf9787f0931960b38cc6 Mon Sep 17 00:00:00 2001 From: Taylor Swanson Date: Mon, 6 Oct 2025 09:40:38 -0500 Subject: [PATCH 1/2] [pfsense] Generate processor tags and normalize error handler - Generate tags for processors missing tags - Normalize the pipeline error handler - Ran elastic-package format --- .../elasticsearch/ingest_pipeline/default.yml | 49 ++++++++++++++++++- .../elasticsearch/ingest_pipeline/dhcp.yml | 23 ++++++++- .../ingest_pipeline/firewall.yml | 17 ++++++- .../elasticsearch/ingest_pipeline/haproxy.yml | 36 +++++++++----- .../elasticsearch/ingest_pipeline/ipsec.yml | 13 ++++- .../elasticsearch/ingest_pipeline/openvpn.yml | 15 +++++- .../elasticsearch/ingest_pipeline/php-fpm.yml | 12 ++++- .../elasticsearch/ingest_pipeline/snort.yml | 12 ++++- .../elasticsearch/ingest_pipeline/squid.yml | 11 ++++- .../elasticsearch/ingest_pipeline/unbound.yml | 18 ++++++- 10 files changed, 180 insertions(+), 26 deletions(-) diff --git a/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/default.yml index ce460f3dcf1..e8424d99e14 100644 --- a/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -4,24 +4,30 @@ processors: - set: field: ecs.version value: '8.17.0' + tag: set_f5923549 - set: field: observer.vendor value: netgate + tag: set_ad9d35cc - set: field: observer.type value: firewall + tag: set_5dddf3ba - rename: field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null + tag: rename_56a77271 - set: field: event.kind value: event + tag: set_de80643c - set: field: event.timezone value: "{{{_tmp.tz_offset}}}" if: ctx._tmp?.tz_offset != null && ctx._tmp?.tz_offset != 'local' + tag: set_4ca44cac - grok: description: Parse syslog header field: event.original @@ -40,12 +46,14 @@ processors: PROCESS: '(\(%{DATA:process.name}\)|(?:%{UNIXPATH})%{BASEPATH:process.name})' BASEPATH: '[[[:alnum:]]_%!$@:.,+~-]+' META: '\[[^\]]*\]' + tag: grok_27d9c8c7 - date: if: ctx._tmp.timestamp8601 != null field: _tmp.timestamp8601 target_field: '@timestamp' formats: - ISO8601 + tag: date_6ac9d3ce - date: if: ctx.event?.timezone != null && ctx._tmp?.timestamp != null field: _tmp.timestamp @@ -55,6 +63,7 @@ processors: - MMM d HH:mm:ss - MMM dd HH:mm:ss timezone: '{{{ event.timezone }}}' + tag: date_f21e536e - grok: description: Set Event Provider field: process.name @@ -62,67 +71,85 @@ processors: - '^%{HYPHENATED_WORDS:event.provider}' pattern_definitions: HYPHENATED_WORDS: '\b[A-Za-z0-9_]+(-[A-Za-z_]+)*\b' + tag: grok_cef3d489 - pipeline: name: '{{ IngestPipeline "firewall" }}' if: ctx.event.provider == 'filterlog' + tag: pipeline_e16851a7 - pipeline: name: '{{ IngestPipeline "openvpn" }}' if: ctx.event.provider == 'openvpn' + tag: pipeline_828590b5 - pipeline: name: '{{ IngestPipeline "ipsec" }}' if: ctx.event.provider == 'charon' + tag: pipeline_9d37039c - pipeline: name: '{{ IngestPipeline "dhcp" }}' if: '["dhcpd", "dhclient", "dhcp6c"].contains(ctx.event.provider)' + tag: pipeline_ad56bbca - pipeline: name: '{{ IngestPipeline "unbound" }}' if: ctx.event.provider == 'unbound' + tag: pipeline_dd85553d - pipeline: name: '{{ IngestPipeline "haproxy" }}' if: ctx.event.provider == 'haproxy' + tag: pipeline_720ed255 - pipeline: name: '{{ IngestPipeline "php-fpm" }}' if: ctx.event.provider == 'php-fpm' + tag: pipeline_456beba5 - pipeline: name: '{{ IngestPipeline "squid" }}' if: ctx.event.provider == 'squid' + tag: pipeline_a0d89375 - pipeline: name: '{{ IngestPipeline "snort" }}' if: ctx.event.provider == 'snort' + tag: pipeline_c2f1ed55 - drop: if: '!["filterlog", "openvpn", "charon", "dhcpd", "dhclient", "dhcp6c", "unbound", "haproxy", "php-fpm", "squid", "snort"].contains(ctx.event?.provider)' + tag: drop_9d7c46f8 - append: field: event.category value: network if: "ctx.network != null" + tag: append_4780a983 - convert: field: source.address target_field: source.ip type: ip ignore_failure: true ignore_missing: true + tag: convert_f5632a20 - convert: field: destination.address target_field: destination.ip type: ip ignore_failure: true ignore_missing: true + tag: convert_f1388f0c - set: field: network.type value: ipv6 if: 'ctx.source?.ip != null && ctx.source.ip.contains(":")' + tag: set_1f1d940a - set: field: network.type value: ipv4 if: 'ctx.source?.ip != null && ctx.source.ip.contains(".")' + tag: set_69deca38 - geoip: field: source.ip target_field: source.geo ignore_missing: true + tag: geoip_da2e41b2 - geoip: field: destination.ip target_field: destination.geo ignore_missing: true + tag: geoip_ab5e2968 - geoip: ignore_missing: true database_file: GeoLite2-ASN.mmdb @@ -131,6 +158,7 @@ processors: properties: - asn - organization_name + tag: geoip_28d69883 - geoip: database_file: GeoLite2-ASN.mmdb field: destination.ip @@ -139,66 +167,81 @@ processors: - asn - organization_name ignore_missing: true + tag: geoip_8a007787 - rename: field: source.as.asn target_field: source.as.number ignore_missing: true + tag: rename_a917047d - rename: field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true + tag: rename_f1362d0b - rename: field: destination.as.asn target_field: destination.as.number ignore_missing: true + tag: rename_3b459fcd - rename: field: destination.as.organization_name target_field: destination.as.organization.name ignore_missing: true + tag: rename_814bd459 - community_id: target_field: network.community_id ignore_failure: true + tag: community_id_d2308e7a - grok: field: observer.ingress.interface.name patterns: - "%{DATA}.%{NONNEGINT:observer.ingress.vlan.id}" ignore_missing: true ignore_failure: true + tag: grok_968018d3 - set: field: network.vlan.id copy_from: observer.ingress.vlan.id ignore_empty_value: true + tag: set_efd4d96a - append: field: related.ip value: "{{{destination.ip}}}" allow_duplicates: false if: ctx.destination?.ip != null + tag: append_c1a6356b - append: field: related.ip value: "{{{source.ip}}}" allow_duplicates: false if: ctx.source?.ip != null + tag: append_8121c591 - append: field: related.ip value: "{{{source.nat.ip}}}" allow_duplicates: false if: ctx.source?.nat?.ip != null + tag: append_53b62ed8 - append: field: related.hosts value: "{{{destination.domain}}}" if: "ctx.destination?.domain != null" + tag: append_6f162628 - append: field: related.user value: "{{{user.name}}}" if: "ctx.user?.name != null" + tag: append_c036eec2 - set: field: network.direction value: "{{{network.direction}}}bound" if: ctx.network?.direction != null && ctx.network?.direction =~ /^(in|out)$/ + tag: set_cb1e3125 - remove: field: - _tmp ignore_failure: true + tag: remove_a82e20f2 - script: lang: painless description: This script processor iterates over the whole document to remove fields with null values. @@ -223,6 +266,7 @@ processors: } } handleMap(ctx); + tag: script_a7f2c062 on_failure: - remove: field: @@ -233,4 +277,7 @@ on_failure: value: pipeline_error - append: field: error.message - value: '{{{ _ingest.on_failure_message }}}' + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/dhcp.yml b/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/dhcp.yml index 517a380e1a1..0f1e4c56ba5 100644 --- a/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/dhcp.yml +++ b/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/dhcp.yml @@ -20,77 +20,96 @@ processors: FOR: 'for %{IP:client.address} \(%{IP:server.address}\)? from %{MAC_ADDRESS:client.mac} \(%{HOSTNAME:pfsense.dhcp.hostname}\)' MIDDLE: '(?:%{FROM}|%{ON}|%{FOR})' DUID: '(?i)[0-9a-f]{2}(:[0-9a-f]{2})+' + tag: grok_efb70afb - append: field: event.type - value: + value: - connection - protocol - info allow_duplicates: false + tag: append_c741503d - set: field: network.protocol value: dhcp + tag: set_ea539779 - set: field: network.protocol value: dhcpv6 if: ctx.event.provider == 'dhcp6c' || (ctx.server?.address != null && ctx.server.address.contains(':')) || (ctx.client?.address != null && ctx.client.address.contains(':')) + tag: set_98d42096 - set: field: network.transport value: udp + tag: set_a6cfd568 - convert: field: client.address target_field: client.ip type: ip ignore_failure: true ignore_missing: true + tag: convert_c718a334 - convert: field: server.address target_field: server.ip type: ip ignore_failure: true ignore_missing: true + tag: convert_940441bc - uppercase: field: client.mac ignore_missing: true + tag: uppercase_230df574 - gsub: field: client.mac pattern: '[:]' replacement: '-' ignore_missing: true + tag: gsub_3eb17338 - uppercase: field: server.mac ignore_missing: true + tag: uppercase_12d7a2a0 - gsub: field: server.mac pattern: '[:]' replacement: '-' ignore_missing: true + tag: gsub_5d413db4 - lowercase: field: _tmp.action ignore_missing: true + tag: lowercase_5d4e19c7 - gsub: field: _tmp.action target_field: event.action pattern: ' ' replacement: '-' ignore_missing: true + tag: gsub_2c25be60 - set: field: source copy_from: client ignore_empty_value: true + tag: set_a194c11b - set: field: destination copy_from: server ignore_empty_value: true + tag: set_bb802684 - append: field: related.hosts value: "{{{pfsense.dhcp.hostname}}}" allow_duplicates: false if: "ctx.pfsense?.log?.dhcp?.hostname != null" + tag: append_8bba1f0b on_failure: - set: field: event.kind value: pipeline_error - append: field: error.message - value: '{{{ _ingest.on_failure_message }}}' + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/firewall.yml b/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/firewall.yml index e4748464594..d4cbd9019da 100644 --- a/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/firewall.yml +++ b/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/firewall.yml @@ -31,60 +31,75 @@ processors: PF_ICMP_TSTAMP_REPLY: "%{INT:pfsense.icmp.id},%{INT:pfsense.icmp.seq},%{INT:pfsense.icmp.otime},%{INT:pfsense.icmp.rtime},%{INT:pfsense.icmp.ttime}" PF_SPEC: "[+]" PF_TRANSPORT: "[0-9a-zA-Z-]+" + tag: grok_518a3bd8 - set: field: event.kind value: event + tag: set_de80643c - append: field: event.type value: connection allow_duplicates: false if: ctx.source?.address != null && ctx.destination?.address != null + tag: append_6c6f283d - append: field: event.type value: denied allow_duplicates: false if: ctx.event.action == 'block' + tag: append_fcaacaaf - append: field: event.type value: allowed allow_duplicates: false if: ctx.event.action == 'pass' + tag: append_77f1e2f8 - lowercase: field: network.transport ignore_missing: true + tag: lowercase_bc8c1c12 - remove: field: ack_number ignore_missing: true if: ctx.ack_number == null || ctx.ack_number == '' + tag: remove_9a55d0f3 - network_direction: internal_networks_field: _tmp.internal_networks + tag: network_direction_6d15af04 - split: field: pfsense.tcp.options separator: ';' ignore_missing: true ignore_failure: true + tag: split_a6459ab2 - date: field: pfsense.icmp.otime ignore_failure: true formats: - UNIX - UNIX_MS + tag: date_a8122a08 - date: field: pfsense.icmp.rtime ignore_failure: true formats: - UNIX - UNIX_MS + tag: date_6f2f5bb7 - date: field: pfsense.icmp.ttime ignore_failure: true formats: - UNIX - UNIX_MS + tag: date_c5011051 on_failure: - set: field: event.kind value: pipeline_error - append: field: error.message - value: '{{{ _ingest.on_failure_message }}}' + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/haproxy.yml b/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/haproxy.yml index bce0e8953fd..7ecb7a6e0a2 100644 --- a/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/haproxy.yml +++ b/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/haproxy.yml @@ -6,17 +6,11 @@ processors: patterns: - 'Connect from (%{IPORHOST:source.address}|-):%{POSINT:source.port:long} %{WORD} %{IPORHOST:destination.address}:%{POSINT:destination.port:long} \(%{NOTSPACE:haproxy.frontend_name}/%{WORD:haproxy.mode}\)' # HTTP(S) - - '(%{IPORHOST:source.address}|-):%{POSINT:source.port:long} \[%{NOTSPACE:haproxy.request_date}\] %{NOTSPACE:haproxy.frontend_name} %{NOTSPACE:haproxy.backend_name}/%{NOTSPACE:haproxy.server_name} - %{NUMBER:haproxy.http.request.time_wait_ms:long}/%{NUMBER:haproxy.total_waiting_time_ms:long}/%{NUMBER:haproxy.connection_wait_time_ms:long}/%{NUMBER:haproxy.http.request.time_wait_without_data_ms:long}/%{NUMBER:_temp.duration:long} - %{NUMBER:http.response.status_code:long} %{NUMBER:haproxy.bytes_read:long} %{NOTSPACE:haproxy.http.request.captured_cookie} %{NOTSPACE:haproxy.http.response.captured_cookie} %{NOTSPACE:haproxy.termination_state} - %{NUMBER:haproxy.connections.active:long}/%{NUMBER:haproxy.connections.frontend:long}/%{NUMBER:haproxy.connections.backend:long}/%{NUMBER:haproxy.connections.server:long}/%{NUMBER:haproxy.connections.retries:long} %{NUMBER:haproxy.server_queue:long}/%{NUMBER:haproxy.backend_queue:long} - (\{%{DATA:haproxy.http.request.captured_headers}\} \{%{DATA:haproxy.http.response.captured_headers}\} |\{%{DATA}\} )?"%{GREEDYDATA:haproxy.http.request.raw_request_line}"' + - '(%{IPORHOST:source.address}|-):%{POSINT:source.port:long} \[%{NOTSPACE:haproxy.request_date}\] %{NOTSPACE:haproxy.frontend_name} %{NOTSPACE:haproxy.backend_name}/%{NOTSPACE:haproxy.server_name} %{NUMBER:haproxy.http.request.time_wait_ms:long}/%{NUMBER:haproxy.total_waiting_time_ms:long}/%{NUMBER:haproxy.connection_wait_time_ms:long}/%{NUMBER:haproxy.http.request.time_wait_without_data_ms:long}/%{NUMBER:_temp.duration:long} %{NUMBER:http.response.status_code:long} %{NUMBER:haproxy.bytes_read:long} %{NOTSPACE:haproxy.http.request.captured_cookie} %{NOTSPACE:haproxy.http.response.captured_cookie} %{NOTSPACE:haproxy.termination_state} %{NUMBER:haproxy.connections.active:long}/%{NUMBER:haproxy.connections.frontend:long}/%{NUMBER:haproxy.connections.backend:long}/%{NUMBER:haproxy.connections.server:long}/%{NUMBER:haproxy.connections.retries:long} %{NUMBER:haproxy.server_queue:long}/%{NUMBER:haproxy.backend_queue:long} (\{%{DATA:haproxy.http.request.captured_headers}\} \{%{DATA:haproxy.http.response.captured_headers}\} |\{%{DATA}\} )?"%{GREEDYDATA:haproxy.http.request.raw_request_line}"' + # TCP - - '(%{IP:source.address}|-):%{POSINT:source.port:long} \[%{NOTSPACE:haproxy.request_date}\] - %{NOTSPACE:haproxy.frontend_name} %{NOTSPACE:haproxy.backend_name}/%{NOTSPACE:haproxy.server_name} - %{NUMBER:haproxy.total_waiting_time_ms:long}/%{NUMBER:haproxy.connection_wait_time_ms:long}/%{NUMBER:_temp.duration:long} - %{NUMBER:haproxy.bytes_read:long} %{NOTSPACE:haproxy.termination_state} %{NUMBER:haproxy.connections.active:long}/%{NUMBER:haproxy.connections.frontend:long}/%{NUMBER:haproxy.connections.backend:long}/%{NUMBER:haproxy.connections.server:long}/%{NUMBER:haproxy.connections.retries:long} - %{NUMBER:haproxy.server_queue:long}/%{NUMBER:haproxy.backend_queue:long}' + - '(%{IP:source.address}|-):%{POSINT:source.port:long} \[%{NOTSPACE:haproxy.request_date}\] %{NOTSPACE:haproxy.frontend_name} %{NOTSPACE:haproxy.backend_name}/%{NOTSPACE:haproxy.server_name} %{NUMBER:haproxy.total_waiting_time_ms:long}/%{NUMBER:haproxy.connection_wait_time_ms:long}/%{NUMBER:_temp.duration:long} %{NUMBER:haproxy.bytes_read:long} %{NOTSPACE:haproxy.termination_state} %{NUMBER:haproxy.connections.active:long}/%{NUMBER:haproxy.connections.frontend:long}/%{NUMBER:haproxy.connections.backend:long}/%{NUMBER:haproxy.connections.server:long}/%{NUMBER:haproxy.connections.retries:long} %{NUMBER:haproxy.server_queue:long}/%{NUMBER:haproxy.backend_queue:long}' + # Error - '(%{IP:source.address}|-):%{POSINT:source.port:long} \[%{NOTSPACE:haproxy.request_date}\] %{NOTSPACE:haproxy.frontend_name}/%{BIND_NAME:haproxy.bind_name}:? %{GREEDYDATA:haproxy.error_message}' ignore_missing: false @@ -26,6 +20,7 @@ processors: on_failure: - drop: description: Drop if not a connection log + tag: grok_27190f7e - date: if: ctx.haproxy?.request_date != null && ctx.event?.timezone == null field: haproxy.request_date @@ -33,6 +28,7 @@ processors: formats: - dd/MMM/yyyy:HH:mm:ss.SSS - MMM dd HH:mm:ss + tag: date_e0201264 - date: if: ctx.haproxy?.request_date != null && ctx.event?.timezone != null field: haproxy.request_date @@ -41,66 +37,82 @@ processors: - dd/MMM/yyyy:HH:mm:ss.SSS - MMM dd HH:mm:ss timezone: '{{{ event.timezone }}}' + tag: date_8084f896 - remove: field: haproxy.request_date ignore_missing: true + tag: remove_ba26ea6b - grok: field: haproxy.http.request.raw_request_line patterns: - '%{WORD:http.request.method}%{SPACE}%{URIPATHPARAM:url.original}%{SPACE}HTTP/%{NUMBER:http.version}' ignore_missing: true if: 'ctx.haproxy?.http?.request?.raw_request_line != null && !ctx.haproxy?.http?.request?.raw_request_line.isEmpty() && ctx.haproxy?.http?.request?.raw_request_line != ""' + tag: grok_edca39ab - uri_parts: field: url.original ignore_failure: true if: ctx.url?.original != null + tag: uri_parts_c5d45689 - split: field: haproxy.http.request.captured_headers separator: \| ignore_failure: true ignore_missing: true + tag: split_066d989f - split: field: haproxy.http.response.captured_headers separator: \| ignore_failure: true ignore_missing: true + tag: split_f2f5c8fd - script: lang: painless source: ctx.event.duration = Math.round(ctx._temp.duration * params.scale) params: scale: 1000000 if: ctx._temp?.duration != null + tag: script_bd0ddda1 - convert: field: haproxy.bytes_read target_field: http.response.bytes type: long ignore_missing: true if: ctx.containsKey('http') + tag: convert_414f1a1d - append: field: event.category value: web if: "ctx.haproxy?.mode == 'HTTP' || ctx.haproxy?.http != null" + tag: append_f910d3f5 - append: field: event.type value: access if: "ctx.source?.address != null && ctx.destination?.address != null" + tag: append_fa71cb3a - set: field: event.outcome value: success if: "ctx.http?.response?.status_code != null && ctx.http.response.status_code < 400" + tag: set_b50e8224 - set: field: event.outcome value: failure if: "ctx.http?.response?.status_code != null && ctx.http.response.status_code >= 400" + tag: set_75086854 - remove: - field: + field: - _temp - haproxy.request_date ignore_missing: true + tag: remove_b050d090 on_failure: - set: field: event.kind value: pipeline_error - append: field: error.message - value: '{{{ _ingest.on_failure_message }}}' + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/ipsec.yml b/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/ipsec.yml index b5099b6ba4f..b8c2c1d9b0f 100644 --- a/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/ipsec.yml +++ b/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/ipsec.yml @@ -10,31 +10,40 @@ processors: PREFIX: '\d+\[%{WORD}\]' SOURCE: '%{IP:source.address}\[%{NONNEGINT:source.port:long}\]' DEST: '%{IP:destination.address}\[%{NONNEGINT:destination.port:long}\]' + tag: grok_d7d9a2d7 - append: field: event.type value: connection allow_duplicates: false if: ctx.source?.address != null + tag: append_ac4d9a80 - append: field: event.type value: end allow_duplicates: false if: ctx.message.toLowerCase().contains('disconnected') + tag: append_6887ab9d - set: field: source.ip value: "{{{source.address}}}" ignore_empty_value: true + tag: set_0313ebc9 - set: field: destination.ip value: "{{{destination.address}}}" - ignore_empty_value: true + ignore_empty_value: true + tag: set_ca415005 - set: field: network.protocol value: ipsec + tag: set_2d8e203a on_failure: - set: field: event.kind value: pipeline_error - append: field: error.message - value: '{{{ _ingest.on_failure_message }}}' + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/openvpn.yml b/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/openvpn.yml index ee893ee37d9..767883463bd 100644 --- a/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/openvpn.yml +++ b/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/openvpn.yml @@ -13,41 +13,52 @@ processors: pattern_definitions: SOURCE: '%{IP:source.address}:%{NONNEGINT:source.port:long}' USERNAME: '[a-zA-Z0-9._-]+' + tag: grok_52bb7c6e - append: field: event.category value: authentication allow_duplicates: false if: ctx.message.contains('auth') + tag: append_e7e9b076 - append: field: event.type value: connection allow_duplicates: false if: ctx.source?.address != null + tag: append_ac4d9a80 - append: field: event.type value: info allow_duplicates: false + tag: append_3f43a39d - append: field: event.outcome value: failure allow_duplicates: false - if: ctx.message.toLowerCase().contains('error') || ctx.message.toLowerCase().contains('not auth') + if: ctx.message.toLowerCase().contains('error') || ctx.message.toLowerCase().contains('not auth') + tag: append_42491fe9 - append: field: event.type value: start allow_duplicates: false if: ctx.message.toLowerCase().contains('initiat') + tag: append_6cc50f6b - set: field: source.ip value: "{{{source.address}}}" ignore_empty_value: true + tag: set_0313ebc9 - set: field: network.protocol value: openvpn + tag: set_1caba15c on_failure: - set: field: event.kind value: pipeline_error - append: field: error.message - value: '{{{ _ingest.on_failure_message }}}' + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/php-fpm.yml b/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/php-fpm.yml index 9203a1ae243..1c1272ffe20 100644 --- a/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/php-fpm.yml +++ b/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/php-fpm.yml @@ -11,36 +11,46 @@ processors: PF_APP_LOGIN: "(%{DATA:_tmp.action}) for user '%{USER:user.name}' from: %{IP:source.address} \\(%{DATA}\\)" PF_APP_LOGOUT: "User (%{DATA:_tmp.action}) for user '%{USER:user.name}' from: %{IP:source.address}" PF_APP_ERROR: "webConfigurator %{DATA:_tmp.action} for user '%{DATA:user.name}' from: %{IP:source.address}" + tag: grok_dc596a09 - append: field: event.category value: authentication allow_duplicates: false + tag: append_80db0fcd - set: field: event.outcome value: success if: 'ctx._tmp?.action.toLowerCase().contains("success")' + tag: set_b39ddcc4 - set: field: event.outcome value: failure if: 'ctx._tmp?.action.toLowerCase().contains("authentication error")' + tag: set_83ced4a6 - convert: field: source.address target_field: source.ip type: ip ignore_missing: true ignore_failure: true + tag: convert_f5632a20 - rename: field: observer.ip target_field: host.ip ignore_missing: true + tag: rename_92a4a9e0 - rename: field: observer.name target_field: host.name ignore_missing: true + tag: rename_34df84ec on_failure: - set: field: event.kind value: pipeline_error - append: field: error.message - value: '{{{ _ingest.on_failure_message }}}' + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/snort.yml b/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/snort.yml index 17281c0f978..ed256613587 100644 --- a/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/snort.yml +++ b/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/snort.yml @@ -5,10 +5,18 @@ processors: field: message patterns: - '\[%{NUMBER:snort.generator_id}:%{NUMBER:snort.signature_id}:%{NUMBER:snort.signature_revision}\] \(%{DATA:snort.preprocessor}\) %{GREEDYDATA:snort.alert_message} \[Classification: %{DATA:snort.classification}\] \[Priority: %{NONNEGINT:snort.priority:long}\] \{%{WORD:network.protocol}\} %{IP:source.address}:%{NUMBER:source.port:long} -> %{IP:destination.address}:%{NUMBER:destination.port:long}' + tag: grok_26af318e - lowercase: field: network.protocol ignore_missing: true + tag: lowercase_49872259 on_failure: - set: - field: error.message - value: "{{{ _ingest.on_failure_message }}}" + field: error.message + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}failed with message '{{{ _ingest.on_failure_message }}}' + - set: + field: event.kind + value: pipeline_error diff --git a/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/squid.yml b/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/squid.yml index e6106798e61..05f69a6e897 100644 --- a/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/squid.yml +++ b/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/squid.yml @@ -6,29 +6,38 @@ processors: patterns: - '%{IPORHOST:source.address} %{NOTSPACE:squid.request_status}/%{NUMBER:http.response.status_code:long} %{NUMBER:http.response.bytes:long} %{NOTSPACE:http.request.method} (?:%{URI:url.original}|(%{IPORHOST:url.domain}(?::%{DATA:url.port})?))?%{SPACE}%{NOTSPACE:http.request.referrer}%{SPACE}%{NOTSPACE:squid.hierarchy_status}/(?:%{IPORHOST:destination.address}|-)%{SPACE}%{NOTSPACE:http.response.mime_type}' ignore_missing: false + tag: grok_55f0c6b7 - uri_parts: field: url.original ignore_failure: true if: ctx.url?.original != null + tag: uri_parts_c5d45689 - convert: field: url.port type: long ignore_missing: true + tag: convert_537f953d - append: field: event.category value: web + tag: append_4595ee28 - set: field: event.outcome value: success if: "ctx.http?.response?.status_code != null && ctx.http.response.status_code < 400" + tag: set_b50e8224 - set: field: event.outcome value: failure if: "ctx.http?.response?.status_code != null && ctx.http.response.status_code >= 400" + tag: set_75086854 on_failure: - set: field: event.kind value: pipeline_error - append: field: error.message - value: '{{{ _ingest.on_failure_message }}}' + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/unbound.yml b/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/unbound.yml index 67c662a4f91..85244054b6d 100644 --- a/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/unbound.yml +++ b/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/unbound.yml @@ -3,58 +3,72 @@ description: Pipeline for PFsense Unbound DNS logs processors: - grok: field: message - patterns: + patterns: - '%{LOGLEVEL:log.level}: %{IP:source.address} %{HOSTNAME:_tmp.question.name}(\.) %{WORD:_tmp.question.type} %{WORD:_tmp.question.class}' on_failure: - drop: description: Drop if not a query log + tag: grok_88ceaee5 - append: field: event.type value: connection allow_duplicates: false if: ctx.source?.address != null + tag: append_ac4d9a80 - append: field: event.type value: end allow_duplicates: false if: ctx.message.toLowerCase().contains('disconnected') + tag: append_6887ab9d - set: field: network.protocol value: dns + tag: set_96a840d1 - set: field: dns.type value: question if: ctx._tmp?.question?.name != null + tag: set_4812efdb - registered_domain: field: _tmp.question.name target_field: dns.question ignore_missing: true + tag: registered_domain_a89b448b - rename: field: dns.question.domain target_field: dns.question.name ignore_missing: true + tag: rename_88a92ba9 - rename: field: _tmp.question.type target_field: dns.question.type ignore_missing: true + tag: rename_e2286aa7 - rename: field: _tmp.question.class target_field: dns.question.class ignore_missing: true + tag: rename_e864f0db - convert: field: source.address target_field: source.ip type: ip ignore_failure: true ignore_missing: true + tag: convert_f5632a20 - set: field: client copy_from: source ignore_empty_value: true + tag: set_721030b3 on_failure: - set: field: event.kind value: pipeline_error - append: field: error.message - value: '{{{ _ingest.on_failure_message }}}' + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}failed with message '{{{ _ingest.on_failure_message }}}' From bc7b8e60d44a6b4e7c665124147efcbbc63f8194 Mon Sep 17 00:00:00 2001 From: Taylor Swanson Date: Mon, 6 Oct 2025 09:41:33 -0500 Subject: [PATCH 2/2] changelog --- packages/pfsense/changelog.yml | 5 +++++ packages/pfsense/manifest.yml | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/pfsense/changelog.yml b/packages/pfsense/changelog.yml index ec97ec2c368..a8462ff8de6 100644 --- a/packages/pfsense/changelog.yml +++ b/packages/pfsense/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.23.2" + changes: + - description: Generate processor tags and normalize error handler. + type: enhancement + link: https://github.com/elastic/integrations/pull/15566 - version: "1.23.1" changes: - description: Changed owners. diff --git a/packages/pfsense/manifest.yml b/packages/pfsense/manifest.yml index 72f666ed212..595f1efae25 100644 --- a/packages/pfsense/manifest.yml +++ b/packages/pfsense/manifest.yml @@ -1,6 +1,6 @@ name: pfsense title: pfSense -version: "1.23.1" +version: "1.23.2" description: Collect logs from pfSense and OPNsense with Elastic Agent. type: integration icons: