Add output of Mermaid syntax overview to Logstash (#377)

This feature was born out of an Elastic Stack workshop I did with
customers. We used [Mermaid](https://mermaid.js.org/) to visualize how
pipelines are interconnected. So the idea was born that it might be very
helpful if the collection could create Mermaid code automatically.

I tried the following to test this code.

Added the following variable to `group_vars/logstash.yml`.

```
logstash_pipelines:
  - name: sorter
    exclusive: true
    input:
      - name: input
        key: input
    output:
      - name: syslog
        key: syslog-input
        condition: '[log][file][path] == "/var/log/syslog"'
        congestion: 5000
      - name: apache
        key: apache
        condition: '[log][file][path] =~ /\/var\/log\/apache2\/.*access.*log$/'
        congestion: 5000
      - name: apache-error
        key: apache-error
        condition: '[log][file][path] =~ /\/var\/log\/apache2\/.*error.*log$/'
        congestion: 5000
      - name: haproxy
        key: syslog-input
        condition: '[log][file][path] == "/var/log/haproxy.log"'
        congestion: 5000
      - name: mysql-error
        key: mysql-error
        condition: '[log][file][path] == "/var/log/mysql/error.log" or [log][file][path] == "/var/log/mysql/mysql.err"'
        congestion: 5000
      - name: mysql-slow
        key: mysql-slow
        condition: '[log][file][path] =~ /\/var\/log\/mysql\/.*-slow.log/'
        congestion: 5000
      - name: haproxy
        key: syslog-input
        condition: '[log][file][path] =~ /^\/var\/log\/mail/'
        congestion: 5000
      - name: forwarder
        key: forwarder
        congestion: 5000
  - name: syslog
    source: https://github.com/NETWAYS/syslog-logstash-pipeline.git
    version: master
    exclusive: true
    input:
      - name: syslog-input
        key: syslog-input
    output:
      - name: postfix
        condition: '[program] =~ /^postfix/ or [log][file][path] =~ /^\/var\/log\/mail/'
        key: postfix
        congestion: 5000
      - name: haproxy
        key: haproxy
        condition: '[program] == "haproxy"'
        congestion: 5000
      - name: forwarder
        key: forwarder
        congestion: 5000
  - name: postfix
    source: https://github.com/NETWAYS/postfix-logstash-pipeline.git
    version: master
    exclusive: false
    input:
      - name: postfix
        key: postfix
    output:
      - name: forwarder
        key: forwarder
        congestion: 5000
  - name: apache
    source: https://github.com/widhalmt/apache-access-logstash-pipeline.git
    version: master
    exclusive: false
    input:
      - name: apache
        key: apache
    output:
      - name: forwarder
        key: forwarder
        congestion: 5000
  - name: apache-error
    #source: https://github.com/widhalmt/apache-error-logstash-pipeline.git
    #version: main
    exclusive: false
    input:
      - name: apache-error
        key: apache-error
    output:
      - name: forwarder
        key: forwarder
        congestion: 5000
  - name: haproxy
    source: https://github.com/widhalmt/haproxy-logstash-pipeline.git
    version: master
    exclusive: false
    input:
      - name: haproxy
        key: haproxy
    output:
      - name: forwarder
        key: forwarder
        congestion: 5000
  - name: mysql-error
    source: https://github.com/widhalmt/mysql-error-logstash-pipeline.git
    version: main
    exclusive: false
    input:
      - name: mysql-error
        key: mysql-error
    output:
      - name: forwarder
        key: forwarder
        congestion: 5000
  - name: mysql-slow
    source: https://github.com/widhalmt/mysql-slowlog-logstash-pipeline.git
    version: main
    exclusive: false
    input:
      - name: mysql-slow
        key: mysql-slow
    output:
      - name: forwarder
        key: forwarder
        congestion: 1000
```
This resulted in the following contents in
`/etc/logstash/pipelines.mermaid`.
```
# Managed via Ansible role
# https://github.com/netways/ansible-role-logstash

# Use the following code with your favorite Mermaid editor
# Or paste into: https://mermaid.live/
# To get a graphical overview of your Logstash pipelines

flowchart TD
p_ansible-input[ansible-input] --> k_input{input}
k_forwarder{forwarder} --> p_ansible-forwarder[ansible-forwarder]
k_input{input} --> p_sorter[sorter]
p_sorter[sorter] --> |if log.file.path == /var/log/syslog|k_syslog-input{syslog-input}
p_sorter[sorter] --> |if log.file.path =~ /\/var\/log\/apache2\/.*access.*log$/|k_apache{apache}
p_sorter[sorter] --> |if log.file.path =~ /\/var\/log\/apache2\/.*error.*log$/|k_apache-error{apache-error}
p_sorter[sorter] --> |if log.file.path == /var/log/haproxy.log|k_syslog-input{syslog-input}
p_sorter[sorter] --> |if log.file.path == /var/log/mysql/error.log or log.file.path == /var/log/mysql/mysql.err|k_mysql-error{mysql-error}
p_sorter[sorter] --> |if log.file.path =~ /\/var\/log\/mysql\/.*-slow.log/|k_mysql-slow{mysql-slow}
p_sorter[sorter] --> |if log.file.path =~ /^\/var\/log\/mail/|k_syslog-input{syslog-input}
p_sorter[sorter] --> k_forwarder{forwarder}
k_syslog-input{syslog-input} --> p_syslog[syslog]
p_syslog[syslog] --> |if program =~ /^postfix/ or log.file.path =~ /^\/var\/log\/mail/|k_postfix{postfix}
p_syslog[syslog] --> |if program == haproxy|k_haproxy{haproxy}
p_syslog[syslog] --> k_forwarder{forwarder}
k_postfix{postfix} --> p_postfix[postfix]
p_postfix[postfix] --> k_forwarder{forwarder}
k_apache{apache} --> p_apache[apache]
p_apache[apache] --> k_forwarder{forwarder}
k_apache-error{apache-error} --> p_apache-error[apache-error]
p_apache-error[apache-error] --> k_forwarder{forwarder}
k_haproxy{haproxy} --> p_haproxy[haproxy]
p_haproxy[haproxy] --> k_forwarder{forwarder}
k_mysql-error{mysql-error} --> p_mysql-error[mysql-error]
p_mysql-error[mysql-error] --> k_forwarder{forwarder}
k_mysql-slow{mysql-slow} --> p_mysql-slow[mysql-slow]
p_mysql-slow[mysql-slow] --> k_forwarder{forwarder}
```
When I post it into an [online Merm

![mermaid-diagram-2025-03-14-115941](https://github.com/user-attachments/assets/fbbd9d25-8bc3-4035-95ca-81ba03034a40)
aid Editor](https://mermaid.live/), then I get the following:

Author: widhalmt

docs/logstash-pipelines.md

@@ -1,5 +1,13 @@
 # Pipelines #
 
+## Keeping an overview ##
+
+It can be quite difficult to stay on top of your pipeline configuration because they tend to become very complex.
+
+This collection will leave some comments about how pipelines are interconnected within the `/etc/logstash/pipelines.yml` configuration file.
+
+If you set `logstash_mermaid` to `true` (which is the default), then you will also get a new file in `/etc/logstash/pipelines.mermaid`. You can paste it into a Mermaid editor in your documentation tool or in an [online Mermaid editor](https://mermaid.live/). The same content will be available on your control node in a temporary file. You can even add arbitrary code to reflect manually managed pipelines by using `logstash_mermaid_extra`.
+
 ## Git managed ##
 
 If you have pipeline code managed in (and available via) Git repositories, you can use this role to check them out and integrate them into `pipelines.yml`.

docs/role-logstash.md

@@ -89,6 +89,11 @@ Aside from `logstash.yml` we can manage Logstashs pipelines.
 * *logstash_legacy_monitoring*: Enables legacy monitoring - ignored when `elasticstack_full_stack` is not set. (default: `true`)
 * *logstash_redis_password*: If set this will use this password when connecting our simple inputs and outputs to Redis. (default: not set)
 
+* *logstash_mermaid*: Print overview over Logstash pipelines in Mermaid syntax. (default: `true`)
+* *logstash_mermaid_logstash*: Place Mermaid syntax into `/etc/logstash/pipelines.mermaid` on Logstash hosts. (default: `true`)
+* *logstash_mermaid_local*: Place Mermaid syntax into temporary file on control node. (default: `false`)
+* *logstash_mermaid_extra*: You can add extra Mermaid syntax to the output by adding it to this variable. YAML-multiline is supported. (default: none)
+
 The following variables configure Log4j for Logstash. All default to `true` as this is the default after the installation.
 
 * *logstash_logging_console*: Log to console - syslog when run via systemd

roles/logstash/defaults/main.yml

@@ -88,6 +88,10 @@ logstash_pipeline_identifier: true
 logstash_pipeline_identifier_field_name: "[netways][pipeline]"
 logstash_pipeline_identifier_defaults: false
 
+logstash_mermaid: true
+logstash_mermaid_local: false
+logstash_mermaid_logstash: true
+
 # Only for internal use
 
 logstash_freshstart:

roles/logstash/tasks/logstash-mermaid.yml

@@ -0,0 +1,36 @@
+---
+
+- name: Print Logstash pipelines in Mermaid syntax on Logstash hosts
+  ansible.builtin.template:
+    src: pipelines.mermaid.j2
+    dest: /etc/logstash/pipelines.mermaid
+    owner: root
+    group: root
+    mode: 0655
+  when:
+    - logstash_mermaid_logstash | bool
+
+- name: Provide Logstash pipelines in Mermaid syntax on control node
+  when:
+    - logstash_mermaid_local | bool
+  block:
+
+    - name: Create temporary directory on control node
+      ansible.builtin.tempfile:
+        prefix: logstash_mermaid
+      register: mermaid_path
+      delegate_to: localhost
+
+    - name: Print Logstash pipelines in Mermaid syntax on control node
+      ansible.builtin.template:
+        src: pipelines.mermaid.j2
+        dest: "{{ mermaid_path.path }}"
+        owner: root
+        group: root
+        mode: 0655
+      delegate_to: localhost
+
+    - name: Send user to Mermaid file
+      ansible.builtin.debug:
+        msg:
+          - "You can find an overview of your pipeline configuration in Mermaid syntax at {{ mermaid_path.path }}"

roles/logstash/tasks/main.yml

@@ -251,6 +251,13 @@
     - configuration
     - logstash_configuration
 
+- name: Print Logstash pipelines in Mermaid syntax
+  ansible.builtin.import_tasks: logstash-mermaid.yml
+  when:
+    - logstash_mermaid | bool
+  tags:
+    - mermaid
+
 - name: Install Logstash plugins
   community.general.logstash_plugin:
     state: present

roles/logstash/templates/pipelines.mermaid.j2

@@ -0,0 +1,31 @@
+# Managed via Ansible role
+# https://github.com/netways/ansible-role-logstash
+
+# Use the following code with your favorite Mermaid editor
+# Or paste into: https://mermaid.live/
+# To get a graphical overview of your Logstash pipelines
+
+flowchart TD
+{% if logstash_beats_input | bool %}
+p_ansible-input[ansible-input] --> k_input{input}
+{% endif %}
+{% if logstash_elasticsearch_output | bool %}
+k_forwarder{forwarder} --> p_ansible-forwarder[ansible-forwarder]
+{% endif %}
+{% if logstash_pipelines is defined %}
+{% for item in logstash_pipelines %}
+{% if item.input is defined %}
+{% for input in item.input %}
+k_{{ input.key }}{{ '{' }}{{ input.key }}{{ '}' }} --> p_{{ item.name }}{{ '[' }}{{item.name}}{{ ']' }}
+{% endfor %}
+{% endif %}
+{% if item.output is defined %}
+{% for output in item.output %}
+p_{{ item.name }}{{ '[' }}{{ item.name }}{{ ']' }} --> {% if output.condition is defined %}{{ '|' }}if {{ output.condition | replace("][", ".") | replace("[","") | replace("]","") | replace('"', '')}}{{ '|' }}{% endif %}k_{{ output.key }}{{ '{' }}{{ output.key }}}
+{% endfor %}
+{% endif %}
+{% endfor %}
+{% endif %}
+{% if logstash_mermaid_extra is defined %}
+{{ logstash_mermaid_extra }}
+{% endif %}