filebeat.yml

filebeat.config:
  modules:
    path: ${path.config}/modules.d/*.yml
    reload.enabled: false

setup.ilm:
  # Failed to connect to backoff(elasticsearch(https://localhost:9200)): Connection marked as failed because
  # the onConnect callback failed: request checking for ILM availability failed: 500 Internal Server Error:
  #  {"error":{"root_cause":[{"type":"security_exception","reason":"Unexpected exception indices:admin/get"}],
  # "type":"security_exception","reason":"Unexpected exception indices:admin/get"},"status":500}
  enabled: false
  rollover_alias: "filebeat"
  pattern: "{now/d}-000001"

filebeat.inputs:
  - type: log
    json.add_error_key: true
    paths:
      - '/logs/slurm-stats/*.json'
    fields:
      event.kind: event
    fields_under_root: true

processors:
  # Want to use the Slurm JobID as the ElasticSearch id to avoid duplicated records
  # Don't use filebeat.inputs:json.document_id as this removes the JobID from the record
  - fingerprint:
      fields: ["json.JobID"]
      target_field: "@metadata._id"
  - timestamp:
      field: json.End
      layouts:
        - '2006-01-02T15:04:05'
      test:
        - '2020-06-17T10:17:48'
  - timestamp:
      target_field: 'event.end'
      field: json.End
      layouts:
        - '2006-01-02T15:04:05'
      test:
        - '2020-06-17T10:17:48'
  - timestamp:
      target_field: 'event.start'
      field: json.Start
      layouts:
        - '2006-01-02T15:04:05'
      test:
        - '2020-06-17T10:17:48'
  - convert:
      fields:
        - {from: "json.NNodes", type: "integer"}
        - {from: "json.NCPUS", type: "integer"}
        - {from: "json.ElapsedRaw", type: "integer"}

output.elasticsearch:
  hosts: ["{{ opensearch_address }}:9200"]
  protocol: "https"
  ssl.verification_mode: none
  username: "admin"
  password: "{{ vault_elasticsearch_admin_password }}"

{% if filebeat_debug | default(false) | bool %}logging.level: debug{% endif %}