[7.9.x][ANSIENG-4730] | Implement logrotate for prometheus & alertmanager

roles/control_center_next_gen/defaults/main.yml

@@ -101,3 +101,20 @@ control_center_next_gen_dependency_alertmanager_service_unit_overrides:
 control_center_next_gen_health_check_delay: 30
 
 control_center_next_gen_secrets_protection_file: "{{ ssl_file_dir_final }}/control-center-security.properties"
+
+# Logrotate Configuration for Prometheus & Alertmanager
+control_center_next_gen_logrotate_enabled: true
+
+# Logrotate Parameters for Prometheus
+control_center_next_gen_dependency_prometheus_logrotate_config:
+  # Size limit for rotation (e.g., "10M", "100M", "1G")
+  size: "10M"
+  # Number of rotated files to keep
+  rotate: 5
+
+# Logrotate Parameters for Alertmanager
+control_center_next_gen_dependency_alertmanager_logrotate_config:
+  # Size limit for rotation (e.g., "10M", "100M", "1G")
+  size: "10M"
+  # Number of rotated files to keep
+  rotate: 5

roles/control_center_next_gen/tasks/health_check.yml

@@ -80,6 +80,130 @@
   delay: 10
   ignore_errors: true
 
+# Logrotate Validation Tasks
+- name: Validate Logrotate Package Installation
+  command: which logrotate
+  register: logrotate_package_check
+  failed_when: logrotate_package_check.rc != 0
+  when: control_center_next_gen_logrotate_enabled|bool
+  tags:
+    - logrotate_validation
+    - validation
+
+- name: Validate Logrotate Configuration Files Exist
+  stat:
+    path: "{{ item }}"
+  register: logrotate_config_files
+  loop:
+    - /etc/logrotate.d/control-center-next-gen-prometheus
+    - /etc/logrotate.d/control-center-next-gen-alertmanager
+  when: control_center_next_gen_logrotate_enabled|bool
+  tags:
+    - logrotate_validation
+    - validation
+
+- name: Validate Logrotate Configuration Files Content
+  command: logrotate -d "{{ item.item }}"
+  register: logrotate_config_validation
+  loop: "{{ logrotate_config_files.results }}"
+  failed_when: logrotate_config_validation.rc != 0
+  when: 
+    - control_center_next_gen_logrotate_enabled|bool
+    - item.stat.exists
+  tags:
+    - logrotate_validation
+    - validation
+
+- name: Validate Logrotate Wrapper Scripts Exist and Executable
+  stat:
+    path: "{{ item }}"
+  register: logrotate_wrapper_scripts
+  loop:
+    - /usr/local/bin/logrotate-control-center-next-gen-prometheus.sh
+    - /usr/local/bin/logrotate-control-center-next-gen-alertmanager.sh
+  when: control_center_next_gen_logrotate_enabled|bool
+  tags:
+    - logrotate_validation
+    - validation
+
+- name: Validate Logrotate Wrapper Scripts Execution
+  command: "{{ item.item }}"
+  register: logrotate_wrapper_execution
+  loop: "{{ logrotate_wrapper_scripts.results }}"
+  failed_when: logrotate_wrapper_execution.rc != 0
+  when: 
+    - control_center_next_gen_logrotate_enabled|bool
+    - item.stat.exists
+    - item.stat.executable
+  tags:
+    - logrotate_validation
+    - validation
+
+- name: Validate Cron Jobs for Logrotate
+  shell: crontab -l | grep -c "{{ item }}"
+  register: logrotate_cron_validation
+  loop:
+    - "Control Center Next Gen Prometheus Log Rotation"
+    - "Control Center Next Gen Alertmanager Log Rotation"
+  when: control_center_next_gen_logrotate_enabled|bool
+  tags:
+    - logrotate_validation
+    - validation
+
+- name: Validate Log Files Ownership
+  stat:
+    path: "{{ item }}"
+  register: log_files_ownership
+  loop:
+    - "{{ control_center_next_gen_dep_prometheus.log_path }}/prometheus_access.log"
+    - "{{ control_center_next_gen_dep_prometheus.log_path }}/prometheus_application.log"
+    - "{{ control_center_next_gen_dep_alertmanager.log_path }}/alertmanager_access.log"
+    - "{{ control_center_next_gen_dep_alertmanager.log_path }}/alertmanager_application.log"
+  when: control_center_next_gen_logrotate_enabled|bool
+  tags:
+    - logrotate_validation
+    - validation
+
+- name: Validate Logrotate Status Files
+  stat:
+    path: "{{ item }}"
+  register: logrotate_status_files
+  loop:
+    - /var/lib/logrotate/status-control-center-next-gen-prometheus
+    - /var/lib/logrotate/status-control-center-next-gen-alertmanager
+  when: control_center_next_gen_logrotate_enabled|bool
+  tags:
+    - logrotate_validation
+    - validation
+
+- name: Validate Systemd Logging Configuration
+  stat:
+    path: "{{ item }}"
+  register: systemd_logging_config
+  loop:
+    - "{{ control_center_next_gen_dep_prometheus.systemd_override | dirname }}/logging.conf"
+    - "{{ control_center_next_gen_dep_alertmanager.systemd_override | dirname }}/logging.conf"
+  when: control_center_next_gen_logrotate_enabled|bool
+  tags:
+    - logrotate_validation
+    - validation
+
+- name: Logrotate Validation Summary
+  debug:
+    msg: |
+      Logrotate Validation Results:
+      - Package Installation: {{ 'PASS' if logrotate_package_check.rc == 0 else 'FAIL' }}
+      - Configuration Files: {{ logrotate_config_files.results | selectattr('stat.exists') | list | length }}/2 files exist
+      - Wrapper Scripts: {{ logrotate_wrapper_scripts.results | selectattr('stat.exists') | list | length }}/2 scripts exist
+      - Cron Jobs: {{ logrotate_cron_validation.results | selectattr('stdout', 'defined') | map(attribute='stdout') | map('int') | sum }}/2 jobs configured
+      - Log Files: {{ log_files_ownership.results | selectattr('stat.exists') | list | length }}/4 files exist
+      - Status Files: {{ logrotate_status_files.results | selectattr('stat.exists') | list | length }}/2 files exist
+      - Systemd Config: {{ systemd_logging_config.results | selectattr('stat.exists') | list | length }}/2 configs exist
+  when: control_center_next_gen_logrotate_enabled|bool
+  tags:
+    - logrotate_validation
+    - validation
+
 - name: Fetch Log Files and Error out
   block:
     - name: Fetch Files for Debugging Failure

roles/control_center_next_gen/tasks/logging.yml

@@ -0,0 +1,178 @@
+---
+- name: Create Control Center Next Gen Dependencies Prometheus & Alertmanager Logs Directory
+  file:
+    path: "{{ item }}"
+    state: directory
+    group: "{{ control_center_next_gen_group }}"
+    owner: "{{ control_center_next_gen_user }}"
+    mode: '770'
+  loop:
+    - "{{ control_center_next_gen_dep_prometheus.log_path }}"
+    - "{{ control_center_next_gen_dep_alertmanager.log_path }}"
+  tags:
+    - filesystem
+    - log
+
+- name: Create Control Center Next Gen Dependencies Prometheus & Alertmanager Log Files
+  file:
+    path: "{{ item }}"
+    state: touch
+    group: "{{ control_center_next_gen_group }}"
+    owner: "{{ control_center_next_gen_user }}"
+    mode: '640'
+  loop:
+    - "{{ control_center_next_gen_dep_prometheus.log_path }}/prometheus.log"
+    - "{{ control_center_next_gen_dep_alertmanager.log_path }}/alertmanager.log"
+  tags:
+    - filesystem
+    - log
+
+- name: Create Control Center Next Gen Dependencies Prometheus & Alertmanager Systemd Override Directories
+  file:
+    path: "{{ item | dirname }}"
+    state: directory
+    mode: '755'
+    owner: root
+    group: root
+  loop:
+    - "{{ control_center_next_gen_dep_prometheus.systemd_override }}"
+    - "{{ control_center_next_gen_dep_alertmanager.systemd_override }}"
+  tags:
+    - systemd
+    - log
+
+- name: Configure Control Center Next Gen Dependencies Prometheus & Alertmanager Systemd Logging
+  template:
+    src: "{{ item.template }}"
+    dest: "{{ item.dest }}"
+    mode: '644'
+    owner: root
+    group: root
+  loop:
+    - template: "prometheus-logging.conf.j2"
+      dest: "{{ control_center_next_gen_dep_prometheus.systemd_override | dirname }}/logging.conf"
+    - template: "alertmanager-logging.conf.j2"
+      dest: "{{ control_center_next_gen_dep_alertmanager.systemd_override | dirname }}/logging.conf"
+  notify: restart control center next gen
+  tags:
+    - systemd
+    - log
+
+- name: Configure Control Center Next Gen Dependencies Prometheus & Alertmanager Logrotate
+  template:
+    src: "{{ item.template }}"
+    dest: "{{ item.dest }}"
+    mode: '644'
+    owner: root
+    group: root
+  loop:
+    - template: "prometheus.logrotate.j2"
+      dest: "/etc/logrotate.d/control-center-next-gen-prometheus"
+    - template: "alertmanager.logrotate.j2"
+      dest: "/etc/logrotate.d/control-center-next-gen-alertmanager"
+  when: control_center_next_gen_logrotate_enabled|bool
+  tags:
+    - logrotate
+
+- name: Create Control Center Next Gen Dependencies Prometheus & Alertmanager Logrotate Wrapper Scripts
+  template:
+    src: "{{ item.template }}"
+    dest: "{{ item.dest }}"
+    mode: '755'
+    owner: root
+    group: root
+  loop:
+    - template: "logrotate-prometheus.sh.j2"
+      dest: "/usr/local/bin/logrotate-control-center-next-gen-prometheus.sh"
+    - template: "logrotate-alertmanager.sh.j2"
+      dest: "/usr/local/bin/logrotate-control-center-next-gen-alertmanager.sh"
+  when: control_center_next_gen_logrotate_enabled|bool
+  tags:
+    - logrotate
+
+- name: Install Logrotate Package for Control Center Next Gen Dependencies Log Rotation
+  package:
+    name: logrotate
+    state: present
+  when: control_center_next_gen_logrotate_enabled|bool
+  tags:
+    - logrotate
+
+- name: Install Cron Package for Control Center Next Gen Dependencies Log Rotation
+  package:
+    name: "{{ 'cronie' if ansible_os_family == 'RedHat' else 'cron' }}"
+    state: present
+  register: install_cron_result
+  until: install_cron_result is success or ansible_check_mode
+  retries: 3
+  delay: 30
+  ignore_errors: "{{ ansible_check_mode }}"
+  when: control_center_next_gen_logrotate_enabled|bool
+  tags:
+    - logrotate
+
+- name: Reset Failed Systemd Units (Clean up orphaned timer references)
+  command: systemctl reset-failed
+  when: control_center_next_gen_logrotate_enabled|bool
+  tags:
+    - systemd
+    - logrotate
+
+- name: Reload Systemd Daemon (Clean up orphaned units)
+  systemd:
+    daemon_reload: true
+  when: control_center_next_gen_logrotate_enabled|bool
+  tags:
+    - systemd
+    - logrotate
+
+- name: Ensure Cron Service is Running (Skip if systemd manages it)
+  systemd:
+    name: "{{ 'crond' if ansible_os_family == 'RedHat' else 'cron' }}"
+    enabled: true
+    state: started
+  ignore_errors: true
+  when: control_center_next_gen_logrotate_enabled|bool
+  tags:
+    - systemd
+    - logrotate
+
+- name: Add Control Center Next Gen Dependencies Prometheus & Alertmanager Logrotate Cron Jobs
+  cron:
+    name: "{{ item.name }}"
+    minute: "*/10"
+    job: "{{ item.job }}"
+    user: root
+  loop:
+    - name: "Control Center Next Gen Prometheus Log Rotation"
+      job: "/usr/local/bin/logrotate-control-center-next-gen-prometheus.sh >> /tmp/control-center-next-gen-prometheus-rotate.log 2>&1"
+    - name: "Control Center Next Gen Alertmanager Log Rotation"
+      job: "/usr/local/bin/logrotate-control-center-next-gen-alertmanager.sh >> /tmp/control-center-next-gen-alertmanager-rotate.log 2>&1"
+  when: control_center_next_gen_logrotate_enabled|bool
+  tags:
+    - logrotate
+
+# Remove logrotate configuration if disabled
+- name: Remove Control Center Next Gen Dependencies Logrotate Configuration and Scripts
+  file:
+    path: "{{ item }}"
+    state: absent
+  loop:
+    - /etc/logrotate.d/control-center-next-gen-prometheus
+    - /etc/logrotate.d/control-center-next-gen-alertmanager
+    - /usr/local/bin/logrotate-control-center-next-gen-prometheus.sh
+    - /usr/local/bin/logrotate-control-center-next-gen-alertmanager.sh
+  when: not control_center_next_gen_logrotate_enabled|bool
+  tags:
+    - disable_logrotate
+
+- name: Remove Control Center Next Gen Dependencies Logrotate Cron Jobs
+  cron:
+    name: "{{ item }}"
+    state: absent
+  loop:
+    - "Control Center Next Gen Prometheus Log Rotation"
+    - "Control Center Next Gen Alertmanager Log Rotation"
+  when: not control_center_next_gen_logrotate_enabled|bool
+  tags:
+    - disable_logrotate

roles/control_center_next_gen/tasks/main.yml

@@ -477,6 +477,12 @@
   tags:
     - log
 
+- name: Configure Control Center Next Gen Dependencies Prometheus & Alertmanager Logging
+  include_tasks: logging.yml
+  tags:
+    - log
+    - logrotate
+
 - name: Create RocksDB Directory
   file:
     path: "{{control_center_next_gen_rocksdb_path}}"

roles/control_center_next_gen/templates/alertmanager-logging.conf.j2

@@ -0,0 +1,4 @@
+[Service]
+# Redirect stdout and stderr to log files
+StandardOutput=append:{{ control_center_next_gen_dep_alertmanager.log_path }}/alertmanager.log
+StandardError=append:{{ control_center_next_gen_dep_alertmanager.log_path }}/alertmanager.log

roles/control_center_next_gen/templates/alertmanager.logrotate.j2

@@ -0,0 +1,10 @@
+{{ control_center_next_gen_dep_alertmanager.log_path }}/alertmanager.log {
+    size {{ control_center_next_gen_dependency_alertmanager_logrotate_config.size }}
+    rotate {{ control_center_next_gen_dependency_alertmanager_logrotate_config.rotate }}
+    compress
+    delaycompress
+    missingok
+    notifempty
+    copytruncate
+    su {{ control_center_next_gen_user }} {{ control_center_next_gen_group }}
+}

roles/control_center_next_gen/templates/logrotate-alertmanager.sh.j2

@@ -0,0 +1,2 @@
+#!/bin/bash
+/usr/sbin/logrotate -s /var/lib/logrotate/status-control-center-next-gen-alertmanager /etc/logrotate.d/control-center-next-gen-alertmanager

roles/control_center_next_gen/templates/logrotate-prometheus.sh.j2

@@ -0,0 +1,2 @@
+#!/bin/bash
+/usr/sbin/logrotate -s /var/lib/logrotate/status-control-center-next-gen-prometheus /etc/logrotate.d/control-center-next-gen-prometheus

roles/control_center_next_gen/templates/prometheus-logging.conf.j2

@@ -0,0 +1,4 @@
+[Service]
+# Redirect stdout and stderr to log files
+StandardOutput=append:{{ control_center_next_gen_dep_prometheus.log_path }}/prometheus.log
+StandardError=append:{{ control_center_next_gen_dep_prometheus.log_path }}/prometheus.log

roles/control_center_next_gen/templates/prometheus.logrotate.j2

@@ -0,0 +1,10 @@
+{{ control_center_next_gen_dep_prometheus.log_path }}/prometheus.log {
+    size {{ control_center_next_gen_dependency_prometheus_logrotate_config.size }}
+    rotate {{ control_center_next_gen_dependency_prometheus_logrotate_config.rotate }}
+    compress
+    delaycompress
+    missingok
+    notifempty
+    copytruncate
+    su {{ control_center_next_gen_user }} {{ control_center_next_gen_group }}
+}

tests/sanity/ignore-2.16.txt

@@ -12,3 +12,5 @@ test_roles/confluent.test.kerberos/files/create_db.sh shebang
 .semaphore/sanity_tests.sh shebang!skip
 githooks/prepare-commit-msg shebang
 githooks/install-githooks.sh shebang
+roles/control_center_next_gen/templates/logrotate-prometheus.sh.j2 shebang
+roles/control_center_next_gen/templates/logrotate-alertmanager.sh.j2 shebang