Autogenerate ProfileEvents and CurrentMetrics descriptions

scripts/settings/autogenerate-settings.sh

@@ -107,23 +107,26 @@ setup_custom_binary() {
 }
 
 # Function to insert content between AUTOGENERATED tags
+# Usage: insert_content_between_tags <src_file> <dest_file> [start_tag] [end_tag]
 insert_content_between_tags() {
   local src_file="$1"
   local dest_file="$2"
-
+  local start_tag="${3:-<!--AUTOGENERATED_START-->}"
+  local end_tag="${4:-<!--AUTOGENERATED_END-->}"
+
   # Create a temporary file
   local tmp_file=$(mktemp)
   trap 'rm -f "$tmp_file"' EXIT
-  
-  # 1. Copy everything up to and including <!--AUTOGENERATED_START-->
-  awk '/<!--AUTOGENERATED_START-->/ {print; exit}; {print}' "$dest_file" > "$tmp_file"
-  
+
+  # 1. Copy everything up to and including the start tag
+  awk -v tag="$start_tag" 'index($0, tag) {print; exit}; {print}' "$dest_file" > "$tmp_file"
+
   # 2. Add the content from the source file
   cat "$src_file" >> "$tmp_file"
-  
-  # 3. Add everything from <!--AUTOGENERATED_END--> to the end of the file
-  awk 'BEGIN {found=0}; /<!--AUTOGENERATED_END-->/ {found=1}; found {print}' "$dest_file" >> "$tmp_file"
-  
+
+  # 3. Add everything from the end tag to the end of the file
+  awk -v tag="$end_tag" 'BEGIN {found=0}; index($0, tag) {found=1}; found {print}' "$dest_file" >> "$tmp_file"
+
   # Replace the original file with the modified content
   mv "$tmp_file" "$dest_file"
 }
@@ -226,12 +229,12 @@ fi
 if [ "$GENERATE_SETTINGS" = "true" ]; then
     echo "[$SCRIPT_NAME] Auto-generating settings markdown pages..."
     # Process other SQL files first (non-function and non-system-tables related)
-    sql_files_found=$(find "$SCRIPT_DIR" -maxdepth 1 -name '*.sql' ! -name 'generate-functions.sql' ! -name 'system-tables.sql' ! -name 'generate-aggregate-functions.sql' -print -quit)
+    sql_files_found=$(find "$SCRIPT_DIR" -maxdepth 1 -name '*.sql' ! -name 'generate-functions.sql' ! -name 'system-tables.sql' ! -name 'generate-aggregate-functions.sql' ! -name 'generate-events-list.sql' ! -name 'generate-metrics-list.sql' ! -name 'generate-asynchronous-metrics-list.sql' ! -name 'generate-histogram-metrics-list.sql' -print -quit)
 if [ -z "$sql_files_found" ]; then
     echo "[$SCRIPT_NAME] Warning: No settings *.sql files found in $SCRIPT_DIR to process."
 else
     for SQL_FILE in "$SCRIPT_DIR"/*.sql; do
-      if [ -f "$SQL_FILE" ] && [ "$(basename "$SQL_FILE")" != "generate-functions.sql" ] && [ "$(basename "$SQL_FILE")" != "system-tables.sql" ] && [ "$(basename "$SQL_FILE")" != "generate-aggregate-functions.sql" ]; then
+      if [ -f "$SQL_FILE" ] && [ "$(basename "$SQL_FILE")" != "generate-functions.sql" ] && [ "$(basename "$SQL_FILE")" != "system-tables.sql" ] && [ "$(basename "$SQL_FILE")" != "generate-aggregate-functions.sql" ] && [ "$(basename "$SQL_FILE")" != "generate-events-list.sql" ] && [ "$(basename "$SQL_FILE")" != "generate-metrics-list.sql" ] && [ "$(basename "$SQL_FILE")" != "generate-asynchronous-metrics-list.sql" ] && [ "$(basename "$SQL_FILE")" != "generate-histogram-metrics-list.sql" ]; then
         echo "[$SCRIPT_NAME] Processing SQL file: $SQL_FILE"
         # Run ClickHouse from CWD ($tmp_dir), passing absolute path to SQL file
         "$script_path" --queries-file "$SQL_FILE" || {
@@ -578,6 +581,96 @@ done
 
 echo "[$SCRIPT_NAME] System tables documentation completed."
 echo "[$SCRIPT_NAME] Processed ${#SYSTEM_TABLES[@]} system tables."
+
+# Function: generate descriptions list from a SQL file and inject into a doc page
+# Replaces any existing hardcoded "## ... descriptions" section with autogenerated content.
+# Usage: generate_descriptions_list <label> <sql_file> <temp_file> <dest_md> <section_heading>
+generate_descriptions_list() {
+    local label="$1"
+    local sql_file="$2"
+    local temp_file="$3"
+    local dest_full_path="$4"
+    local section_heading="$5"
+
+    echo "[$SCRIPT_NAME] Generating $label descriptions list..."
+    if [ ! -f "$sql_file" ]; then
+        echo "[$SCRIPT_NAME] Warning: SQL file not found at $sql_file. Skipping $label."
+        return
+    fi
+
+    "$script_path" --queries-file "$sql_file" || {
+        echo "[$SCRIPT_NAME] Warning: Failed to generate $label descriptions list."
+        return
+    }
+
+    if [ ! -f "$temp_file" ]; then
+        echo "[$SCRIPT_NAME] Warning: $temp_file not generated. Skipping $label."
+        return
+    fi
+
+    if [ ! -f "$dest_full_path" ]; then
+        echo "[$SCRIPT_NAME] Warning: $dest_full_path not found. Skipping $label."
+        rm -f "$temp_file"
+        return
+    fi
+
+    # Ensure AUTOGENERATED_DESCRIPTIONS tags exist in the destination file
+    if ! grep -q "<!--AUTOGENERATED_DESCRIPTIONS_START-->" "$dest_full_path"; then
+        echo "[$SCRIPT_NAME] Adding AUTOGENERATED_DESCRIPTIONS tags to $(basename "$dest_full_path")"
+        # If there is an existing hardcoded descriptions section, replace it
+        if grep -q "^## .* descriptions" "$dest_full_path"; then
+            echo "[$SCRIPT_NAME] Replacing existing hardcoded descriptions section"
+            awk -v heading="$section_heading" '
+                /^## .* descriptions/ { in_section=1; next }
+                in_section && /^\*\*See Also\*\*/ { in_section=0; print heading; print ""; print "<!--AUTOGENERATED_DESCRIPTIONS_START-->"; print "<!--AUTOGENERATED_DESCRIPTIONS_END-->"; print ""; print $0; next }
+                in_section { next }
+                { print }
+            ' "$dest_full_path" > "${dest_full_path}.tmp" && mv "${dest_full_path}.tmp" "$dest_full_path"
+        elif grep -q '^\*\*See Also\*\*' "$dest_full_path"; then
+            # Insert before **See Also** if it exists
+            sed -i.bak '/^\*\*See Also\*\*/i\
+'"$section_heading"'\
+\
+<!--AUTOGENERATED_DESCRIPTIONS_START-->\
+<!--AUTOGENERATED_DESCRIPTIONS_END-->\
+' "$dest_full_path"
+            rm -f "${dest_full_path}.bak"
+        else
+            # Append at end of file
+            printf '\n%s\n\n<!--AUTOGENERATED_DESCRIPTIONS_START-->\n<!--AUTOGENERATED_DESCRIPTIONS_END-->\n' "$section_heading" >> "$dest_full_path"
+        fi
+    fi
+
+    echo "[$SCRIPT_NAME] Inserting $label descriptions into $dest_full_path"
+    insert_content_between_tags "$temp_file" "$dest_full_path" "<!--AUTOGENERATED_DESCRIPTIONS_START-->" "<!--AUTOGENERATED_DESCRIPTIONS_END-->"
+    rm -f "$temp_file"
+}
+
+# --- Generate descriptions lists for events, metrics, asynchronous_metrics, histogram_metrics ---
+generate_descriptions_list "ProfileEvents" \
+    "$SCRIPT_DIR/generate-events-list.sql" \
+    "temp-events-list.md" \
+    "$target_dir/docs/operations/system-tables/events.md" \
+    "## Event descriptions {#event-descriptions}"
+
+generate_descriptions_list "CurrentMetrics" \
+    "$SCRIPT_DIR/generate-metrics-list.sql" \
+    "temp-metrics-list.md" \
+    "$target_dir/docs/operations/system-tables/metrics.md" \
+    "## Metric descriptions {#metric-descriptions}"
+
+generate_descriptions_list "AsynchronousMetrics" \
+    "$SCRIPT_DIR/generate-asynchronous-metrics-list.sql" \
+    "temp-asynchronous-metrics-list.md" \
+    "$target_dir/docs/operations/system-tables/asynchronous_metrics.md" \
+    "## Metric descriptions {#metric-descriptions}"
+
+generate_descriptions_list "HistogramMetrics" \
+    "$SCRIPT_DIR/generate-histogram-metrics-list.sql" \
+    "temp-histogram-metrics-list.md" \
+    "$target_dir/docs/operations/system-tables/histogram_metrics.md" \
+    "## Metric descriptions {#metric-descriptions}"
+
 else
     echo "[$SCRIPT_NAME] System tables generation is disabled (GENERATE_SYSTEM_TABLES=false). Skipping."
 fi

scripts/settings/generate-asynchronous-metrics-list.sql

@@ -0,0 +1,14 @@
+-- Generate a Markdown list of all asynchronous metrics with descriptions
+SELECT
+    arrayStringConcat(
+        groupArray(formatted_entry),
+        concat(char(10), char(10))
+    )
+FROM (
+    SELECT concat('### ', metric, ' {#', lower(metric), '}', char(10), char(10), description) as formatted_entry
+    FROM system.asynchronous_metrics
+    ORDER BY metric
+)
+LIMIT 1
+INTO OUTFILE 'temp-asynchronous-metrics-list.md' TRUNCATE
+FORMAT LineAsString

scripts/settings/generate-events-list.sql

@@ -0,0 +1,17 @@
+-- Generate a Markdown list of all ProfileEvents with descriptions
+-- Uses system_events_show_zero_values to include events that have not yet fired
+SET system_events_show_zero_values = 1;
+
+SELECT
+    arrayStringConcat(
+        groupArray(formatted_entry),
+        concat(char(10), char(10))
+    )
+FROM (
+    SELECT concat('### ', event, ' {#', lower(event), '}', char(10), char(10), description) as formatted_entry
+    FROM system.events
+    ORDER BY event
+)
+LIMIT 1
+INTO OUTFILE 'temp-events-list.md' TRUNCATE
+FORMAT LineAsString

scripts/settings/generate-histogram-metrics-list.sql

@@ -0,0 +1,14 @@
+-- Generate a Markdown list of all histogram metrics with descriptions
+SELECT
+    arrayStringConcat(
+        groupArray(formatted_entry),
+        concat(char(10), char(10))
+    )
+FROM (
+    SELECT concat('### ', metric, ' {#', lower(metric), '}', char(10), char(10), description) as formatted_entry
+    FROM system.histogram_metrics
+    ORDER BY metric
+)
+LIMIT 1
+INTO OUTFILE 'temp-histogram-metrics-list.md' TRUNCATE
+FORMAT LineAsString

scripts/settings/generate-metrics-list.sql

@@ -0,0 +1,14 @@
+-- Generate a Markdown list of all CurrentMetrics with descriptions
+SELECT
+    arrayStringConcat(
+        groupArray(formatted_entry),
+        concat(char(10), char(10))
+    )
+FROM (
+    SELECT concat('### ', metric, ' {#', lower(metric), '}', char(10), char(10), description) as formatted_entry
+    FROM system.metrics
+    ORDER BY metric
+)
+LIMIT 1
+INTO OUTFILE 'temp-metrics-list.md' TRUNCATE
+FORMAT LineAsString