|
| 1 | +#!/usr/bin/env bash |
| 2 | + |
| 3 | +# dependencies: curl, jq |
| 4 | + |
| 5 | +# see also: https://www.haproxy.com/blog/encoding-haproxy-logs-in-machine-readable-json-or-cbor |
| 6 | + |
| 7 | +if [ -z "$1" ] || [ ! -f "$1" ] |
| 8 | +then |
| 9 | + echo 'Provide a log-file to watch' |
| 10 | + exit 1 |
| 11 | +fi |
| 12 | + |
| 13 | +LOG_FILE="$1" |
| 14 | + |
| 15 | +# change to your JSON fields if not default |
| 16 | +FIELD_IP='client_ip' |
| 17 | +FIELD_STATUS='status_code' |
| 18 | + |
| 19 | +# change to the status-code you use when blocking attacks |
| 20 | +BLOCK_STATUS_PROBE='418' |
| 21 | +BLOCK_STATUS_BOT='425' |
| 22 | + |
| 23 | +TOKEN='' # optional supply an API token |
| 24 | +EXCLUDE_REGEX='##########' |
| 25 | +EXCLUDE_IP_REGEX='192.168.|172.16.|172.17.|172.18.|172.19.|172.20.|172.21.|172.22.|172.23.|172.24.|172.25.|172.26.|172.27.|172.28.|172.29.|172.30.|172.31.|10.|127.' |
| 26 | +MAX_PARALLEL=10 |
| 27 | + |
| 28 | +# NOTE: Bash regex does not support PCRE like '\d' '\s' nor non-greedy '*?' |
| 29 | + |
| 30 | +# HAProxy logs |
| 31 | +# example: |
| 32 | +# setenv HTTPLOG_JSON "%{+json}o %(client_ip)ci %(client_port)cp %(request_date)tr %(fe_name_transport)ft %(be_name)b %(server_name)s %(time_request)TR %(time_wait)Tw %(time_connect)Tc %(time_response)Tr/%(time_active)Ta %(status_code)ST %(bytes_read)B %(captured_request_cookie)CC %(captured_response_cookie)CS %(termination_state_cookie)tsc %(actconn)ac %(feconn)fc %(beconn)bc %(srv_conn)sc %(retries)rc %(srv_queue)sq %(backend_queue)bq %(captured_request_headers)hr %(captured_response_headers)hs %(http_request){+Q}r" |
| 33 | +# log-format: "${HTTPLOG_JSON}" |
| 34 | + |
| 35 | +# rsyslog rule: |
| 36 | +# file: /etc/rsyslog.d/haproxy.conf |
| 37 | +# content: |
| 38 | +# $AddUnixListenSocket /var/lib/haproxy/dev/log |
| 39 | +# :programname, startswith, "haproxy" { |
| 40 | +# /var/log/haproxy.log |
| 41 | +# stop |
| 42 | +# } |
| 43 | + |
| 44 | +USER_AGENT='Abuse Reporter' |
| 45 | + |
| 46 | +function report_json() { |
| 47 | + json="$1" |
| 48 | + curl -s -o /dev/null -XPOST 'https://risk.oxl.app/api/report' --data "$json" -H 'Content-Type: application/json' -H "Token: ${TOKEN}" -A "$USER_AGENT" |
| 49 | +} |
| 50 | + |
| 51 | +function log_report() { |
| 52 | + ip="$1" |
| 53 | + category="$2" |
| 54 | + echo "REPORTING: ${ip} because of ${category}" |
| 55 | +} |
| 56 | + |
| 57 | +# NOTE: you may want to add the user-agent as comment ('cmt' field) if you can extract it from your logs |
| 58 | +function report_ip_with_msg() { |
| 59 | + ip="$1" |
| 60 | + category="$2" |
| 61 | + comment="$3" |
| 62 | + log_report "$ip" "$category" |
| 63 | + report_json "{\"ip\": \"${ip}\", \"cat\": \"${category}\", \"cmt\": \"${comment}\"}" |
| 64 | +} |
| 65 | + |
| 66 | +function analyze_log_line() { |
| 67 | + l="$1" |
| 68 | + |
| 69 | + if [[ "$l" != *} ]] |
| 70 | + then |
| 71 | + return |
| 72 | + fi |
| 73 | + |
| 74 | + # anti loop |
| 75 | + if echo "$l" | grep -q "$USER_AGENT" |
| 76 | + then |
| 77 | + return |
| 78 | + fi |
| 79 | + |
| 80 | + # excludes |
| 81 | + if echo "$l" | grep -E -q "$EXCLUDE_REGEX" |
| 82 | + then |
| 83 | + return |
| 84 | + fi |
| 85 | + |
| 86 | + json="{$(echo "$l" | cut -d '{' -f2-)" |
| 87 | + ip="$(echo "$json" | jq -r ".${FIELD_IP}")" |
| 88 | + status="$(echo "$json" | jq -r ".${FIELD_STATUS}")" |
| 89 | + |
| 90 | + if [[ "$ip" == 'null' ]] || [[ "$status" == 'null' ]] |
| 91 | + then |
| 92 | + return |
| 93 | + fi |
| 94 | + |
| 95 | + # excludes by IP |
| 96 | + if echo "$ip" | grep -E -q "$EXCLUDE_IP_REGEX" |
| 97 | + then |
| 98 | + return |
| 99 | + fi |
| 100 | + |
| 101 | + # exclude by IP-list |
| 102 | + if [[ "$(python3 in_ip_list.py --iplist 'iplist.txt' --ip "$ip")" == "1" ]] |
| 103 | + then |
| 104 | + return |
| 105 | + fi |
| 106 | + |
| 107 | + if [[ "$status" == '429' ]] |
| 108 | + then |
| 109 | + report_ip "$ip" 'rate' 'http' |
| 110 | + |
| 111 | + elif [[ "$status" == "$BLOCK_STATUS_PROBE" ]] |
| 112 | + then |
| 113 | + report_ip "$ip" 'probe' 'http' |
| 114 | + |
| 115 | + elif [[ "$status" == "$BLOCK_STATUS_BOT" ]] |
| 116 | + then |
| 117 | + report_ip "$ip" 'bot' 'http' |
| 118 | + |
| 119 | + elif [[ "$status" == '400' ]] && echo "$l" | grep -v -q '/api' |
| 120 | + then |
| 121 | + report_ip "$ip" 'probe' 'http' |
| 122 | + |
| 123 | + fi |
| 124 | +} |
| 125 | + |
| 126 | +function read_log_line() { |
| 127 | + local l='' |
| 128 | + read -r |
| 129 | + while true |
| 130 | + do |
| 131 | + if [[ "$(jobs | wc -l)" -lt "$MAX_PARALLEL" ]] |
| 132 | + then |
| 133 | + analyze_log_line "$REPLY" & |
| 134 | + fi |
| 135 | + read -r |
| 136 | + done |
| 137 | +} |
| 138 | + |
| 139 | +tail "$LOG_FILE" -n0 -f | read_log_line |
0 commit comments