Skip to content

Commit 0d682f6

Browse files
committed
add report-script for haproxy json-log-format
1 parent 7bf725b commit 0d682f6

File tree

2 files changed

+140
-1
lines changed

2 files changed

+140
-1
lines changed

reporting/README.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,7 @@ Content:
9494

9595
### HAProxy
9696

97-
See: [Report Script - HAProxy](https://github.com/O-X-L/risk-db/blob/latest/reporting/log_watcher_haproxy.sh)
97+
See: [Report Script - HAProxy](https://github.com/O-X-L/risk-db/blob/latest/reporting/log_watcher_haproxy.sh) & [Report Script - HAProxy JSON Log-Format](https://github.com/O-X-L/risk-db/blob/latest/reporting/log_watcher_haproxy.sh)
9898

9999
#### Logs
100100

reporting/log_watcher_haproxy_json.sh

+139
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
#!/usr/bin/env bash
2+
3+
# dependencies: curl, jq
4+
5+
# see also: https://www.haproxy.com/blog/encoding-haproxy-logs-in-machine-readable-json-or-cbor
6+
7+
if [ -z "$1" ] || [ ! -f "$1" ]
8+
then
9+
echo 'Provide a log-file to watch'
10+
exit 1
11+
fi
12+
13+
LOG_FILE="$1"
14+
15+
# change to your JSON fields if not default
16+
FIELD_IP='client_ip'
17+
FIELD_STATUS='status_code'
18+
19+
# change to the status-code you use when blocking attacks
20+
BLOCK_STATUS_PROBE='418'
21+
BLOCK_STATUS_BOT='425'
22+
23+
TOKEN='' # optional supply an API token
24+
EXCLUDE_REGEX='##########'
25+
EXCLUDE_IP_REGEX='192.168.|172.16.|172.17.|172.18.|172.19.|172.20.|172.21.|172.22.|172.23.|172.24.|172.25.|172.26.|172.27.|172.28.|172.29.|172.30.|172.31.|10.|127.'
26+
MAX_PARALLEL=10
27+
28+
# NOTE: Bash regex does not support PCRE like '\d' '\s' nor non-greedy '*?'
29+
30+
# HAProxy logs
31+
# example:
32+
# setenv HTTPLOG_JSON "%{+json}o %(client_ip)ci %(client_port)cp %(request_date)tr %(fe_name_transport)ft %(be_name)b %(server_name)s %(time_request)TR %(time_wait)Tw %(time_connect)Tc %(time_response)Tr/%(time_active)Ta %(status_code)ST %(bytes_read)B %(captured_request_cookie)CC %(captured_response_cookie)CS %(termination_state_cookie)tsc %(actconn)ac %(feconn)fc %(beconn)bc %(srv_conn)sc %(retries)rc %(srv_queue)sq %(backend_queue)bq %(captured_request_headers)hr %(captured_response_headers)hs %(http_request){+Q}r"
33+
# log-format: "${HTTPLOG_JSON}"
34+
35+
# rsyslog rule:
36+
# file: /etc/rsyslog.d/haproxy.conf
37+
# content:
38+
# $AddUnixListenSocket /var/lib/haproxy/dev/log
39+
# :programname, startswith, "haproxy" {
40+
# /var/log/haproxy.log
41+
# stop
42+
# }
43+
44+
USER_AGENT='Abuse Reporter'
45+
46+
function report_json() {
47+
json="$1"
48+
curl -s -o /dev/null -XPOST 'https://risk.oxl.app/api/report' --data "$json" -H 'Content-Type: application/json' -H "Token: ${TOKEN}" -A "$USER_AGENT"
49+
}
50+
51+
function log_report() {
52+
ip="$1"
53+
category="$2"
54+
echo "REPORTING: ${ip} because of ${category}"
55+
}
56+
57+
# NOTE: you may want to add the user-agent as comment ('cmt' field) if you can extract it from your logs
58+
function report_ip_with_msg() {
59+
ip="$1"
60+
category="$2"
61+
comment="$3"
62+
log_report "$ip" "$category"
63+
report_json "{\"ip\": \"${ip}\", \"cat\": \"${category}\", \"cmt\": \"${comment}\"}"
64+
}
65+
66+
function analyze_log_line() {
67+
l="$1"
68+
69+
if [[ "$l" != *} ]]
70+
then
71+
return
72+
fi
73+
74+
# anti loop
75+
if echo "$l" | grep -q "$USER_AGENT"
76+
then
77+
return
78+
fi
79+
80+
# excludes
81+
if echo "$l" | grep -E -q "$EXCLUDE_REGEX"
82+
then
83+
return
84+
fi
85+
86+
json="{$(echo "$l" | cut -d '{' -f2-)"
87+
ip="$(echo "$json" | jq -r ".${FIELD_IP}")"
88+
status="$(echo "$json" | jq -r ".${FIELD_STATUS}")"
89+
90+
if [[ "$ip" == 'null' ]] || [[ "$status" == 'null' ]]
91+
then
92+
return
93+
fi
94+
95+
# excludes by IP
96+
if echo "$ip" | grep -E -q "$EXCLUDE_IP_REGEX"
97+
then
98+
return
99+
fi
100+
101+
# exclude by IP-list
102+
if [[ "$(python3 in_ip_list.py --iplist 'iplist.txt' --ip "$ip")" == "1" ]]
103+
then
104+
return
105+
fi
106+
107+
if [[ "$status" == '429' ]]
108+
then
109+
report_ip "$ip" 'rate' 'http'
110+
111+
elif [[ "$status" == "$BLOCK_STATUS_PROBE" ]]
112+
then
113+
report_ip "$ip" 'probe' 'http'
114+
115+
elif [[ "$status" == "$BLOCK_STATUS_BOT" ]]
116+
then
117+
report_ip "$ip" 'bot' 'http'
118+
119+
elif [[ "$status" == '400' ]] && echo "$l" | grep -v -q '/api'
120+
then
121+
report_ip "$ip" 'probe' 'http'
122+
123+
fi
124+
}
125+
126+
function read_log_line() {
127+
local l=''
128+
read -r
129+
while true
130+
do
131+
if [[ "$(jobs | wc -l)" -lt "$MAX_PARALLEL" ]]
132+
then
133+
analyze_log_line "$REPLY" &
134+
fi
135+
read -r
136+
done
137+
}
138+
139+
tail "$LOG_FILE" -n0 -f | read_log_line

0 commit comments

Comments
 (0)