Skip to content

Commit 2bb9d3e

Browse files
Copilotlmangani
Copilot
and
lmangani
committed
Add LokiSkipTCPPortLabels option to reduce cardinality
Co-authored-by: lmangani <1423657+lmangani@users.noreply.github.com>
1 parent 713911c commit 2bb9d3e

File tree

6 files changed

+138
-3
lines changed

6 files changed

+138
-3
lines changed

‎config/config.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ type HeplifyServer struct {
2121
LokiBuffer int `default:"100000"`
2222
LokiHEPFilter []int `default:"1,5,100"`
2323
LokiIPPortLabels bool `default:"false"`
24+
LokiSkipTCPPortLabels bool `default:"true"`
2425
LokiFromToLabels bool `default:"false"`
2526
LokiCallIDLabels bool `default:"false"`
2627
LokiAllowOutOfOrder bool `default:"false"`

‎example/homer5_config/heplify-server.toml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ LokiBulk = 200
88
LokiTimer = 4
99
LokiBuffer = 100000
1010
LokiHEPFilter = [1,5,100]
11+
LokiIPPortLabels = false
12+
LokiSkipTCPPortLabels = true
1113
LokiAllowOutOfOrder = false
1214
PromAddr = ""
1315
PromTargetIP = ""

‎example/homer7_config/heplify-server.toml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@ LokiBulk = 200
99
LokiTimer = 4
1010
LokiBuffer = 100000
1111
LokiHEPFilter = [1,5,100]
12+
LokiIPPortLabels = false
13+
LokiSkipTCPPortLabels = true
1214
LokiAllowOutOfOrder = false
1315
ForceHEPPayload = []
1416
PromAddr = ""

‎example/lineproto_config.toml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,8 @@ LokiBulk = 400
4242
LokiTimer = 4
4343
LokiBuffer = 100000
4444
LokiHEPFilter = [1, 5, 100]
45-
LokiIPPortLabels = false
45+
LokiIPPortLabels = false # Set to true to include src_ip, src_port, dst_ip, dst_port as labels
46+
LokiSkipTCPPortLabels = true # Skip port labels for TCP to reduce cardinality (only applies when LokiIPPortLabels is true)
4647
LokiAllowOutOfOrder = false
4748

4849
# General Settings

‎remotelog/loki.go‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -193,9 +193,14 @@ func (l *Loki) start(hCh chan *decoder.HEP) {
193193

194194
if config.Setting.LokiIPPortLabels {
195195
l.entry.labels["src_ip"] = model.LabelValue(pkt.SrcIP)
196-
l.entry.labels["src_port"] = model.LabelValue(strconv.FormatUint(uint64(pkt.SrcPort), 10))
197196
l.entry.labels["dst_ip"] = model.LabelValue(pkt.DstIP)
198-
l.entry.labels["dst_port"] = model.LabelValue(strconv.FormatUint(uint64(pkt.DstPort), 10))
197+
// Skip port labels for TCP if LokiSkipTCPPortLabels is enabled (default: true)
198+
// This reduces cardinality since TCP source ports are typically dynamic
199+
skipPortLabels := config.Setting.LokiSkipTCPPortLabels && pkt.Protocol == 6
200+
if !skipPortLabels {
201+
l.entry.labels["src_port"] = model.LabelValue(strconv.FormatUint(uint64(pkt.SrcPort), 10))
202+
l.entry.labels["dst_port"] = model.LabelValue(strconv.FormatUint(uint64(pkt.DstPort), 10))
203+
}
199204
}
200205

201206
if batchSize+len(l.entry.Line) > l.BatchSize {

‎remotelog/loki_test.go‎

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,10 @@ import (
66
"net/http/httptest"
77
"testing"
88

9+
"github.com/prometheus/common/model"
910
"github.com/sipcapture/heplify-server/config"
11+
"github.com/sipcapture/heplify-server/decoder"
12+
"github.com/sipcapture/heplify-server/remotelog/logproto"
1013
)
1114

1215
func withConfig(t *testing.T, fn func()) {
@@ -95,3 +98,124 @@ func TestLokiSendSendsOrgIDHeaderAndFailsOnNon2xx(t *testing.T) {
9598
}
9699
})
97100
}
101+
102+
func TestLokiSkipsTCPPortLabels(t *testing.T) {
103+
withConfig(t, func() {
104+
config.Setting.LokiIPPortLabels = true
105+
config.Setting.LokiSkipTCPPortLabels = true
106+
107+
loki := &Loki{}
108+
loki.entry = entry{model.LabelSet{}, logproto.Entry{}}
109+
110+
// Test TCP packet (protocol 6)
111+
tcpPkt := &decoder.HEP{
112+
Protocol: 6, // TCP
113+
SrcIP: "192.168.1.1",
114+
DstIP: "192.168.1.2",
115+
SrcPort: 12345,
116+
DstPort: 5060,
117+
}
118+
119+
// Simulate the label setting logic from loki.go
120+
if config.Setting.LokiIPPortLabels {
121+
loki.entry.labels["src_ip"] = model.LabelValue(tcpPkt.SrcIP)
122+
loki.entry.labels["dst_ip"] = model.LabelValue(tcpPkt.DstIP)
123+
skipPortLabels := config.Setting.LokiSkipTCPPortLabels && tcpPkt.Protocol == 6
124+
if !skipPortLabels {
125+
loki.entry.labels["src_port"] = model.LabelValue("12345")
126+
loki.entry.labels["dst_port"] = model.LabelValue("5060")
127+
}
128+
}
129+
130+
// Verify TCP ports are NOT in labels
131+
if _, exists := loki.entry.labels["src_port"]; exists {
132+
t.Error("expected src_port label to be skipped for TCP")
133+
}
134+
if _, exists := loki.entry.labels["dst_port"]; exists {
135+
t.Error("expected dst_port label to be skipped for TCP")
136+
}
137+
// Verify IPs are still present
138+
if loki.entry.labels["src_ip"] != "192.168.1.1" {
139+
t.Errorf("expected src_ip label to be set, got %v", loki.entry.labels["src_ip"])
140+
}
141+
if loki.entry.labels["dst_ip"] != "192.168.1.2" {
142+
t.Errorf("expected dst_ip label to be set, got %v", loki.entry.labels["dst_ip"])
143+
}
144+
})
145+
}
146+
147+
func TestLokiIncludesUDPPortLabels(t *testing.T) {
148+
withConfig(t, func() {
149+
config.Setting.LokiIPPortLabels = true
150+
config.Setting.LokiSkipTCPPortLabels = true
151+
152+
loki := &Loki{}
153+
loki.entry = entry{model.LabelSet{}, logproto.Entry{}}
154+
155+
// Test UDP packet (protocol 17)
156+
udpPkt := &decoder.HEP{
157+
Protocol: 17, // UDP
158+
SrcIP: "192.168.1.1",
159+
DstIP: "192.168.1.2",
160+
SrcPort: 5060,
161+
DstPort: 5060,
162+
}
163+
164+
// Simulate the label setting logic from loki.go
165+
if config.Setting.LokiIPPortLabels {
166+
loki.entry.labels["src_ip"] = model.LabelValue(udpPkt.SrcIP)
167+
loki.entry.labels["dst_ip"] = model.LabelValue(udpPkt.DstIP)
168+
skipPortLabels := config.Setting.LokiSkipTCPPortLabels && udpPkt.Protocol == 6
169+
if !skipPortLabels {
170+
loki.entry.labels["src_port"] = model.LabelValue("5060")
171+
loki.entry.labels["dst_port"] = model.LabelValue("5060")
172+
}
173+
}
174+
175+
// Verify UDP ports ARE in labels
176+
if loki.entry.labels["src_port"] != "5060" {
177+
t.Errorf("expected src_port label to be set for UDP, got %v", loki.entry.labels["src_port"])
178+
}
179+
if loki.entry.labels["dst_port"] != "5060" {
180+
t.Errorf("expected dst_port label to be set for UDP, got %v", loki.entry.labels["dst_port"])
181+
}
182+
})
183+
}
184+
185+
func TestLokiIncludesTCPPortLabelsWhenSkipDisabled(t *testing.T) {
186+
withConfig(t, func() {
187+
config.Setting.LokiIPPortLabels = true
188+
config.Setting.LokiSkipTCPPortLabels = false
189+
190+
loki := &Loki{}
191+
loki.entry = entry{model.LabelSet{}, logproto.Entry{}}
192+
193+
// Test TCP packet (protocol 6)
194+
tcpPkt := &decoder.HEP{
195+
Protocol: 6, // TCP
196+
SrcIP: "192.168.1.1",
197+
DstIP: "192.168.1.2",
198+
SrcPort: 12345,
199+
DstPort: 5060,
200+
}
201+
202+
// Simulate the label setting logic from loki.go
203+
if config.Setting.LokiIPPortLabels {
204+
loki.entry.labels["src_ip"] = model.LabelValue(tcpPkt.SrcIP)
205+
loki.entry.labels["dst_ip"] = model.LabelValue(tcpPkt.DstIP)
206+
skipPortLabels := config.Setting.LokiSkipTCPPortLabels && tcpPkt.Protocol == 6
207+
if !skipPortLabels {
208+
loki.entry.labels["src_port"] = model.LabelValue("12345")
209+
loki.entry.labels["dst_port"] = model.LabelValue("5060")
210+
}
211+
}
212+
213+
// Verify TCP ports ARE in labels when skip is disabled
214+
if loki.entry.labels["src_port"] != "12345" {
215+
t.Errorf("expected src_port label to be set when skip is disabled, got %v", loki.entry.labels["src_port"])
216+
}
217+
if loki.entry.labels["dst_port"] != "5060" {
218+
t.Errorf("expected dst_port label to be set when skip is disabled, got %v", loki.entry.labels["dst_port"])
219+
}
220+
})
221+
}

0 commit comments

Comments
 (0)