Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Journald E2E flake #34505

Draft
wants to merge 4 commits into
base: main
Choose a base branch
from
+447 −95
Draft

Journald E2E flake #34505

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
f6d99c5
pre-incident code
rahulkaukuntla Feb 25, 2025
66e1fb8
testing every parseable struct and including inputs used in e2e tests
rahulkaukuntla Feb 25, 2025
5767fb8
lint
rahulkaukuntla Feb 25, 2025
ededddd
delay test
DDuongNguyen Feb 26, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions comp/logs/agent/config/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,9 @@ require (
github.com/DataDog/datadog-agent/pkg/util/fxutil v0.61.0
github.com/DataDog/datadog-agent/pkg/util/log v0.64.0-devel
github.com/DataDog/datadog-agent/pkg/util/pointer v0.61.0
github.com/DataDog/viper v1.14.0
github.com/stretchr/testify v1.10.0
go.uber.org/fx v1.23.0
gopkg.in/yaml.v3 v3.0.1
)

require (
Expand All @@ -37,6 +37,7 @@ require (
github.com/DataDog/datadog-agent/pkg/util/system/socket v0.61.0 // indirect
github.com/DataDog/datadog-agent/pkg/util/winutil v0.61.0 // indirect
github.com/DataDog/datadog-agent/pkg/version v0.62.3 // indirect
github.com/DataDog/viper v1.14.0 // indirect
github.com/Microsoft/go-winio v0.6.2 // indirect
github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
Expand Down Expand Up @@ -72,7 +73,6 @@ require (
golang.org/x/sys v0.30.0 // indirect
golang.org/x/text v0.22.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

// This section was automatically added by 'invoke modules.add-all-replace' command, do not edit manually
Expand Down
62 changes: 43 additions & 19 deletions comp/logs/agent/config/integration_config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,22 +42,22 @@ type LogsConfig struct {
IntegrationName string

Port int // Network
IdleTimeout string `mapstructure:"idle_timeout" json:"idle_timeout"` // Network
IdleTimeout string `mapstructure:"idle_timeout" json:"idle_timeout" yaml:"idle_timeout"` // Network
Path string // File, Journald

Encoding string `mapstructure:"encoding" json:"encoding"` // File
ExcludePaths []string `mapstructure:"exclude_paths" json:"exclude_paths"` // File
TailingMode string `mapstructure:"start_position" json:"start_position"` // File
Encoding string `mapstructure:"encoding" json:"encoding" yaml:"encoding"` // File
ExcludePaths []string `mapstructure:"exclude_paths" json:"exclude_paths" yaml:"exclude_paths"` // File
TailingMode string `mapstructure:"start_position" json:"start_position" yaml:"start_position"` // File

//nolint:revive // TODO(AML) Fix revive linter
ConfigId string `mapstructure:"config_id" json:"config_id"` // Journald
IncludeSystemUnits []string `mapstructure:"include_units" json:"include_units"` // Journald
ExcludeSystemUnits []string `mapstructure:"exclude_units" json:"exclude_units"` // Journald
IncludeUserUnits []string `mapstructure:"include_user_units" json:"include_user_units"` // Journald
ExcludeUserUnits []string `mapstructure:"exclude_user_units" json:"exclude_user_units"` // Journald
IncludeMatches []string `mapstructure:"include_matches" json:"include_matches"` // Journald
ExcludeMatches []string `mapstructure:"exclude_matches" json:"exclude_matches"` // Journald
ContainerMode bool `mapstructure:"container_mode" json:"container_mode"` // Journald
ConfigId string `mapstructure:"config_id" json:"config_id" yaml:"config_id"` // Journald
IncludeSystemUnits []string `mapstructure:"include_units" json:"include_units" yaml:"include_units"` // Journald
ExcludeSystemUnits []string `mapstructure:"exclude_units" json:"exclude_units" yaml:"exclude_units"` // Journald
IncludeUserUnits []string `mapstructure:"include_user_units" json:"include_user_units" yaml:"include_user_units"` // Journald
ExcludeUserUnits []string `mapstructure:"exclude_user_units" json:"exclude_user_units" yaml:"exclude_user_units"` // Journald
IncludeMatches []string `mapstructure:"include_matches" json:"include_matches" yaml:"include_matches"` // Journald
ExcludeMatches []string `mapstructure:"exclude_matches" json:"exclude_matches" yaml:"exclude_matches"` // Journald
ContainerMode bool `mapstructure:"container_mode" json:"container_mode" yaml:"container_mode"` // Journald

Image string // Docker
Label string // Docker
Expand All @@ -67,7 +67,7 @@ type LogsConfig struct {
// determine the appropriate tags for the logs.
Identifier string // Docker, File

ChannelPath string `mapstructure:"channel_path" json:"channel_path"` // Windows Event
ChannelPath string `mapstructure:"channel_path" json:"channel_path" yaml:"channel_path"` // Windows Event
Query string // Windows Event

// used as input only by the Channel tailer.
Expand All @@ -84,14 +84,38 @@ type LogsConfig struct {
Service string
Source string
SourceCategory string
Tags []string
ProcessingRules []*ProcessingRule `mapstructure:"log_processing_rules" json:"log_processing_rules"`
Tags TagsField
ProcessingRules []*ProcessingRule `mapstructure:"log_processing_rules" json:"log_processing_rules" yaml:"log_processing_rules"`
// ProcessRawMessage is used to process the raw message instead of only the content part of the message.
ProcessRawMessage *bool `mapstructure:"process_raw_message" json:"process_raw_message"`
ProcessRawMessage *bool `mapstructure:"process_raw_message" json:"process_raw_message" yaml:"process_raw_message"`

AutoMultiLine *bool `mapstructure:"auto_multi_line_detection" json:"auto_multi_line_detection"`
AutoMultiLineSampleSize int `mapstructure:"auto_multi_line_sample_size" json:"auto_multi_line_sample_size"`
AutoMultiLineMatchThreshold float64 `mapstructure:"auto_multi_line_match_threshold" json:"auto_multi_line_match_threshold"`
AutoMultiLine *bool `mapstructure:"auto_multi_line_detection" json:"auto_multi_line_detection" yaml:"auto_multi_line_detection"`
AutoMultiLineSampleSize int `mapstructure:"auto_multi_line_sample_size" json:"auto_multi_line_sample_size" yaml:"auto_multi_line_sample_size"`
AutoMultiLineMatchThreshold float64 `mapstructure:"auto_multi_line_match_threshold" json:"auto_multi_line_match_threshold" yaml:"auto_multi_line_match_threshold"`
}

// TagsField is a custom type for unmarshalling comma-separated string values or typical yaml fields into a slice of strings.
type TagsField []string

// UnmarshalYAML is a custom unmarshalling function is needed for string array fields to split comma-separated values.
func (t *TagsField) UnmarshalYAML(unmarshal func(interface{}) error) error {
var str string
if err := unmarshal(&str); err == nil {
// note that we are intentionally avoiding the trimming of any spaces whilst splitting the string
*t = strings.Split(str, ",")
return nil
}

var raw []interface{}
if err := unmarshal(&raw); err == nil {
for _, item := range raw {
if str, ok := item.(string); ok {
*t = append(*t, str)
}
}
return nil
}
return fmt.Errorf("invalid tags format")
}

// Dump dumps the contents of this struct to a string, for debugging purposes.
Expand Down
24 changes: 8 additions & 16 deletions comp/logs/agent/config/parser.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,15 @@
package config

import (
"bytes"
"encoding/json"
"fmt"

"github.com/DataDog/viper"
yaml "gopkg.in/yaml.v3"
)

type yamlLogsConfigsWrapper struct {
Logs []*LogsConfig
}

// ParseJSON parses the data formatted in JSON
// returns an error if the parsing failed.
func ParseJSON(data []byte) ([]*LogsConfig, error) {
Expand All @@ -24,23 +26,13 @@ func ParseJSON(data []byte) ([]*LogsConfig, error) {
return configs, nil
}

const yaml = "yaml"
const logsPath = "logs"

// ParseYAML parses the data formatted in YAML,
// returns an error if the parsing failed.
func ParseYAML(data []byte) ([]*LogsConfig, error) {
var configs []*LogsConfig
var err error
v := viper.New()
v.SetConfigType(yaml)
err = v.ReadConfig(bytes.NewBuffer(data))
var yamlConfigsWrapper yamlLogsConfigsWrapper
err := yaml.Unmarshal(data, &yamlConfigsWrapper)
if err != nil {
return nil, fmt.Errorf("could not decode YAML logs config: %v", err)
}
err = v.UnmarshalKey(logsPath, &configs)
if err != nil {
return nil, fmt.Errorf("could not parse YAML logs config: %v", err)
}
return configs, nil
return yamlConfigsWrapper.Logs, nil
}
Loading
Loading