When collecting application logs, the log lines are disordered and the logs are lost.

[Wangwei9138]

What is a problem?

When I collect the application log (the application log will be cut on the hour, and the last hour's log will be named according to the IP address-timestamp of the application pod), some log lines will be lost. The bottom layer uses NAS storage, and the following is my configuration (multiple log sources) and log format (JSON). My application is deployed in K8S, with 7 resident PODs and 5 elastic PODs.

Describe the configuration of Fluentd

<source>
  @type tail
  path /hwdata/log/adlink_adx_log/adlink_adx.log
  pos_file /hwdata/log/adlink_adx_pos/adlink_adx.pos
  tag adlink_adx
  rotate_wait 5s
  refresh_interval 1s
  follow_inodes true
  
  <parse>
    @type json
    time_key "time"
    time_format "%Y-%m-%d %H:%M:%S"
    keep_time_key true
    time_type string
  </parse>
</source>

<source>
  @type tail
  path /hwdata/log/dsp_response_log/dsp_response.log
  pos_file /hwdata/log/dsp_response_pos/dsp_response.pos
  tag dsp_response
  
  <parse>
    @type none
  </parse>
</source>

<source>
  @type tail
  path /hwdata/log/dsp_original_response_log/dsp_original_response_log.log
  pos_file /hwdata/log/dsp_original_response_pos/dsp_original_response.pos
  tag dsp_original_response
  
  <parse>
    @type none
  </parse>
</source>


<source>
  @type tail
  path /hwdata/log/adx_monitor_log/adx_monitor.log
  pos_file /hwdata/log/adx_monitor_pos/adx_monitor.pos
  tag adx_monitor
  rotate_wait 5s			# 旧文件继续读取5秒
  refresh_interval 1s	# 每1秒检查新文件
  follow_inodes true	# 跟踪inode变化
  
  <parse>
    @type json
    time_key "time"
    time_format "%Y-%m-%d %H:%M:%S"
    keep_time_key true
    time_type string
  </parse>
</source>

<source>
  @type tail
  path /hwdata/log/dmp_request_log/dmp_request_log.log
  pos_file /hwdata/log/dmp_request_pos/dmp_request.pos
  tag dmp_request
  <parse>
    @type none
  </parse>
</source>


<source>
  @type tail  
  path /hwdata/log/deep_zero_log/deep_zero_log.log
  pos_file /hwdata/log/deep_zero_pos/deep_zero.pos
  tag deep_zero   
  <parse>
    @type none
  </parse>
</source>


<source>
  @type tail                
  path /hwdata/log/ak_ad_resp_sample_log/ak_ad_resp_sample_log.log
  pos_file /hwdata/log/ak_ad_resp_sample_pos/ak_ad_resp_sample.pos
  tag ak_ad_resp_sample
  <parse>
    @type none
  </parse>
</source>
    
<source>
  @type tail
  path /hwdata/log/ivt_log/ivt_log.log    
  pos_file /hwdata/log/ivt_pos/ivt.pos
  tag ivt
  <parse>
    @type none
  </parse>  
</source>

<source>
  @type forward
  port 24231
  bind 0.0.0.0
</source>


<match  adlink_adx>
  @type file
  enable_ruby true
  path "/opt/logs/adlink_adx/adlink_adx.%Y-%m-%d-%H-#{ENV['ip_host']}"
  append true
  <format>
    @type json
  </format>
  <buffer time>
    timekey_use_utc false
    flush_mode interval
    flush_interval 5s
    timekey 3600s
    timekey_wait 60s
    flush_thread_count 8
    flush_thread_interval 5s
    chunk_limit_size 32m
    total_limit_size 2G
    overflow_action block
    path /tmp/buffer/adlink_adx
  </buffer>
</match>

<match  dsp_response>
  @type file
  enable_ruby true
  path "/opt/logs/dsp_response/dsp_response.%Y-%m-%d-%H-#{ENV['ip_host']}"
  append true
  <format>
    @type single_value
  </format>
  <buffer time>
    timekey_use_utc false
    flush_mode interval
    flush_interval 5s
    timekey 3600s
    timekey_wait 120s
    flush_thread_count 8
    flush_thread_interval 5s
    chunk_limit_size 32m
    total_limit_size 2G
    overflow_action block
    path /tmp/buffer/dsp_response
  </buffer>
</match>

<match  dsp_original_response>
  @type file
  enable_ruby true
  path "/opt/logs/dsp_original_response/dsp_original_response.%Y-%m-%d-%H-#{ENV['ip_host']}"
  append true
  <format>
    @type single_value
  </format>
  <buffer time>
    timekey_use_utc false
    flush_mode interval
    flush_interval 5s
    timekey 3600s
    timekey_wait 120s
    flush_thread_count 8
    flush_thread_interval 5s
    chunk_limit_size 32m
    total_limit_size 2G
    overflow_action block
    path /tmp/buffer/dsp_original_response
  </buffer>
</match>

<match  adx_monitor>
  @type file
  enable_ruby true
  path "/opt/logs/adx_monitor/adx_monitor.%Y-%m-%d-%H-#{ENV['ip_host']}"
  append true
  <format>
    @type json
  </format>
  <buffer time>
    timekey_use_utc false
    flush_mode interval
    flush_interval 5s
    timekey 3600s
    timekey_wait 60s
    flush_thread_count 8
    flush_thread_interval 5s
    chunk_limit_size 32m
    total_limit_size 2G
    overflow_action block
    path /tmp/buffer/adx_monitor
  </buffer>
</match>


<match  dmp_request>
  @type file
  enable_ruby true
  path "/opt/logs/dmp_request/dmp_request.%Y-%m-%d-%H-#{ENV['ip_host']}"
  append true
  <format>
    @type single_value
  </format>
  <buffer time>
    timekey_use_utc false
    flush_mode interval
    flush_interval 5s
    timekey 3600s
    timekey_wait 120s
    flush_thread_count 8
    flush_thread_interval 5s
    chunk_limit_size 32m
    total_limit_size 2G
    overflow_action block
    path /tmp/buffer/dmp_request
  </buffer>
</match>


<match  deep_zero>
  @type file
  enable_ruby true
  path "/opt/logs/deep_zero/deep_zero.%Y-%m-%d-%H-#{ENV['ip_host']}"
  append true
  <format>
    @type single_value
  </format>
  <buffer time>
    timekey_use_utc false
    flush_mode interval
    flush_interval 5s
    timekey 3600s
    timekey_wait 120s
    flush_thread_count 8
    flush_thread_interval 5s
    chunk_limit_size 32m
    total_limit_size 2G
    overflow_action block
    path /tmp/buffer/deep_zero
  </buffer>
</match>


<match  ak_ad_resp_sample>
  @type file
  enable_ruby true
  path "/opt/logs/ak_ad_resp_sample/ak_ad_resp_sample.%Y-%m-%d-%H-#{ENV['ip_host']}"
  append true
  <format>
    @type single_value
  </format>
  <buffer time>
    timekey_use_utc false
    flush_mode interval
    flush_interval 5s
    timekey 3600s
    timekey_wait 120s
    flush_thread_count 8
    flush_thread_interval 5s
    chunk_limit_size 32m
    total_limit_size 2G
    overflow_action block
    path /tmp/buffer/ak_ad_resp_sample
  </buffer>
</match>


<match ivt>
  @type file
  enable_ruby true
  path "/opt/logs/ivt/ivt.%Y-%m-%d-%H-#{ENV['ip_host']}"
  append true
  <format>
    @type single_value
  </format>
  <buffer time>
    timekey_use_utc false
    flush_mode interval
    flush_interval 5s
    timekey 3600s
    timekey_wait 120s
    flush_thread_count 8
    flush_thread_interval 5s
    chunk_limit_size 32m
    total_limit_size 2G
    overflow_action block
    path /tmp/buffer/ivt
  </buffer>
</match>

Describe the logs of Fluentd

No Error

Environment

'fluentd' version '1.16.5'