Merge pull request #1097 from rschirin/fix-scan-timeframe

fix scan timeframe with use_terms_query

Author: jertel

CHANGELOG.md

@@ -14,6 +14,7 @@
 - Add support for Kibana 8.6 for Kibana Discover - [#1080](https://github.com/jertel/elastalert2/pull/1080) - @nsano-rururu
 - Modify schema to allow string and boolean for `*_ca_certs` to allow for one to specify a cert bundle for SSL certificate verification - [#1082](https://github.com/jertel/elastalert2/pull/1082) - @goggin
 - Fix UnicodeEncodeError in PagerDutyAlerter - [#1091](https://github.com/jertel/elastalert2/pull/1091) - @nsano-rururu
+- The scan_entire_timeframe setting, when used with use_count_query or use_terms_query will now scan entire timeframe on subsequent rule runs - [#1097](https://github.com/jertel/elastalert2/pull/1097) - @rschirin
 
 # 2.9.0
 

elastalert/elastalert.py

@@ -698,7 +698,8 @@ def set_starttime(self, rule, endtime):
                 # Query from the end of the last run, if it exists, otherwise a run_every sized window
                 rule['starttime'] = rule.get('previous_endtime', endtime - self.run_every)
             else:
-                rule['starttime'] = rule.get('previous_endtime', endtime - rule['timeframe'])
+                #Based on PR 3141 old Yelp/elastalert - rschirin
+                rule['starttime'] = endtime - rule['timeframe']
 
     def adjust_start_time_for_overlapping_agg_query(self, rule):
         if rule.get('aggregation_query_element'):

tests/base_test.py

@@ -876,8 +876,7 @@ def test_set_starttime(ea):
     ea.set_starttime(ea.rules[0], end)
     assert ea.rules[0]['starttime'] == end - ea.buffer_time
 
-    # scan_entire_timeframe
-    ea.rules[0].pop('previous_endtime')
+    # scan_entire_timeframe without use_count_query or use_terms_query
     ea.rules[0].pop('starttime')
     ea.rules[0]['timeframe'] = datetime.timedelta(days=3)
     ea.rules[0]['scan_entire_timeframe'] = True
@@ -886,6 +885,27 @@ def test_set_starttime(ea):
         ea.set_starttime(ea.rules[0], end)
     assert ea.rules[0]['starttime'] == end - datetime.timedelta(days=3)
 
+    # scan_entire_timeframe with use_count_query, first run
+    ea.rules[0].pop('starttime')
+    ea.rules[0]['timeframe'] = datetime.timedelta(days=3)
+    ea.rules[0]['scan_entire_timeframe'] = True
+    ea.rules[0]['use_count_query'] = True
+    with mock.patch.object(ea, 'get_starttime') as mock_gs:
+        mock_gs.return_value = None
+        ea.set_starttime(ea.rules[0], end)
+    assert ea.rules[0]['starttime'] == end - datetime.timedelta(days=3)
+
+    # scan_entire_timeframe with use_count_query, subsequent run
+    ea.rules[0].pop('starttime')
+    ea.rules[0]['timeframe'] = datetime.timedelta(days=3)
+    ea.rules[0]['scan_entire_timeframe'] = True
+    ea.rules[0]['use_count_query'] = True
+    ea.rules[0]['previous_endtime'] = end
+    with mock.patch.object(ea, 'get_starttime') as mock_gs:
+        mock_gs.return_value = None
+        ea.set_starttime(ea.rules[0], end)
+    assert ea.rules[0]['starttime'] == end - datetime.timedelta(days=3)
+
 
 def test_rule_changes(ea):
     ea.rule_hashes = {'rules/rule1.yaml': 'ABC',