Reduce bookmark rewind

.gitignore

@@ -102,3 +102,6 @@ ENV/
 
 # Emacs
 .tramp_history
+
+# ignore PyCharm
+.idea

config.json.example

@@ -17,5 +17,7 @@
     "pagination__data_extension_interval_unit": "days",
     "pagination__data_extension_interval_quantity": "7",
     "pagination__list_subscriber_interval_unit": "days",
-    "pagination__list_subscriber_interval_quantity": "7"
+    "pagination__list_subscriber_interval_quantity": "7",
+    "state__bookmark_interval_unit": "days",
+    "state__bookmark_interval": "2"
 }

tap_exacttarget/endpoints/content_areas.py

@@ -110,7 +110,7 @@ def sync_data(self):
         selector = FuelSDK.ET_ContentArea
 
         search_filter = None
-        retrieve_all_since = get_last_record_value_for_table(self.state, table)
+        retrieve_all_since = get_last_record_value_for_table(self.state, table, self.config)
 
         if retrieve_all_since is not None:
             search_filter = {

tap_exacttarget/endpoints/data_extensions.py

@@ -232,7 +232,7 @@ def sync_data(self):
 
         replication_key = None
 
-        start = get_last_record_value_for_table(self.state, table)
+        start = get_last_record_value_for_table(self.state, table, self.config)
 
         if start is None:
             start = self.config.get('start_date')

tap_exacttarget/endpoints/emails.py

@@ -125,7 +125,7 @@ def sync_data(self):
         selector = FuelSDK.ET_Email
 
         search_filter = None
-        retrieve_all_since = get_last_record_value_for_table(self.state, table)
+        retrieve_all_since = get_last_record_value_for_table(self.state, table, self.config)
 
         if retrieve_all_since is not None:
             search_filter = {

tap_exacttarget/endpoints/events.py

@@ -47,7 +47,7 @@ def sync_data(self):
         for event_name, selector in endpoints.items():
             search_filter = None
 
-            start = get_last_record_value_for_table(self.state, event_name)
+            start = get_last_record_value_for_table(self.state, event_name, self.config)
 
             if start is None:
                 start = self.config.get('start_date')

tap_exacttarget/endpoints/folders.py

@@ -66,7 +66,7 @@ def sync_data(self):
 
         search_filter = None
 
-        retrieve_all_since = get_last_record_value_for_table(self.state, table)
+        retrieve_all_since = get_last_record_value_for_table(self.state, table, self.config)
 
         if retrieve_all_since is not None:
             search_filter = {

tap_exacttarget/endpoints/list_sends.py

@@ -107,7 +107,7 @@ def sync_data(self):
         selector = FuelSDK.ET_ListSend
 
         search_filter = None
-        retrieve_all_since = get_last_record_value_for_table(self.state, table)
+        retrieve_all_since = get_last_record_value_for_table(self.state, table, self.config)
 
         if retrieve_all_since is not None:
             search_filter = {

tap_exacttarget/endpoints/list_subscribers.py

@@ -90,7 +90,7 @@ def sync_data(self):
             self.auth_stub,
             self.subscriber_catalog)
 
-        start = get_last_record_value_for_table(self.state, table)
+        start = get_last_record_value_for_table(self.state, table, self.config)
 
         if start is None:
             start = self.config.get('start_date')

tap_exacttarget/endpoints/lists.py

@@ -55,7 +55,7 @@ def sync_data(self):
         selector = FuelSDK.ET_List
 
         search_filter = None
-        retrieve_all_since = get_last_record_value_for_table(self.state, table)
+        retrieve_all_since = get_last_record_value_for_table(self.state, table, self.config)
 
         if retrieve_all_since is not None:
             search_filter = {

tap_exacttarget/endpoints/sends.py

@@ -94,7 +94,7 @@ def sync_data(self):
         selector = FuelSDK.ET_Send
 
         search_filter = None
-        retrieve_all_since = get_last_record_value_for_table(self.state, table)
+        retrieve_all_since = get_last_record_value_for_table(self.state, table, self.config)
 
         if retrieve_all_since is not None:
             search_filter = {

tap_exacttarget/pagination.py

@@ -22,6 +22,15 @@ def increment_date(date_value, unit=None):
 
     return datetime.datetime.strftime(incremented_date_obj, DATE_FORMAT)
 
+def decrement_date(date_value, unit=None):
+    if unit is None:
+        unit = {'days': 1}
+
+    date_obj = datetime.datetime.strptime(date_value, DATE_FORMAT)
+
+    decrement_date_obj = date_obj - datetime.timedelta(**unit)
+
+    return datetime.datetime.strftime(decrement_date_obj, DATE_FORMAT)
 
 def get_date_page(field, start, unit):
     return between(field, start, increment_date(start, unit))

tap_exacttarget/state.py

@@ -5,7 +5,7 @@
 
 from voluptuous import Schema, Required
 
-from tap_exacttarget.pagination import DATE_FORMAT
+from tap_exacttarget.pagination import DATE_FORMAT, decrement_date
 
 LOGGER = singer.get_logger()
 
@@ -19,18 +19,20 @@
 })
 
 
-def get_last_record_value_for_table(state, table):
+def get_last_record_value_for_table(state, table, config=None):
     raw = state.get('bookmarks', {}) \
                .get(table, {}) \
                .get('last_record')
 
     if raw is None:
         return None
 
-    date_obj = datetime.datetime.strptime(raw, DATE_FORMAT)
-    date_obj = date_obj - datetime.timedelta(days=1)
+    interval_unit = config.get('state__bookmark_interval_unit', 'days')
+    interval = config.get('state__bookmark_interval', 1)
 
-    return date_obj.strftime(DATE_FORMAT)
+    unit = {interval_unit: int(interval)}
+
+    return decrement_date(state, unit)
 
 
 def incorporate(state, table, field, value):

tap_exacttarget/test/test_pagination.py

@@ -1,6 +1,6 @@
 import unittest
 
-from tap_exacttarget.pagination import increment_date
+from tap_exacttarget.pagination import increment_date, decrement_date
 
 
 class TestPagination(unittest.TestCase):
@@ -12,3 +12,11 @@ def test_increment_date(self):
         self.assertEqual(
             increment_date("2015-09-28T10:05:53Z", {'hours': 1}),
             "2015-09-28T11:05:53Z")
+
+    def test_decrement_date(self):
+        self.assertEqual(
+            decrement_date("2015-09-28T10:05:53Z"),
+            "2015-09-27T10:05:53Z")
+        self.assertEqual(
+            decrement_date("2015-09-28T10:05:53Z", {'hours': 1}),
+            "2015-09-28T09:05:53Z")