diff --git a/.gitignore b/.gitignore index ff51507..822ec07 100644 --- a/.gitignore +++ b/.gitignore @@ -102,3 +102,6 @@ ENV/ # Emacs .tramp_history + +# ignore PyCharm +.idea diff --git a/config.json.example b/config.json.example index 8eba722..2dab96c 100644 --- a/config.json.example +++ b/config.json.example @@ -17,5 +17,7 @@ "pagination__data_extension_interval_unit": "days", "pagination__data_extension_interval_quantity": "7", "pagination__list_subscriber_interval_unit": "days", - "pagination__list_subscriber_interval_quantity": "7" + "pagination__list_subscriber_interval_quantity": "7", + "state__bookmark_interval_unit": "days", + "state__bookmark_interval": "2" } diff --git a/tap_exacttarget/endpoints/content_areas.py b/tap_exacttarget/endpoints/content_areas.py index e44f6a9..ea9412a 100644 --- a/tap_exacttarget/endpoints/content_areas.py +++ b/tap_exacttarget/endpoints/content_areas.py @@ -110,7 +110,7 @@ def sync_data(self): selector = FuelSDK.ET_ContentArea search_filter = None - retrieve_all_since = get_last_record_value_for_table(self.state, table) + retrieve_all_since = get_last_record_value_for_table(self.state, table, self.config) if retrieve_all_since is not None: search_filter = { diff --git a/tap_exacttarget/endpoints/data_extensions.py b/tap_exacttarget/endpoints/data_extensions.py index c5dcec8..73b09c2 100644 --- a/tap_exacttarget/endpoints/data_extensions.py +++ b/tap_exacttarget/endpoints/data_extensions.py @@ -232,7 +232,7 @@ def sync_data(self): replication_key = None - start = get_last_record_value_for_table(self.state, table) + start = get_last_record_value_for_table(self.state, table, self.config) if start is None: start = self.config.get('start_date') diff --git a/tap_exacttarget/endpoints/emails.py b/tap_exacttarget/endpoints/emails.py index 4c0c089..ffcc7ee 100644 --- a/tap_exacttarget/endpoints/emails.py +++ b/tap_exacttarget/endpoints/emails.py @@ -125,7 +125,7 @@ def sync_data(self): selector = FuelSDK.ET_Email search_filter = None - retrieve_all_since = get_last_record_value_for_table(self.state, table) + retrieve_all_since = get_last_record_value_for_table(self.state, table, self.config) if retrieve_all_since is not None: search_filter = { diff --git a/tap_exacttarget/endpoints/events.py b/tap_exacttarget/endpoints/events.py index 7cc80fe..e124fd8 100644 --- a/tap_exacttarget/endpoints/events.py +++ b/tap_exacttarget/endpoints/events.py @@ -47,7 +47,7 @@ def sync_data(self): for event_name, selector in endpoints.items(): search_filter = None - start = get_last_record_value_for_table(self.state, event_name) + start = get_last_record_value_for_table(self.state, event_name, self.config) if start is None: start = self.config.get('start_date') diff --git a/tap_exacttarget/endpoints/folders.py b/tap_exacttarget/endpoints/folders.py index e52247a..d4d4eda 100644 --- a/tap_exacttarget/endpoints/folders.py +++ b/tap_exacttarget/endpoints/folders.py @@ -66,7 +66,7 @@ def sync_data(self): search_filter = None - retrieve_all_since = get_last_record_value_for_table(self.state, table) + retrieve_all_since = get_last_record_value_for_table(self.state, table, self.config) if retrieve_all_since is not None: search_filter = { diff --git a/tap_exacttarget/endpoints/list_sends.py b/tap_exacttarget/endpoints/list_sends.py index 136697a..23fd37b 100644 --- a/tap_exacttarget/endpoints/list_sends.py +++ b/tap_exacttarget/endpoints/list_sends.py @@ -107,7 +107,7 @@ def sync_data(self): selector = FuelSDK.ET_ListSend search_filter = None - retrieve_all_since = get_last_record_value_for_table(self.state, table) + retrieve_all_since = get_last_record_value_for_table(self.state, table, self.config) if retrieve_all_since is not None: search_filter = { diff --git a/tap_exacttarget/endpoints/list_subscribers.py b/tap_exacttarget/endpoints/list_subscribers.py index 5a7d23f..21157c7 100644 --- a/tap_exacttarget/endpoints/list_subscribers.py +++ b/tap_exacttarget/endpoints/list_subscribers.py @@ -90,7 +90,7 @@ def sync_data(self): self.auth_stub, self.subscriber_catalog) - start = get_last_record_value_for_table(self.state, table) + start = get_last_record_value_for_table(self.state, table, self.config) if start is None: start = self.config.get('start_date') diff --git a/tap_exacttarget/endpoints/lists.py b/tap_exacttarget/endpoints/lists.py index b9f2dbf..2a3771d 100644 --- a/tap_exacttarget/endpoints/lists.py +++ b/tap_exacttarget/endpoints/lists.py @@ -55,7 +55,7 @@ def sync_data(self): selector = FuelSDK.ET_List search_filter = None - retrieve_all_since = get_last_record_value_for_table(self.state, table) + retrieve_all_since = get_last_record_value_for_table(self.state, table, self.config) if retrieve_all_since is not None: search_filter = { diff --git a/tap_exacttarget/endpoints/sends.py b/tap_exacttarget/endpoints/sends.py index 5129197..9ffcd03 100644 --- a/tap_exacttarget/endpoints/sends.py +++ b/tap_exacttarget/endpoints/sends.py @@ -94,7 +94,7 @@ def sync_data(self): selector = FuelSDK.ET_Send search_filter = None - retrieve_all_since = get_last_record_value_for_table(self.state, table) + retrieve_all_since = get_last_record_value_for_table(self.state, table, self.config) if retrieve_all_since is not None: search_filter = { diff --git a/tap_exacttarget/pagination.py b/tap_exacttarget/pagination.py index 893731f..cceb9d7 100644 --- a/tap_exacttarget/pagination.py +++ b/tap_exacttarget/pagination.py @@ -22,6 +22,15 @@ def increment_date(date_value, unit=None): return datetime.datetime.strftime(incremented_date_obj, DATE_FORMAT) +def decrement_date(date_value, unit=None): + if unit is None: + unit = {'days': 1} + + date_obj = datetime.datetime.strptime(date_value, DATE_FORMAT) + + decrement_date_obj = date_obj - datetime.timedelta(**unit) + + return datetime.datetime.strftime(decrement_date_obj, DATE_FORMAT) def get_date_page(field, start, unit): return between(field, start, increment_date(start, unit)) diff --git a/tap_exacttarget/state.py b/tap_exacttarget/state.py index 966a88a..ed8a70a 100644 --- a/tap_exacttarget/state.py +++ b/tap_exacttarget/state.py @@ -5,7 +5,7 @@ from voluptuous import Schema, Required -from tap_exacttarget.pagination import DATE_FORMAT +from tap_exacttarget.pagination import DATE_FORMAT, decrement_date LOGGER = singer.get_logger() @@ -19,7 +19,7 @@ }) -def get_last_record_value_for_table(state, table): +def get_last_record_value_for_table(state, table, config=None): raw = state.get('bookmarks', {}) \ .get(table, {}) \ .get('last_record') @@ -27,10 +27,12 @@ def get_last_record_value_for_table(state, table): if raw is None: return None - date_obj = datetime.datetime.strptime(raw, DATE_FORMAT) - date_obj = date_obj - datetime.timedelta(days=1) + interval_unit = config.get('state__bookmark_interval_unit', 'days') + interval = config.get('state__bookmark_interval', 1) - return date_obj.strftime(DATE_FORMAT) + unit = {interval_unit: int(interval)} + + return decrement_date(state, unit) def incorporate(state, table, field, value): diff --git a/tap_exacttarget/test/test_pagination.py b/tap_exacttarget/test/test_pagination.py index 9e09507..b36d09d 100644 --- a/tap_exacttarget/test/test_pagination.py +++ b/tap_exacttarget/test/test_pagination.py @@ -1,6 +1,6 @@ import unittest -from tap_exacttarget.pagination import increment_date +from tap_exacttarget.pagination import increment_date, decrement_date class TestPagination(unittest.TestCase): @@ -12,3 +12,11 @@ def test_increment_date(self): self.assertEqual( increment_date("2015-09-28T10:05:53Z", {'hours': 1}), "2015-09-28T11:05:53Z") + + def test_decrement_date(self): + self.assertEqual( + decrement_date("2015-09-28T10:05:53Z"), + "2015-09-27T10:05:53Z") + self.assertEqual( + decrement_date("2015-09-28T10:05:53Z", {'hours': 1}), + "2015-09-28T09:05:53Z") \ No newline at end of file