diff --git a/CHANGELOG.md b/CHANGELOG.md index b007509..f6a0597 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 2.5.0 + * Adds parent-tap-stream-id field to catalog for child streams [#88](https://github.com/singer-io/tap-typeform/pull/88) + ## 2.4.3 * Bump dependency versions for twistlock compliance [#85](https://github.com/singer-io/tap-typeform/pull/85) diff --git a/setup.py b/setup.py index 464707a..7f29dad 100755 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ setup( name="tap-typeform", - version="2.4.3", + version="2.5.0", description="Singer.io tap for extracting data from the TypeForm Responses API", author="bytcode.io", url="http://singer.io", diff --git a/tap_typeform/schema.py b/tap_typeform/schema.py index e014370..f22796a 100644 --- a/tap_typeform/schema.py +++ b/tap_typeform/schema.py @@ -43,6 +43,10 @@ def get_schemas(): if replication_keys and field_name in replication_keys: mdata = metadata.write(mdata, ('properties', field_name), 'inclusion', 'automatic') + # Check if the stream has any parent attribute + parent_attribute = getattr(stream_metadata, "parent", None) + if parent_attribute: + mdata = metadata.write(mdata, (), "parent-tap-stream-id", parent_attribute) mdata = metadata.to_list(mdata) field_metadata[stream_name] = mdata diff --git a/tests/base.py b/tests/base.py index 9a8b51a..e0f56f8 100644 --- a/tests/base.py +++ b/tests/base.py @@ -22,6 +22,8 @@ class TypeformBaseTest(unittest.TestCase): PRIMARY_KEYS = "table-key-properties" FOREIGN_KEYS = "table-foreign-key-properties" REPLICATION_METHOD = "forced-replication-method" + PARENT_TAP_STREAM_ID = "parent-tap-stream-id" + EXPECTED_PARENT_STREAM = "expected-parent-stream" API_LIMIT = "max-row-limit" INCREMENTAL = "INCREMENTAL" FULL_TABLE = "FULL_TABLE" @@ -86,7 +88,8 @@ def expected_metadata(self): self.PRIMARY_KEYS: {"landing_id", "question_id"}, self.REPLICATION_METHOD: self.INCREMENTAL, self.REPLICATION_KEYS: {"submitted_at"}, - self.OBEYS_START_DATE: True + self.OBEYS_START_DATE: True, + self.EXPECTED_PARENT_STREAM: "submitted_landings" }, "submitted_landings": { self.PRIMARY_KEYS: {"landing_id"}, diff --git a/tests/test_typeform_discovery.py b/tests/test_typeform_discovery.py index f76480a..e34d09a 100644 --- a/tests/test_typeform_discovery.py +++ b/tests/test_typeform_discovery.py @@ -56,6 +56,8 @@ def test_run(self): expected_replication_keys = self.expected_replication_keys()[stream] expected_automatic_fields = expected_primary_keys | expected_replication_keys expected_replication_method = self.expected_replication_method()[stream] + expected_parent_stream = self.expected_metadata().get( + stream, {}).get(self.EXPECTED_PARENT_STREAM) # collecting actual values... schema_and_metadata = menagerie.get_annotated_schema(conn_id, catalog['stream_id']) @@ -77,6 +79,9 @@ def test_run(self): item.get("breadcrumb", ["properties", None])[1] for item in metadata if item.get("metadata").get("inclusion") == "automatic" ) + # Get parent-tap-stream-id if present + actual_parent_stream_id = stream_properties[0].get( + "metadata", {}).get(self.PARENT_TAP_STREAM_ID) actual_fields = [] for md_entry in metadata: @@ -109,6 +114,17 @@ def test_run(self): self.assertEqual(expected_replication_method, actual_replication_method, msg="The actual replication method {} doesn't match the expected {}".format( actual_replication_method, expected_replication_method)) + # Verify parent-tap-stream-id for child streams + if expected_parent_stream: + self.assertEqual( + actual_parent_stream_id, expected_parent_stream, + msg=f"Parent stream mismatch for {stream}" + ) + else: + self.assertIsNone( + actual_parent_stream_id, + msg=f"Expected no parent stream for {stream}, but found {actual_parent_stream_id}" + ) # Verify that if there is a replication key we are doing INCREMENTAL otherwise FULL if expected_replication_keys: