Skip to content

Commit 204812e

Browse files
apentoriapentori
committed
simplecat: adding new endpoint and pagination
Signed-off-by: Alexis Pentori <[email protected]>
1 parent 36555d0 commit 204812e

File tree

5 files changed

+234
-19
lines changed

5 files changed

+234
-19
lines changed

Diff for: ‎source-simplecast-fecther/metadata.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ data:
1414
connectorSubtype: api
1515
connectorType: source
1616
definitionId: 464a7cea-0317-485e-9a9c-bcd06155bfff
17-
dockerImageTag: 1.0.0
17+
dockerImageTag: 1.1.0
1818
dockerRepository: harbor.status.im/status-im/airbyte/source-simplecast-fetcher
1919
githubIssueLabel: source-simplecast-fecther
2020
icon: simplecast-fecther.svg

Diff for: ‎source-simplecast-fecther/sample_files/configured_catalog.json

+28-1
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,34 @@
125125
},
126126
"sync_mode": "incremental",
127127
"destination_sync_mode": "overwrite"
128+
},
129+
{
130+
"stream": {
131+
"name": "analytic_podcast_v2",
132+
"json_schema": {
133+
"$schema": "http://json-schema.org/draft-04/schema#",
134+
"type": "object"
135+
},
136+
"supported_sync_modes": [
137+
"full_refresh", "incremental"
138+
]
139+
},
140+
"sync_mode": "incremental",
141+
"destination_sync_mode": "overwrite"
142+
},
143+
{
144+
"stream": {
145+
"name": "analytic_episode_v2",
146+
"json_schema": {
147+
"$schema": "http://json-schema.org/draft-04/schema#",
148+
"type": "object"
149+
},
150+
"supported_sync_modes": [
151+
"full_refresh", "incremental"
152+
]
153+
},
154+
"sync_mode": "incremental",
155+
"destination_sync_mode": "overwrite"
128156
}
129-
130157
]
131158
}

Diff for: ‎source-simplecast-fecther/source_simplecast_fecther/schemas/analytic_episode_v2.json

+60
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
{
2+
"$schema": "http://json-schema.org/draft-07/schema#",
3+
"title": "Generated schema for Root",
4+
"type": "object",
5+
"properties": {
6+
"href": {
7+
"type": ["null", "string"]
8+
},
9+
"time_of_week": {},
10+
"technology": {
11+
"type": "object",
12+
"properties": {
13+
"href": {
14+
"type": ["null", "string"]
15+
}
16+
}
17+
},
18+
"recast": {
19+
"type": ["null", "object"],
20+
"properties": {
21+
"href": {
22+
"type": ["null", "string"]
23+
}
24+
}
25+
},
26+
"mapbox": {
27+
"type": ["null", "object"],
28+
"properties": {
29+
"href": {
30+
"type": ["null", "string"]
31+
}
32+
}
33+
},
34+
"location": {
35+
"type": ["null", "object"],
36+
"properties": {
37+
"href": {
38+
"type": ["null", "string"]
39+
}
40+
}
41+
},
42+
"episodes": {},
43+
"embed": {
44+
"type": ["null", "object"],
45+
"properties": {
46+
"href": {
47+
"type": ["null", "string"]
48+
}
49+
}
50+
},
51+
"downloads": {
52+
"type": ["null", "object"],
53+
"properties": {
54+
"href": {
55+
"type": ["null", "string"]
56+
}
57+
}
58+
}
59+
}
60+
}

Diff for: ‎source-simplecast-fecther/source_simplecast_fecther/schemas/analytic_podcast_v2.json

+73
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
{
2+
"$schema": "http://json-schema.org/draft-07/schema#",
3+
"title": "Generated schema for Root",
4+
"type": "object",
5+
"properties": {
6+
"href": {
7+
"type": ["null", "string"]
8+
},
9+
"time_of_week": {
10+
"type": "object",
11+
"properties": {
12+
"href": {
13+
"type": ["null", "string"]
14+
}
15+
}
16+
},
17+
"technology": {
18+
"type": ["null", "object"],
19+
"properties": {
20+
"href": {
21+
"type": ["null", "string"]
22+
}
23+
}
24+
},
25+
"recast": {
26+
"type": ["null", "object"],
27+
"properties": {
28+
"href": {
29+
"type": ["null", "string"]
30+
}
31+
}
32+
},
33+
"mapbox": {
34+
"type": ["null", "object"],
35+
"properties": {
36+
"href": {
37+
"type": ["null", "string"]
38+
}
39+
}
40+
},
41+
"location": {
42+
"type": ["null", "object"],
43+
"properties": {
44+
"href": {
45+
"type": ["null", "string"]
46+
}
47+
}
48+
},
49+
"episodes": {
50+
"type": ["null", "object"],
51+
"properties": {
52+
"href": {
53+
"type": ["null", "string"]
54+
}
55+
}
56+
},
57+
"embed": {
58+
"type": ["null", "object"],
59+
"properties": {
60+
"href": {
61+
"type": ["null", "string"]
62+
}
63+
}
64+
},
65+
"downloads": {
66+
"type": ["null", "object"],
67+
"properties": {
68+
"href": {
69+
"type": ["null", "string"]
70+
}
71+
}
72+
}
73+
}

Diff for: ‎source-simplecast-fecther/source_simplecast_fecther/source.py

+72-17
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple
88
import logging
99
import requests
10+
import time
1011
from airbyte_cdk.sources import AbstractSource
1112
from airbyte_cdk.sources.streams import Stream
1213
from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream
@@ -22,9 +23,27 @@
2223
# Basic full refresh stream
2324
class SimplecastFectherStream(HttpStream):
2425
url_base = "https://api.simplecast.com/"
26+
primary_key = None
27+
2528

2629
def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]:
27-
return None
30+
pages = response.json().get('pages')
31+
if pages and pages.get('next'):
32+
time.sleep(2)
33+
return {
34+
'limit': pages.get('limit'),
35+
'offset': pages.get('limit')* pages.get('current')
36+
}
37+
38+
def request_params(
39+
self,
40+
stream_state: Optional[Mapping[str, Any]],
41+
stream_slice: Optional[Mapping[str, Any]] = None,
42+
next_page_token: Optional[Mapping[str, Any]] = None,
43+
) -> MutableMapping[str, Any]:
44+
if next_page_token:
45+
return next_page_token
46+
2847

2948
class Podcast(SimplecastFectherStream):
3049

@@ -55,11 +74,12 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp
5574
}
5675
yield podcast
5776

58-
class Episode(HttpSubStream, Podcast):
77+
class Episode(HttpSubStream, SimplecastFectherStream):
5978
primary_key="episode_id"
6079

61-
def __init__(self, **kwargs):
62-
super().__init__(Podcast(**kwargs), **kwargs)
80+
@property
81+
def use_cache(self) -> bool:
82+
return True
6383

6484
def path(
6585
self,
@@ -89,11 +109,11 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp
89109
}
90110
yield episode
91111

92-
class AnalyticSubStream(HttpSubStream, Podcast, ABC):
112+
class AnalyticSubStream(HttpSubStream, SimplecastFectherStream, ABC):
93113
primary_key=None
94114

95115
def __init__(self, endpoint:str, keys_dict:dict, collection_name:str, **kwargs):
96-
super().__init__(Podcast(**kwargs), **kwargs)
116+
super().__init__(**kwargs)
97117
self.endpoint=endpoint
98118
self.keys_dict=keys_dict
99119
self.collection_name = collection_name
@@ -151,7 +171,7 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp
151171
}
152172
yield analytic_episode
153173

154-
class AnalyticDownload(AnalyticSubStream, Podcast):
174+
class AnalyticDownload(AnalyticSubStream):
155175

156176
def __init__(self, **kwargs):
157177
super().__init__(endpoint="downloads", keys_dict=DOWNLOADS_KEY, collection_name="by_interval", **kwargs)
@@ -171,22 +191,57 @@ class TechnologyListeningMethod(AnalyticSubStream):
171191
def __init__(self, **kwargs):
172192
super().__init__(endpoint="technology/listening_methods", keys_dict=TECH_KEY, collection_name="collection", **kwargs)
173193

194+
class AnalyticEpisodeV2(HttpSubStream,SimplecastFectherStream):
195+
def path(
196+
self,
197+
stream_state: Mapping[str, Any] = None,
198+
stream_slice: Mapping[str, Any] = None,
199+
next_page_token: Mapping[str, Any] = None
200+
) -> str:
201+
episode_id=stream_slice.get("parent").get("id")
202+
return f"analytics?episode={episode_id}"
203+
204+
def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]:
205+
data=response.json()
206+
logger.debug("Response: %s", data)
207+
yield data
208+
209+
class AnalyticPodcastV2(HttpSubStream, SimplecastFectherStream):
210+
def path(
211+
self,
212+
stream_state: Mapping[str, Any] = None,
213+
stream_slice: Mapping[str, Any] = None,
214+
next_page_token: Mapping[str, Any] = None
215+
) -> str:
216+
podcast_id=stream_slice.get("parent").get("id")
217+
return f"analytics?podcast={podcast_id}"
218+
219+
def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]:
220+
data=response.json()
221+
logger.debug("Response: %s", data)
222+
yield data
223+
224+
174225

175226
# Source
176227
class SourceSimplecastFecther(AbstractSource):
177228
def check_connection(self, logger, config) -> Tuple[bool, any]:
178229
return True, None
179230

180231
def streams(self, config: Mapping[str, Any]) -> List[Stream]:
181-
auth = TokenAuthenticator(token=config["api_key"])
232+
auth = TokenAuthenticator(token=config["api_key"])
233+
podcasts=Podcast(authenticator=auth)
234+
episodes=Episode(authenticator=auth, parent=podcasts)
182235
return [
183-
Podcast(authenticator=auth),
184-
Episode(authenticator=auth),
185-
AnalyticLocation(authenticator=auth),
186-
AnalyticTimeOfWeek(authenticator=auth),
187-
AnalyticEpisode(authenticator=auth),
188-
AnalyticDownload(authenticator=auth),
189-
TechnologyApplication(authenticator=auth),
190-
TechnologyDeviceClass(authenticator=auth),
191-
TechnologyListeningMethod(authenticator=auth)
236+
podcasts,
237+
episodes,
238+
AnalyticLocation(authenticator=auth, parent=podcasts),
239+
AnalyticTimeOfWeek(authenticator=auth, parent=podcasts),
240+
AnalyticEpisode(authenticator=auth, parent=podcasts),
241+
AnalyticDownload(authenticator=auth,parent=podcasts),
242+
TechnologyApplication(authenticator=auth, parent=podcasts),
243+
TechnologyDeviceClass(authenticator=auth, parent=podcasts),
244+
TechnologyListeningMethod(authenticator=auth, parent=podcasts),
245+
AnalyticEpisodeV2(authenticator=auth, parent=episodes),
246+
AnalyticPodcastV2(authenticator=auth, parent=podcasts)
192247
]

0 commit comments

Comments
 (0)