Skip to content

Commit ad1fdf1

Browse files
author
Alberto Juan
committed
Add change_sorting_key_landing_data_source directory with initialization files
1 parent 2320ac8 commit ad1fdf1

23 files changed

+625
-0
lines changed

‎change_sorting_key_landing_data_source/.tinyenv

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
TB_VERSION_WARNING=0
2+
VERSION=0.0.0

‎change_sorting_key_landing_data_source/README.md

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
# Tinybird Versions - {{ YOUR USE CASE NAME HERE }}
2+
3+
Work in progress ...

‎change_sorting_key_landing_data_source/datasources/analytics_events.datasource

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
TOKEN "tracker" APPEND
2+
3+
DESCRIPTION >
4+
Analytics events landing data source
5+
6+
SCHEMA >
7+
`timestamp` DateTime `json:$.timestamp`,
8+
`session_id` String `json:$.session_id`,
9+
`action` LowCardinality(String) `json:$.action`,
10+
`version` LowCardinality(String) `json:$.version`,
11+
`payload` String `json:$.payload`
12+
13+
ENGINE "MergeTree"
14+
ENGINE_PARTITION_KEY "toYYYYMM(timestamp)"
15+
ENGINE_SORTING_KEY "timestamp"
16+
ENGINE_TTL "timestamp + toIntervalDay(60)"

‎change_sorting_key_landing_data_source/datasources/analytics_pages_mv.datasource

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
2+
SCHEMA >
3+
`date` Date,
4+
`device` String,
5+
`browser` String,
6+
`location` String,
7+
`pathname` String,
8+
`visits` AggregateFunction(uniq, String),
9+
`hits` AggregateFunction(count)
10+
11+
ENGINE "AggregatingMergeTree"
12+
ENGINE_PARTITION_KEY "toYYYYMM(date)"
13+
ENGINE_SORTING_KEY "date, device, browser, location, pathname"

‎change_sorting_key_landing_data_source/datasources/analytics_sessions_mv.datasource

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
2+
SCHEMA >
3+
`date` Date,
4+
`session_id` String,
5+
`device` SimpleAggregateFunction(any, String),
6+
`browser` SimpleAggregateFunction(any, String),
7+
`location` SimpleAggregateFunction(any, String),
8+
`first_hit` SimpleAggregateFunction(min, DateTime),
9+
`latest_hit` SimpleAggregateFunction(max, DateTime),
10+
`hits` AggregateFunction(count)
11+
12+
ENGINE "AggregatingMergeTree"
13+
ENGINE_PARTITION_KEY "toYYYYMM(date)"
14+
ENGINE_SORTING_KEY "date, session_id"

‎change_sorting_key_landing_data_source/datasources/analytics_sources_mv.datasource

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
2+
SCHEMA >
3+
`date` Date,
4+
`device` String,
5+
`browser` String,
6+
`location` String,
7+
`referrer` String,
8+
`visits` AggregateFunction(uniq, String),
9+
`hits` AggregateFunction(count)
10+
11+
ENGINE "AggregatingMergeTree"
12+
ENGINE_PARTITION_KEY "toYYYYMM(date)"
13+
ENGINE_SORTING_KEY "date, device, browser, location, referrer"

‎change_sorting_key_landing_data_source/datasources/fixtures/analytics_events.ndjson

+4
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
{"timestamp":"2023-10-11T07:58:10.324Z","session_id":"3cda774e-adb4-4b3b-88fa-5744ee81ea5f","action":"page_hit","version":"1","payload":"{ \"user-agent\":\"Mozilla/5.0 (Linux; Android 13) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.5249.118 Mobile Safari/537.36\", \"locale\":\"en-US\", \"location\":\"US\", \"referrer\":\"https://www.yandex.com\", \"pathname\":\"/product\", \"href\":\"https://www.tinybird.co/product\"}"}
2+
{"timestamp":"2023-10-11T07:57:36.694Z","session_id":"85c4d468-f480-4610-978d-d64cb665bf5f","action":"page_hit","version":"1","payload":"{ \"user-agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:103.0) Gecko/20100101 Firefox/103.0\", \"locale\":\"en-US\", \"location\":\"US\", \"referrer\":\"https://www.google.com\", \"pathname\":\"/\", \"href\":\"https://www.tinybird.co\"}"}
3+
{"timestamp":"2023-10-11T07:57:56.172Z","session_id":"dcc0ca3e-4f9c-435b-97e5-daa5a8c48e2a","action":"page_hit","version":"1","payload":"{ \"user-agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.102 Safari/537.36\", \"locale\":\"en-US\", \"location\":\"US\", \"referrer\":\"https://www.google.com\", \"pathname\":\"/product\", \"href\":\"https://www.tinybird.co/product\"}"}
4+
{"timestamp":"2023-10-12T07:57:56.172Z","session_id":"dcc0ca3e-4f9c-435b-97e5-daa5a8c48e2a","action":"page_hit","version":"1","payload":"{ \"user-agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.102 Safari/537.36\", \"locale\":\"en-US\", \"location\":\"US\", \"referrer\":\"https://www.google.com\", \"pathname\":\"/product\", \"href\":\"https://www.tinybird.co/product\"}"}

‎change_sorting_key_landing_data_source/pipes/analytics_hits.pipe

+59
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
DESCRIPTION >
2+
Parsed `page_hit` events, implementing `browser` and `device` detection logic.
3+
4+
5+
TOKEN "dashboard" READ
6+
7+
NODE parsed_hits
8+
DESCRIPTION >
9+
Parse raw page_hit events
10+
11+
SQL >
12+
13+
SELECT
14+
timestamp,
15+
action,
16+
version,
17+
coalesce(session_id,'0') as session_id,
18+
JSONExtractString(payload, 'locale') as locale,
19+
JSONExtractString(payload, 'location') as location,
20+
JSONExtractString(payload, 'referrer') as referrer,
21+
JSONExtractString(payload, 'pathname') as pathname,
22+
JSONExtractString(payload, 'href') as href,
23+
lower(JSONExtractString(payload, 'user-agent')) as user_agent
24+
FROM
25+
analytics_events
26+
where action = 'page_hit'
27+
28+
29+
30+
NODE endpoint
31+
SQL >
32+
33+
SELECT
34+
timestamp,
35+
action,
36+
version,
37+
session_id,
38+
location,
39+
referrer,
40+
pathname,
41+
href,
42+
case
43+
when match(user_agent, 'wget|ahrefsbot|curl|urllib|bitdiscovery|\+https://|googlebot') then 'bot'
44+
when match(user_agent, 'android') then 'mobile-android'
45+
when match(user_agent, 'ipad|iphone|ipod') then 'mobile-ios'
46+
else 'desktop'
47+
END as device,
48+
case
49+
when match(user_agent, 'firefox') then 'firefox'
50+
when match(user_agent, 'chrome|crios') then 'chrome'
51+
when match(user_agent, 'opera') then 'opera'
52+
when match(user_agent, 'msie|trident') then 'ie'
53+
when match(user_agent, 'iphone|ipad|safari') then 'safari'
54+
else 'Unknown'
55+
END as browser
56+
FROM
57+
parsed_hits
58+
59+

‎change_sorting_key_landing_data_source/pipes/analytics_pages.pipe

+26
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
NODE analytics_pages_1
2+
DESCRIPTION >
3+
Aggregate by pathname and calculate session and hits
4+
5+
SQL >
6+
7+
SELECT
8+
toDate(timestamp) AS date,
9+
device,
10+
browser,
11+
location,
12+
pathname,
13+
uniqState(session_id) AS visits,
14+
countState() AS hits
15+
FROM analytics_hits
16+
GROUP BY
17+
date,
18+
device,
19+
browser,
20+
location,
21+
pathname
22+
23+
TYPE materialized
24+
DATASOURCE analytics_pages_mv
25+
26+

‎change_sorting_key_landing_data_source/pipes/analytics_sessions.pipe

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
NODE analytics_sessions_1
2+
DESCRIPTION >
3+
Aggregate by session_id and calculate session metrics
4+
5+
SQL >
6+
7+
SELECT
8+
toDate(timestamp) AS date,
9+
session_id,
10+
anySimpleState(device) AS device,
11+
anySimpleState(browser) AS browser,
12+
anySimpleState(location) AS location,
13+
minSimpleState(timestamp) AS first_hit,
14+
maxSimpleState(timestamp) AS latest_hit,
15+
countState() AS hits
16+
FROM analytics_hits
17+
GROUP BY
18+
date,
19+
session_id
20+
21+
TYPE materialized
22+
DATASOURCE analytics_sessions_mv
23+
24+

‎change_sorting_key_landing_data_source/pipes/analytics_sources.pipe

+33
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
NODE analytics_sources_1
2+
DESCRIPTION >
3+
Aggregate by referral and calculate session and hits
4+
5+
SQL >
6+
7+
WITH (
8+
SELECT domainWithoutWWW(href)
9+
FROM analytics_hits
10+
WHERE href is not null and href != ''
11+
LIMIT 1
12+
) AS currenct_domain
13+
SELECT
14+
toDate(timestamp) AS date,
15+
device,
16+
browser,
17+
location,
18+
referrer,
19+
uniqState(session_id) AS visits,
20+
countState() AS hits
21+
FROM analytics_hits
22+
WHERE domainWithoutWWW(referrer) != currenct_domain
23+
GROUP BY
24+
date,
25+
device,
26+
browser,
27+
location,
28+
referrer
29+
30+
TYPE materialized
31+
DATASOURCE analytics_sources_mv
32+
33+

‎change_sorting_key_landing_data_source/pipes/kpis.pipe

+123
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
DESCRIPTION >
2+
Summary with general KPIs per date, including visits, page views, bounce rate and average session duration.
3+
Accepts `date_from` and `date_to` date filter, all historical data if not passed.
4+
Daily granularity, except when filtering one single day (hourly)
5+
6+
7+
TOKEN "dashboard" READ
8+
9+
NODE timeseries
10+
DESCRIPTION >
11+
Generate a timeseries for the specified time range, so we call fill empty data points.
12+
Filters "future" data points.
13+
14+
SQL >
15+
16+
%
17+
{% set _single_day = defined(date_from) and day_diff(date_from, date_to) == 0 %}
18+
with
19+
{% if defined(date_from) %}
20+
toStartOfDay(toDate({{Date(date_from, description="Starting day for filtering a date range", required=False)}})) as start,
21+
{% else %}
22+
toStartOfDay(timestampAdd(today(), interval -7 day)) as start,
23+
{% end %}
24+
{% if defined(date_to) %}
25+
toStartOfDay(toDate({{Date(date_to, description="Finishing day for filtering a date range", required=False)}})) as end
26+
{% else %}
27+
toStartOfDay(today()) as end
28+
{% end %}
29+
{% if _single_day %}
30+
select arrayJoin(arrayMap(x -> toDateTime(x), range(toUInt32(toDateTime(start)), toUInt32(timestampAdd(end, interval 1 day)), 3600))) as date
31+
{% else %}
32+
select arrayJoin(arrayMap(x -> toDate(x), range(toUInt32(start), toUInt32(timestampAdd(end, interval 1 day)), 24 * 3600))) as date
33+
{% end %}
34+
where date <= now()
35+
36+
37+
38+
NODE hits
39+
DESCRIPTION >
40+
Group by sessions and calculate metrics at that level
41+
42+
SQL >
43+
44+
%
45+
{% if defined(date_from) and day_diff(date_from, date_to) == 0 %}
46+
select
47+
toStartOfHour(timestamp) as date,
48+
session_id,
49+
uniq(session_id) as visits,
50+
count() as pageviews,
51+
case when min(timestamp) = max(timestamp) then 1 else 0 end as is_bounce,
52+
max(timestamp) as latest_hit_aux,
53+
min(timestamp) as first_hit_aux
54+
from
55+
analytics_hits
56+
where
57+
toDate(timestamp) = {{Date(date_from)}}
58+
group by
59+
toStartOfHour(timestamp), session_id
60+
{% else %}
61+
select
62+
date,
63+
session_id,
64+
uniq(session_id) as visits,
65+
countMerge(hits) as pageviews,
66+
case when min(first_hit) = max(latest_hit) then 1 else 0 end as is_bounce,
67+
max(latest_hit) as latest_hit_aux,
68+
min(first_hit) as first_hit_aux
69+
from
70+
analytics_sessions_mv
71+
where
72+
{% if defined(date_from) %}
73+
date >= {{Date(date_from)}}
74+
{% else %}
75+
date >= timestampAdd(today(), interval -7 day)
76+
{% end %}
77+
{% if defined(date_to) %}
78+
and date <= {{Date(date_to)}}
79+
{% else %}
80+
and date <= today()
81+
{% end %}
82+
group by
83+
date, session_id
84+
{% end %}
85+
86+
87+
88+
NODE data
89+
DESCRIPTION >
90+
General KPIs per date, works for both summary metrics and trends charts.
91+
92+
SQL >
93+
94+
select
95+
date,
96+
uniq(session_id) as visits,
97+
sum(pageviews) as pageviews,
98+
sum(case when latest_hit_aux = first_hit_aux then 1 end) / visits as bounce_rate,
99+
avg(latest_hit_aux - first_hit_aux) as avg_session_sec
100+
from
101+
hits
102+
group by
103+
date
104+
105+
106+
107+
NODE endpoint
108+
DESCRIPTION >
109+
Join and generate timeseries with metrics
110+
111+
SQL >
112+
113+
select
114+
a.date,
115+
b.visits,
116+
b.pageviews,
117+
b.bounce_rate,
118+
b.avg_session_sec
119+
from
120+
timeseries a
121+
left join data b using date
122+
123+

‎change_sorting_key_landing_data_source/pipes/top_browsers.pipe

+39
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
DESCRIPTION >
2+
Top Browsers ordered by most visits.
3+
Accepts `date_from` and `date_to` date filter. Defaults to last 7 days.
4+
Also `skip` and `limit` parameters for pagination.
5+
6+
7+
TOKEN "dashboard" READ
8+
9+
NODE endpoint
10+
DESCRIPTION >
11+
Group by browser and calcualte hits and visits
12+
13+
SQL >
14+
15+
%
16+
select
17+
browser,
18+
uniqMerge(visits) as visits,
19+
countMerge(hits) as hits
20+
from
21+
analytics_sources_mv
22+
where
23+
{% if defined(date_from) %}
24+
date >= {{Date(date_from, description="Starting day for filtering a date range", required=False)}}
25+
{% else %}
26+
date >= timestampAdd(today(), interval -7 day)
27+
{% end %}
28+
{% if defined(date_to) %}
29+
and date <= {{Date(date_to, description="Finishing day for filtering a date range", required=False)}}
30+
{% else %}
31+
and date <= today()
32+
{% end %}
33+
group by
34+
browser
35+
order by
36+
visits desc
37+
limit {{Int32(skip, 0)}},{{Int32(limit, 50)}}
38+
39+

0 commit comments

Comments
 (0)