Skip to content

Commit 21ad4ac

Browse files
newgnartnewgnart
authored
Merge pull request #6 from newgnart/feature/ci
pr_ci ran successfully
2 parents 63149c3 + 389aba9 commit 21ad4ac

File tree

14 files changed

+422
-103
lines changed

14 files changed

+422
-103
lines changed

‎.github/workflows/pr_ci.yml‎

Lines changed: 11 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -43,52 +43,49 @@ jobs:
4343
4444
- name: Run dbt debug
4545
run: |
46-
uv run dbt debug --target ci --project-dir dbt_project
46+
cd dbt_project && uv run dbt debug --target test
4747
env:
4848
SNOWFLAKE_ACCOUNT: ${{ vars.SNOWFLAKE_ACCOUNT }}
4949
SNOWFLAKE_USER: ${{ vars.SNOWFLAKE_USER }}
5050
SNOWFLAKE_DATABASE: ${{ vars.SNOWFLAKE_DATABASE }}
5151
SNOWFLAKE_WAREHOUSE: ${{ vars.SNOWFLAKE_WAREHOUSE }}
5252
SNOWFLAKE_ROLE: ${{ vars.SNOWFLAKE_ROLE }}
5353
SNOWFLAKE_SCHEMA: ${{ vars.SNOWFLAKE_SCHEMA }}
54-
SNOWFLAKE_PRIVATE_KEY_FILE: ${{ secrets.SNOWFLAKE_PRIVATE_KEY_FILE }}
55-
SNOWFLAKE_PRIVATE_KEY_FILE_PWD: ${{ secrets.SNOWFLAKE_PRIVATE_KEY_FILE_PWD }}
54+
SNOWFLAKE_PRIVATE_KEY: ${{ secrets.SNOWFLAKE_PRIVATE_KEY }}
5655

57-
- name: Run SQLFluff lint
56+
- name: Run dbt deps
5857
run: |
59-
uv run sqlfluff lint dbt_project/
58+
cd dbt_project && uv run dbt deps --target test
6059
env:
6160
SNOWFLAKE_ACCOUNT: ${{ vars.SNOWFLAKE_ACCOUNT }}
6261
SNOWFLAKE_USER: ${{ vars.SNOWFLAKE_USER }}
6362
SNOWFLAKE_DATABASE: ${{ vars.SNOWFLAKE_DATABASE }}
6463
SNOWFLAKE_WAREHOUSE: ${{ vars.SNOWFLAKE_WAREHOUSE }}
6564
SNOWFLAKE_ROLE: ${{ vars.SNOWFLAKE_ROLE }}
6665
SNOWFLAKE_SCHEMA: ${{ vars.SNOWFLAKE_SCHEMA }}
67-
SNOWFLAKE_PRIVATE_KEY_FILE: ${{ secrets.SNOWFLAKE_PRIVATE_KEY_FILE }}
68-
SNOWFLAKE_PRIVATE_KEY_FILE_PWD: ${{ secrets.SNOWFLAKE_PRIVATE_KEY_FILE_PWD }}
66+
SNOWFLAKE_PRIVATE_KEY: ${{ secrets.SNOWFLAKE_PRIVATE_KEY }}
6967

70-
- name: Run dbt deps
68+
- name: Run SQLFluff lint
69+
# if: contains(github.event.head_commit.modified, 'dbt_project/models/*.sql')
7170
run: |
72-
uv run dbt deps --target ci --project-dir dbt_project
71+
cd dbt_project && uv run sqlfluff lint models/
7372
env:
7473
SNOWFLAKE_ACCOUNT: ${{ vars.SNOWFLAKE_ACCOUNT }}
7574
SNOWFLAKE_USER: ${{ vars.SNOWFLAKE_USER }}
7675
SNOWFLAKE_DATABASE: ${{ vars.SNOWFLAKE_DATABASE }}
7776
SNOWFLAKE_WAREHOUSE: ${{ vars.SNOWFLAKE_WAREHOUSE }}
7877
SNOWFLAKE_ROLE: ${{ vars.SNOWFLAKE_ROLE }}
7978
SNOWFLAKE_SCHEMA: ${{ vars.SNOWFLAKE_SCHEMA }}
80-
SNOWFLAKE_PRIVATE_KEY_FILE: ${{ secrets.SNOWFLAKE_PRIVATE_KEY_FILE }}
81-
SNOWFLAKE_PRIVATE_KEY_FILE_PWD: ${{ secrets.SNOWFLAKE_PRIVATE_KEY_FILE_PWD }}
79+
SNOWFLAKE_PRIVATE_KEY: ${{ secrets.SNOWFLAKE_PRIVATE_KEY }}
8280

8381
- name: Run dbt tests
8482
run: |
85-
uv run dbt test --target ci --project-dir dbt_project
83+
cd dbt_project && uv run dbt test --target test
8684
env:
8785
SNOWFLAKE_ACCOUNT: ${{ vars.SNOWFLAKE_ACCOUNT }}
8886
SNOWFLAKE_USER: ${{ vars.SNOWFLAKE_USER }}
8987
SNOWFLAKE_DATABASE: ${{ vars.SNOWFLAKE_DATABASE }}
9088
SNOWFLAKE_WAREHOUSE: ${{ vars.SNOWFLAKE_WAREHOUSE }}
9189
SNOWFLAKE_ROLE: ${{ vars.SNOWFLAKE_ROLE }}
9290
SNOWFLAKE_SCHEMA: ${{ vars.SNOWFLAKE_SCHEMA }}
93-
SNOWFLAKE_PRIVATE_KEY_FILE: ${{ secrets.SNOWFLAKE_PRIVATE_KEY_FILE }}
94-
SNOWFLAKE_PRIVATE_KEY_FILE_PWD: ${{ secrets.SNOWFLAKE_PRIVATE_KEY_FILE_PWD }}
91+
SNOWFLAKE_PRIVATE_KEY: ${{ secrets.SNOWFLAKE_PRIVATE_KEY }}

‎.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,3 +36,5 @@ TODO.md
3636
target/
3737
.user.yml
3838
dbt_packages/
39+
40+
bin/

‎dbt_project/.sqlfluff‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
[sqlfluff]
2+
# Database dialect - Snowflake
3+
dialect = snowflake
4+
5+
# Templater for dbt projects
6+
templater = dbt
7+
8+
# General settings
9+
runaway_limit = 10
10+
max_line_length = 200
11+
indent_unit = space
12+
13+
# Exclude specific rules if needed
14+
exclude_rules = RF02, AM06, RF04
15+
16+
# Indentation settings
17+
[sqlfluff:indentation]
18+
tab_space_size = 4
19+
20+
# Layout and formatting
21+
[sqlfluff:layout:type:comma]
22+
spacing_before = touch
23+
line_position = trailing
24+
25+
# Capitalization rules
26+
[sqlfluff:rules:capitalisation.keywords]
27+
capitalisation_policy = upper
28+
29+
[sqlfluff:rules:capitalisation.identifiers]
30+
extended_capitalisation_policy = lower
31+
32+
[sqlfluff:rules:capitalisation.functions]
33+
capitalisation_policy = upper
34+
35+
[sqlfluff:rules:capitalisation.literals]
36+
capitalisation_policy = consistent
37+
38+
# Aliasing rules
39+
[sqlfluff:rules:aliasing.table]
40+
aliasing = explicit
41+
42+
[sqlfluff:rules:aliasing.column]
43+
aliasing = explicit
44+
45+
[sqlfluff:rules:aliasing.expression]
46+
allow_scalar = False
47+
48+
# Column reference rules
49+
[sqlfluff:rules:ambiguous.column_references]
50+
group_by_and_order_by_style = implicit
51+
52+
# dbt templater configuration
53+
[sqlfluff:templater:dbt]
54+
project_dir = .
55+
profiles_dir = .
56+
profile = stables_analytics
57+
target = test
58+
# Skip dbt compilation and use Jinja templater instead
59+
skip_compilation = True
60+
61+
# Enable dbt builtins for Jinja templater
62+
[sqlfluff:templater:jinja]
63+
apply_dbt_builtins = True

‎dbt_project/.sqlfluffignore‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
# dbt specific directories
2+
dbt_project/target/
3+
dbt_project/dbt_packages/
4+
dbt_project/logs/
5+
6+
# Non-dbt SQL files (if any)
7+
scripts/sql/ad_hoc.sql
8+
9+
# Python environments
10+
.venv/
11+
venv/
12+
13+
# Generated files
14+
*.pyc
15+
__pycache__/
16+
17+
# Data directories
18+
.data/
19+
sampledata/

‎dbt_project/models/01_staging/stg_stablecoin.sql‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,17 @@
1-
with source as (
2-
select * from {{ source('raw_data', 'raw_stablecoin') }}
1+
WITH source AS (
2+
SELECT * FROM {{ source('raw_data', 'raw_stablecoin') }}
33
),
44

5-
casted as (
6-
select
7-
contract_address::varchar(42) as contract_address,
8-
chain::varchar(20) as chain,
9-
symbol::varchar(20) as symbol,
10-
name::varchar(100) as name,
11-
currency::varchar(10) as currency,
12-
backing_type::varchar(20) as backing_type,
13-
decimals::integer as decimals
14-
from source
5+
casted AS (
6+
SELECT
7+
contract_address::VARCHAR(42) AS contract_address,
8+
chain::VARCHAR(20) AS chain,
9+
symbol::VARCHAR(20) AS symbol,
10+
name::VARCHAR(100) AS name,
11+
currency::VARCHAR(10) AS currency,
12+
backing_type::VARCHAR(20) AS backing_type,
13+
decimals::INTEGER AS decimals
14+
FROM source
1515
)
1616

17-
select * from casted
17+
SELECT * FROM casted

‎dbt_project/models/01_staging/stg_transfer.sql‎

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,28 +1,28 @@
1-
with source as (
2-
select * from {{ source('raw_data', 'raw_transfer') }}
1+
WITH source AS (
2+
SELECT * FROM {{ source('raw_data', 'raw_transfer') }}
33
),
44

5-
casted as (
6-
select
5+
casted AS (
6+
SELECT
77
id,
8-
block_number::bigint as block_number,
8+
block_number::BIGINT AS block_number,
99
{% if target.type == 'postgres' %}
10-
to_timestamp(timestamp::bigint) AT TIME ZONE 'UTC' as block_timestamp,
10+
TO_TIMESTAMP(timestamp::BIGINT) AT TIME ZONE 'UTC' AS block_timestamp,
1111
{% else %}
12-
to_timestamp(timestamp::bigint) as block_timestamp,
12+
TO_TIMESTAMP(timestamp::BIGINT) AS block_timestamp,
1313
{% endif %}
14-
contract_address::varchar(42) as contract_address,
14+
contract_address::VARCHAR(42) AS contract_address,
1515
{% if target.type == 'postgres' %}
16-
"from"::varchar(42) as from_address,
17-
"to"::varchar(42) as to_address,
16+
"from"::VARCHAR(42) AS from_address,
17+
"to"::VARCHAR(42) AS to_address,
1818
{% else %}
19-
"FROM"::varchar(42) as from_address,
20-
"TO"::varchar(42) as to_address,
19+
"FROM"::VARCHAR(42) AS from_address,
20+
"TO"::VARCHAR(42) AS to_address,
2121
{% endif %}
22-
value::numeric(38, 0) as amount_raw,
22+
value::NUMERIC(38, 0) AS amount_raw,
2323
_dlt_load_id,
2424
_dlt_id
25-
from source
25+
FROM source
2626
)
2727

28-
select * from casted
28+
SELECT * FROM casted

‎dbt_project/models/03_mart/dim_stablecoin.sql‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
11
-- SCD Type 2 dimension built on dbt snapshot
22
-- This model adds custom SCD2 column names while leveraging dbt's snapshot functionality
33

4-
with snap_stablecoin as (
5-
select * from {{ ref('snap_stablecoin') }}
4+
WITH snap_stablecoin AS (
5+
SELECT * FROM {{ ref('snap_stablecoin') }}
66
),
77

8-
final as (
9-
select
8+
final AS (
9+
SELECT
1010
-- Business keys
1111
contract_address,
1212
chain,
@@ -19,12 +19,12 @@ final as (
1919
decimals,
2020

2121
-- SCD Type 2 columns (mapped from dbt snapshot columns)
22-
dbt_valid_from as valid_from,
23-
dbt_valid_to as valid_to,
24-
case when dbt_valid_to is null then true else false end as is_current,
25-
dbt_updated_at as created_at
22+
dbt_valid_from AS valid_from,
23+
dbt_valid_to AS valid_to,
24+
dbt_updated_at AS created_at,
25+
COALESCE(dbt_valid_to IS NULL, FALSE) AS is_current
2626

27-
from snap_stablecoin
27+
FROM snap_stablecoin
2828
)
2929

30-
select * from final
30+
SELECT * FROM final

‎dbt_project/models/03_mart/fct_transfer.sql‎

Lines changed: 38 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -7,43 +7,43 @@
77
)
88
}}
99

10-
with stg_transfer as (
11-
select * from {{ ref('stg_transfer') }}
10+
WITH stg_transfer AS (
11+
SELECT * FROM {{ ref('stg_transfer') }}
1212
{% if is_incremental() %}
1313
-- Only process new blocks since last run
14-
where block_number > (select coalesce(max(block_number), 0) from {{ this }})
14+
WHERE block_number > (SELECT COALESCE(MAX(block_number), 0) AS max_block FROM {{ this }})
1515
{% endif %}
1616
),
1717

18-
dim_stablecoin as (
19-
select * from {{ ref('dim_stablecoin') }}
20-
where is_current = true -- Only use current stablecoin metadata
18+
dim_stablecoin AS (
19+
SELECT * FROM {{ ref('dim_stablecoin') }}
20+
WHERE is_current = true -- Only use current stablecoin metadata
2121
),
2222

23-
parsed as (
24-
select
23+
parsed AS (
24+
SELECT
2525
-- Parse natural key from id (format: "0xtxhash_logindex")
26-
split_part(id, '_', 1) as transaction_hash,
27-
split_part(id, '_', 2)::integer as log_index,
26+
SPLIT_PART(id, '_', 2)::INTEGER AS log_index,
27+
TO_CHAR(block_timestamp, 'YYYYMMDD')::INTEGER AS date_key,
2828

2929
-- Time dimension
30-
to_char(block_timestamp, 'YYYYMMDD')::integer as date_key,
3130
block_number,
3231
block_timestamp,
32+
contract_address,
3333

3434
-- Contract/token dimension
35-
contract_address,
36-
'ethereum' as chain, -- TODO: get chain from raw data when available
35+
'ethereum' AS chain,
36+
from_address, -- TODO: get chain from raw data when available
3737

3838
-- Address dimensions
39-
from_address,
40-
to_address
39+
to_address,
40+
SPLIT_PART(id, '_', 1) AS transaction_hash
4141

42-
from stg_transfer
42+
FROM stg_transfer
4343
),
4444

45-
enriched as (
46-
select
45+
enriched AS (
46+
SELECT
4747
-- Keys
4848
p.transaction_hash,
4949
p.log_index,
@@ -64,29 +64,31 @@ enriched as (
6464
-- Join stablecoin metadata
6565
d.symbol,
6666
d.name,
67-
coalesce(d.decimals, 18) as decimals,
67+
COALESCE(d.decimals, 18) AS decimals,
6868

69-
-- Business enrichment: Determine transaction type
70-
case
71-
when p.from_address = '0x0000000000000000000000000000000000000000' then 'mint'
72-
when p.to_address = '0x0000000000000000000000000000000000000000' then 'burn'
73-
else 'transfer'
74-
end as transaction_type,
69+
-- Determine transaction type
70+
CASE
71+
WHEN p.from_address = '0x0000000000000000000000000000000000000000' THEN 'mint'
72+
WHEN p.to_address = '0x0000000000000000000000000000000000000000' THEN 'burn'
73+
ELSE 'transfer'
74+
END AS transaction_type,
7575

7676
-- Convert to decimal amount using actual decimals from dim_stablecoin
7777
-- For stablecoins, amount ≈ USD value. TODO: have dim_price table
78-
{{ convert_token_amount('s.amount_raw', 'coalesce(d.decimals, 18)', 2) }} as amount
79-
80-
from parsed p
81-
left join stg_transfer s
82-
on p.transaction_hash = split_part(s.id, '_', 1)
83-
and p.log_index = split_part(s.id, '_', 2)::integer
84-
left join dim_stablecoin d
85-
on lower(p.contract_address) = lower(d.contract_address)
86-
and p.chain = d.chain
78+
{{ convert_token_amount('s.amount_raw', 'COALESCE(d.decimals, 18)', 2) }} AS amount
79+
80+
FROM parsed AS p
81+
LEFT JOIN stg_transfer AS s
82+
ON
83+
p.transaction_hash = SPLIT_PART(s.id, '_', 1)
84+
AND p.log_index = SPLIT_PART(s.id, '_', 2)::INTEGER
85+
LEFT JOIN dim_stablecoin AS d
86+
ON
87+
LOWER(p.contract_address) = LOWER(d.contract_address)
88+
AND p.chain = d.chain
8789
)
8890

89-
select
91+
SELECT
9092
transaction_hash,
9193
log_index,
9294
date_key,
@@ -101,4 +103,4 @@ select
101103
decimals,
102104
transaction_type,
103105
amount
104-
from enriched
106+
FROM enriched

0 commit comments

Comments
 (0)