Merge pull request #108 from fivetran/bug/issue-type

bug/issue-type

Author: fivetran-catfritz

CHANGELOG.md

@@ -1,3 +1,9 @@
+# dbt_jira v0.15.0
+[PR #108](https://github.com/fivetran/dbt_jira/pull/108) contains the following updates:
+## 🚨 Breaking Changes 🚨
+- Updated the `jira__daily_issue_field_history` model to make sure `issue_type` values are correctly joined into the downstream issue models. This applied only if `issue type` is leveraged within the `issue_field_history_columns` variable.
+>**Note**: Please be aware that a `dbt run --full-refresh` will be required after upgrading to this version in order to capture the updates.
+
 # dbt_jira v0.14.0 
 ## 🚨 Breaking Changes 🚨
 - Fixed the `jira__daily_issue_field_history` model to make sure `component` values are correctly joined into the downstream issue models. This applied only if `components` are leveraged within the `issue_field_history_columns` variable. ([PR #99](https://github.com/fivetran/dbt_jira/pull/99))

README.md

@@ -56,7 +56,7 @@ Include the following jira package version in your `packages.yml` file:
 ```yaml
 packages:
   - package: fivetran/jira
-    version: [">=0.14.0", "<0.15.0"]
+    version: [">=0.15.0", "<0.16.0"]
 
 ```
 ## Step 3: Define database and schema variables

dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'jira'
-version: '0.14.0'
+version: '0.15.0'
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 vars:

docs/catalog.json

@@ -1,5 +1,5 @@
 name: 'jira_integration_tests'
-version: '0.14.0'
+version: '0.15.0'
 config-version: 2
 profile: 'integration_tests'
 

docs/manifest.json

@@ -27,7 +27,6 @@ with pivoted_daily_history as (
 -- those values into the future.
 
 ), most_recent_data as ( 
- 
     select 
         *
     from {{ this }}
@@ -47,6 +46,11 @@ statuses as (
     from {{ var('status') }}
 ),
 
+issue_types as (
+    
+    select *
+    from {{ var('issue_type') }}
+),
 
 {% if var('jira_using_components', True) %}
 components as (
@@ -74,18 +78,25 @@ joined as (
         calendar.issue_id
 
         {% if is_incremental() %}    
-            {% for col in pivot_data_columns if col.name|lower == 'components' and var('jira_using_components', True) %}
-            , coalesce(pivoted_daily_history.components, most_recent_data.components) as components
-            {% endfor %} 
-            {% for col in pivot_data_columns if col.name|lower not in ['issue_day_id', 'issue_id', 'valid_starting_on', 'components'] %} 
-            , coalesce(pivoted_daily_history.{{ col.name }}, most_recent_data.{{ col.name }}) as {{ col.name }}
+            {% for col in pivot_data_columns %}
+                {% if col.name|lower == 'components' and var('jira_using_components', True) %}
+                , coalesce(pivoted_daily_history.components, most_recent_data.components) as components
+
+                {% elif col.name|lower not in ['issue_day_id', 'issue_id', 'valid_starting_on', 'components'] %} 
+                , coalesce(pivoted_daily_history.{{ col.name }}, most_recent_data.{{ col.name }}) as {{ col.name }}
+
+                {% endif %}
             {% endfor %} 
+
         {% else %}
-            {% for col in pivot_data_columns if col.name|lower == 'components' and var('jira_using_components', True) %}
-            , pivoted_daily_history.components   
-            {% endfor %} 
-            {% for col in pivot_data_columns if col.name|lower not in ['issue_day_id','issue_id','valid_starting_on','components'] %} 
-            , {{ col.name }}
+            {% for col in pivot_data_columns %}
+                {% if col.name|lower == 'components' and var('jira_using_components', True) %}
+                , pivoted_daily_history.components   
+
+                {% elif col.name|lower not in ['issue_day_id', 'issue_id', 'valid_starting_on', 'components'] %} 
+                , pivoted_daily_history.{{ col.name }}
+
+                {% endif %}
             {% endfor %} 
         {% endif %}
 
@@ -102,36 +113,50 @@ joined as (
 ),
 
 set_values as (
-
     select
         date_day,
         issue_id,
         joined.status_id,
         sum( case when joined.status_id is null then 0 else 1 end) over ( partition by issue_id
             order by date_day rows unbounded preceding) as status_id_field_partition
 
-        {% for col in pivot_data_columns if col.name|lower == 'components' and var('jira_using_components', True) %}
-        , coalesce(components.component_name, joined.components) as components
-        , sum(case when joined.components is null then 0 else 1 end) over (partition by issue_id order by date_day rows unbounded preceding) as component_field_partition
-        {% endfor %}
+        -- list of exception columns
+        {% set exception_cols = ['issue_id', 'issue_day_id', 'valid_starting_on', 'status', 'status_id', 'components', 'issue_type'] %}
 
-        {% for col in pivot_data_columns if col.name|lower not in ['issue_id', 'issue_day_id', 'valid_starting_on', 'status', 'status_id', 'components'] %}
-        , coalesce(field_option_{{ col.name }}.field_option_name, {{ col.name }}) as {{ col.name }}
-        -- create a batch/partition once a new value is provided
-        , sum( case when {{ col.name }} is null then 0 else 1 end) over ( partition by issue_id
-            order by date_day rows unbounded preceding) as {{ col.name }}_field_partition
+        {% for col in pivot_data_columns %}
+            {% if col.name|lower == 'components' and var('jira_using_components', True) %}
+            , coalesce(components.component_name, joined.components) as components
+            , sum(case when joined.components is null then 0 else 1 end) over (partition by issue_id order by date_day rows unbounded preceding) as component_field_partition
+
+            {% elif col.name|lower == 'issue_type' %}
+            , coalesce(issue_types.issue_type_name, joined.issue_type) as issue_type
+            , sum(case when joined.issue_type is null then 0 else 1 end) over (partition by issue_id order by date_day rows unbounded preceding) as issue_type_field_partition
+
+            {% elif col.name|lower not in exception_cols %}
+            , coalesce(field_option_{{ col.name }}.field_option_name, joined.{{ col.name }}) as {{ col.name }}
+            -- create a batch/partition once a new value is provided
+            , sum( case when joined.{{ col.name }} is null then 0 else 1 end) over ( partition by issue_id
+                order by date_day rows unbounded preceding) as {{ col.name }}_field_partition
+
+            {% endif %}
         {% endfor %}
 
     from joined
 
-    {% for col in pivot_data_columns if col.name|lower == 'components' and var('jira_using_components', True) %}
-    left join components   
-        on cast(components.component_id as {{ dbt.type_string() }}) = joined.components
-    {% endfor %}
+    {% for col in pivot_data_columns %}
+        {% if col.name|lower == 'components' and var('jira_using_components', True) %}
+        left join components   
+            on cast(components.component_id as {{ dbt.type_string() }}) = joined.components
+        
+        {% elif col.name|lower == 'issue_type' %}
+        left join issue_types   
+            on cast(issue_types.issue_type_id as {{ dbt.type_string() }}) = joined.issue_type
+
+        {% elif col.name|lower not in exception_cols %}
+        left join field_option as field_option_{{ col.name }}
+            on cast(field_option_{{ col.name }}.field_id as {{ dbt.type_string() }}) = joined.{{ col.name }}
 
-    {% for col in pivot_data_columns if col.name|lower not in ['issue_id', 'issue_day_id', 'valid_starting_on', 'status', 'status_id', 'components'] %}
-    left join field_option as field_option_{{ col.name }}
-        on cast(field_option_{{ col.name }}.field_id as {{ dbt.type_string() }}) = {{ col.name }}
+        {% endif %}
     {% endfor %}
 ),
 
@@ -144,15 +169,19 @@ fill_values as (
             partition by issue_id, status_id_field_partition 
             order by date_day asc rows between unbounded preceding and current row) as status_id
 
-        {% for col in pivot_data_columns if col.name|lower == 'components' and var('jira_using_components', True) %}
-        , first_value(components) over (partition by issue_id, component_field_partition order by date_day asc rows between unbounded preceding and current row) as components
-        {% endfor %}    
+        {% for col in pivot_data_columns %}
+            {% if col.name|lower == 'components' and var('jira_using_components', True) %}
+            , first_value(components) over (
+                partition by issue_id, component_field_partition 
+                order by date_day asc rows between unbounded preceding and current row) as components
 
-        {% for col in pivot_data_columns if col.name|lower not in ['issue_id', 'issue_day_id', 'valid_starting_on', 'status', 'status_id', 'components'] %}
-        -- grab the value that started this batch/partition
-        , first_value( {{ col.name }} ) over (
-            partition by issue_id, {{ col.name }}_field_partition 
-            order by date_day asc rows between unbounded preceding and current row) as {{ col.name }}
+            {% elif col.name|lower not in ['issue_id', 'issue_day_id', 'valid_starting_on', 'status', 'status_id', 'components'] %}
+            -- grab the value that started this batch/partition
+            , first_value( {{ col.name }} ) over (
+                partition by issue_id, {{ col.name }}_field_partition 
+                order by date_day asc rows between unbounded preceding and current row) as {{ col.name }}
+
+            {% endif %}
         {% endfor %}
 
     from set_values
@@ -164,18 +193,17 @@ fix_null_values as (
         date_day,
         issue_id
 
-        {% for col in pivot_data_columns if col.name|lower == 'components' and var('jira_using_components', True) %}
-        , case when components = 'is_null' then null else components end as components
-        {% endfor %}
+        {% for col in pivot_data_columns %}
+            {% if col.name|lower == 'components' and var('jira_using_components', True) %}
+            , case when components = 'is_null' then null else components end as components
 
-        {% for col in pivot_data_columns if col.name|lower not in ['issue_id','issue_day_id','valid_starting_on', 'status', 'components'] %} 
+            {% elif col.name|lower not in ['issue_id','issue_day_id','valid_starting_on', 'status', 'components'] %}
+            -- we de-nulled the true null values earlier in order to differentiate them from nulls that just needed to be backfilled
+            , case when {{ col.name }} = 'is_null' then null else {{ col.name }} end as {{ col.name }}
 
-        -- we de-nulled the true null values earlier in order to differentiate them from nulls that just needed to be backfilled
-        , case when {{ col.name }} = 'is_null' then null else {{ col.name }} end as {{ col.name }}
+            {% endif %}
         {% endfor %}
 
-
-
     from fill_values
 
 ),
@@ -187,12 +215,14 @@ surrogate_key as (
         fix_null_values.issue_id,
         statuses.status_name as status
 
-        {% for col in pivot_data_columns if col.name|lower == 'components' and var('jira_using_components', True) %}
-        , case when components = 'is_null' then null else components end as components
-        {% endfor %}
+        {% for col in pivot_data_columns %}
+            {% if col.name|lower == 'components' and var('jira_using_components', True) %}
+            , fix_null_values.components as components
+
+            {% elif col.name|lower not in ['issue_id','issue_day_id','valid_starting_on', 'status', 'components'] %} 
+            , fix_null_values.{{ col.name }} as {{ col.name }}
 
-        {% for col in pivot_data_columns if col.name|lower not in ['issue_id','issue_day_id','valid_starting_on', 'status', 'components'] %} 
-        , fix_null_values.{{ col.name }} as {{ col.name }}
+            {% endif %}
         {% endfor %}
 
         , {{ dbt_utils.generate_surrogate_key(['date_day','issue_id']) }} as issue_day_id