macro get_columns_in_relation to respect the temp metadata generation schema config

README.md

@@ -849,12 +849,12 @@ sources:
               data_type: CHAR(1)
 ```
 
-## Fallback Schema
+## temporary_metadata_generation_schema (earlier fallback_schema)
 dbt-teradata internally created temporary tables to fetch the metadata of views for manifest and catalog creation. 
-In case if user does not have permission to create tables on the schema they are working on, they can define a fallback_schema(to which they have proper create/drop privileges) in dbt_project.yml as variable.
+In case if user does not have permission to create tables on the schema they are working on, they can define a temporary_metadata_generation_schema(to which they have proper create and drop privileges) in dbt_project.yml as variable.
 ```yaml
      vars:
-        fallback_schema: <schema-name>
+        temporary_metadata_generation_schema: <schema-name>
    ```
 
 ## Credits

dbt/include/teradata/macros/adapters.sql

@@ -120,6 +120,9 @@
 {% macro teradata__get_columns_in_relation(relation) -%}
     {% set use_qvci = var("use_qvci", False) | as_bool %}
     {{ log("use_qvci set to : " ~ use_qvci) }}
+    {% set temporary_metadata_generation_schema = var("temporary_metadata_generation_schema", null) %}
+    {{ log("temporary_metadata_generation_schema set to : " ~ temporary_metadata_generation_schema) }}
+
     {% if use_qvci == False -%}
         {% call statement('check_table_or_view', fetch_result=True) %}
             SELECT TableKind FROM DBC.TablesV WHERE DatabaseName = '{{ relation.schema }}' AND TableName = '{{ relation.identifier }}'
@@ -128,13 +131,26 @@
         {% set table_kind = load_result('check_table_or_view').table.columns['TableKind'].values()[0] | trim %}
 
         {%- if table_kind == 'V' -%}
-            {% set temp_relation_for_view = relation.identifier ~ '_tmp_viw_tbl' %}
+
+            {% set timestamp = teradata_get_current_timestamp() %}
+            {% set rand = range(1, 100000) | random %}
+            {% set uuid = timestamp.replace(":", "").replace("-", "").replace(" ","").replace("+","").replace(".","") ~ rand %}
+
+            {% set temp_relation_for_view = relation.identifier ~ '_tmp_viw_tbl_' ~ uuid %}
+
+            {% if temporary_metadata_generation_schema==null %}
+              {% set schema_name = relation.schema %}
+            {% else %}
+              {% set schema_name = temporary_metadata_generation_schema %}
+            {% endif %}
+
             {% call statement('drop_existing_table', fetch_result=False) %}
-                DROP table /*+ IF EXISTS */ "{{ relation.schema }}"."{{ temp_relation_for_view }}";
+                DROP table /*+ IF EXISTS */ "{{ schema_name }}"."{{ temp_relation_for_view }}";
             {% endcall %}
             load_result('drop_existing_table')
+
             {% call statement('creating_table_from_view', fetch_result=False) %}
-                CREATE TABLE "{{ relation.schema }}"."{{ temp_relation_for_view }}" AS (SELECT * FROM "{{ relation.schema }}"."{{ relation.identifier }}") WITH NO DATA;
+                CREATE TABLE "{{ schema_name }}"."{{ temp_relation_for_view }}" AS (SELECT * FROM "{{ relation.schema }}"."{{ relation.identifier }}") WITH NO DATA;
             {% endcall %}
             load_result('creating_table_from_view')
         {% endif %}
@@ -230,15 +246,17 @@
       AND ColumnsV.TableName = TablesV.TableName
     WHERE
       TablesV.TableKind IN ('T', 'V', 'O')
-      AND ColumnsV.DatabaseName = '{{ relation.schema }}' (NOT CASESPECIFIC)
     {% if use_qvci == False -%}
         {%- if table_kind == 'V' -%}
+            AND ColumnsV.DatabaseName = '{{ schema_name }}' (NOT CASESPECIFIC)
             AND ColumnsV.TableName = '{{ temp_relation_for_view }}' (NOT CASESPECIFIC)
         {%- else -%}
+            AND ColumnsV.DatabaseName = '{{ relation.schema }}' (NOT CASESPECIFIC)
             AND ColumnsV.TableName = '{{ relation.identifier }}' (NOT CASESPECIFIC)
         {%- endif -%}
 
     {%- else -%}
+        AND ColumnsV.DatabaseName = '{{ relation.schema }}' (NOT CASESPECIFIC)
         AND ColumnsV.TableName = '{{ relation.identifier }}' (NOT CASESPECIFIC)
     {%- endif -%}
 
@@ -251,7 +269,7 @@
     {% if use_qvci == False -%}
         {%- if table_kind == 'V' -%}
             {% call statement('drop_table_from_view', fetch_result=False) %}
-                DROP table /*+ IF EXISTS */ "{{ relation.schema }}"."{{ temp_relation_for_view }}";
+                DROP table /*+ IF EXISTS */ "{{ schema_name }}"."{{ temp_relation_for_view }}";
             {% endcall %}
             load_result('drop_table_from_view')
         {% endif %}

dbt/include/teradata/macros/catalog.sql

@@ -23,8 +23,8 @@
 
 {% macro teradata__create_tmp_tables_of_views(view_relations) -%}
 
-    {% set fallback_schema = var("fallback_schema", null) %}
-    {{ log("fallback_schema set to : " ~ fallback_schema) }}
+    {% set temporary_metadata_generation_schema = var("temporary_metadata_generation_schema", null) %}
+    {{ log("temporary_metadata_generation_schema set to : " ~ temporary_metadata_generation_schema) }}
 
     {% set view_tmp_tables_mapping = {} %}
     {%- for relation in view_relations -%}
@@ -37,10 +37,10 @@
 
 
     {%- for relation, temp_relation_for_view in view_tmp_tables_mapping.items() %}
-        {% if fallback_schema==null %}
+        {% if temporary_metadata_generation_schema==null %}
           {% set schema_name = relation.schema %}
         {% else %}
-          {% set schema_name = fallback_schema %}
+          {% set schema_name = temporary_metadata_generation_schema %}
         {% endif %}
 
         {{ teradata__drop_tmp_tables_of_views(schema_name, temp_relation_for_view) }}
@@ -112,12 +112,12 @@
 
     {% set catalog_table = load_result('catalog').table %}
 
-    {% set fallback_schema = var("fallback_schema", null) %}
+    {% set temporary_metadata_generation_schema = var("temporary_metadata_generation_schema", null) %}
      {%- for relation, temp_relation_for_view in view_tmp_tables_mapping.items() %}
-        {% if fallback_schema==null %}
+        {% if temporary_metadata_generation_schema==null %}
           {% set schema_name = relation.schema %}
         {% else %}
-          {% set schema_name = fallback_schema %}
+          {% set schema_name = temporary_metadata_generation_schema %}
         {% endif %}
         {{ teradata__drop_tmp_tables_of_views(schema_name, temp_relation_for_view) }}
      {%- endfor %}
@@ -224,7 +224,7 @@
           {% if view_tmp_tables_mapping is not none and view_tmp_tables_mapping|length > 0 %}
             CASE
               {% for relation, temp_table in view_tmp_tables_mapping.items() %}
-                WHEN columns_transformed.table_schema = upper('{{ var("fallback_schema", null) }}') THEN '{{ relation.schema }}'
+                WHEN columns_transformed.table_schema = upper('{{ var("temporary_metadata_generation_schema", null) }}') THEN '{{ relation.schema }}'
               {% endfor %}
               ELSE columns_transformed.table_schema
             END as table_schema,
@@ -299,8 +299,8 @@
                             upper("table_schema") = upper('{{ relation.schema }}')
                             and upper("table_name") = upper('{{ relation.identifier }}')
                         {% else %}
-                            {% if var("fallback_schema", null) != null %}
-                                upper("table_schema") = upper('{{ var("fallback_schema", null) }}')
+                            {% if var("temporary_metadata_generation_schema", null) != null %}
+                                upper("table_schema") = upper('{{ var("temporary_metadata_generation_schema", null) }}')
                                 and upper("table_name") = upper('{{ temp_table }}')
                             {% else %}
                                 upper("table_schema") = upper('{{ relation.schema }}')

requirements_dev.txt

@@ -7,7 +7,7 @@ wheel
 pytest~=7.0
 tox~=3.2
 pylava~=0.3.0
-teradatasql>=20.00.00.10
+teradatasql<20.00.00.35
 dbt-adapters>=1.7.2
 dbt-common>=1.3.0
 MarkupSafe==2.0.1

tests/functional/adapter/teradata_dbt/test_temporary_metadata_generation_schema.py

@@ -0,0 +1,82 @@
+import pytest
+from dbt.tests.util import run_dbt, read_file
+import pathlib
+import re
+
+seed_csv="""
+id,attrA,attrB,create_date
+1,val1A,val1B,2020-03-05
+2,val2A,val2B,2020-04-05
+3,val3A,val3B,2020-05-05
+4,val4A,val4B,2020-10-05
+""".lstrip()
+
+view_from_source_for_catalog_test_sql="""
+        {{
+            config(
+                materialized="view"
+            )
+        }}
+        SELECT * FROM {{ ref('seed') }}
+"""
+
+table_from_source_for_catalog_test_sql="""
+        {{
+            config(
+                materialized="incremental"
+            )
+        }}
+
+        {% set view_relation = ref('view_from_source_for_catalog_test') %}
+        {% set view_columns = adapter.get_columns_in_relation(view_relation) %}
+
+        SELECT 
+    {{ view_columns|length }} as column_count,
+    CURRENT_TIMESTAMP as test_timestamp
+"""
+
+
+class Test_temporary_metadata_generation_schema:
+    @pytest.fixture(scope="class")
+    def project_config_update(self):
+        return {
+            "name": "test_temporary_metadata_generation_schema",
+            "vars": {
+                    "temporary_metadata_generation_schema": "HASH_TEST"
+            }
+        }
+
+    @pytest.fixture(scope="class")
+    def seeds(self):
+        return {
+            "seed.csv": seed_csv
+        }
+
+    @pytest.fixture(scope="class")
+    def models(self):
+        return {
+            "table_from_source_for_catalog_test.sql": table_from_source_for_catalog_test_sql,
+            "view_from_source_for_catalog_test.sql": view_from_source_for_catalog_test_sql
+        }
+
+
+    def test_temporary_metadata_generation_schema(self, project):
+
+        result1 = run_dbt(["seed"])
+        assert len(result1) == 1
+
+        (pathlib.Path(project.project_root) / "log_output").mkdir(parents=True, exist_ok=True)
+        run_dbt(["--log-path", "log_output","run"])
+        log_output = read_file("log_output", "dbt.log").replace("\n", " ").replace("\\n", " ")
+        pattern = r'"HASH_TEST"\."view_from_source_for_catalog_test_tmp_viw_tbl_\d+"'
+        matches = re.findall(pattern, log_output)
+        # assert '"HASH_TEST".' in log_output
+        assert matches, "No temp_table with random suffix found in dbt run log file!"
+
+        (pathlib.Path(project.project_root) / "log_output_catalog").mkdir(parents=True, exist_ok=True)
+        catalog = run_dbt(["--log-path", "log_output_catalog","docs", "generate"])
+        assert len(catalog.nodes) == 3
+        log_output_catalog = read_file("log_output_catalog", "dbt.log").replace("\n", " ").replace("\\n", " ")
+        pattern = r'"HASH_TEST"\."view_from_source_for_catalog_test_tmp_viw_tbl_\d+"'
+        matches = re.findall(pattern, log_output_catalog)
+        assert matches, "No temp_table with random suffix found in catalog log file!"