diff --git a/.github/workflows/ci-integration-tests-csae.yml b/.github/workflows/ci-integration-tests-csae.yml index e953044f..e8b4c350 100644 --- a/.github/workflows/ci-integration-tests-csae.yml +++ b/.github/workflows/ci-integration-tests-csae.yml @@ -59,18 +59,13 @@ jobs: run: | python -m pip install --upgrade pip pip install -r requirements_dev.txt - - name: Prepare database shell: bash run: | export PATH=$PATH:"/opt/teradata/client/17.20/bin" - cd md5_20080530/md5/src - # prepare bteq script - cat << EOF > /tmp/install_md5.bteq - .SET EXITONDELAY ON MAXREQTIME 20 .logon $DBT_TERADATA_SERVER_NAME/$DBT_TERADATA_USERNAME,$DBT_TERADATA_PASSWORD CREATE DATABASE HASH_TEST AS PERMANENT = 60e6, SPOOL = 120e6; @@ -85,7 +80,6 @@ jobs: GRANT EXECUTE FUNCTION ON GLOBAL_FUNCTIONS TO PUBLIC WITH GRANT OPTION; .logoff EOF - bteq < /tmp/install_md5.bteq cd ../../../tests/functional/adapter @@ -112,7 +106,6 @@ jobs: EOF bteq < /tmp/create_procedure_in_ansi.bteq - env: DBT_TERADATA_SERVER_NAME: ${{ steps.create-csae-environments.outputs.teradata-server-name }} DBT_TERADATA_USERNAME: dbc @@ -126,7 +119,7 @@ jobs: export DBT_TEST_USER_1=test_grants_user1 export DBT_TEST_USER_2=test_grants_user2 export DBT_TEST_USER_3=test_grants_user3 - pytest --cov=dbt -v -k "not TestCurrentTimestamp and not TestProcedureANSIInTERA and not TestProcedureTERAInTERA and not Test_query_band" tests/functional + pytest --cov=dbt -v -k "not TestCurrentTimestamp and not TestProcedureANSIInTERA and not TestProcedureTERAInTERA and not Test_query_band and not TestSafeCast" --ignore=tests/functional/adapter/test_otf_integration.py tests/functional env: DBT_TERADATA_SERVER_NAME: ${{ steps.create-csae-environments.outputs.teradata-server-name }} DBT_TERADATA_USERNAME: dbc @@ -141,7 +134,7 @@ jobs: export DBT_TEST_USER_1=test_grants_user1 export DBT_TEST_USER_2=test_grants_user2 export DBT_TEST_USER_3=test_grants_user3 - pytest --cov=dbt -v -k "not TestCurrentTimestamp and not TestProcedureTERAInANSI and not TestProcedureANSIInANSI" tests/functional --cov-append + pytest --cov=dbt -v -k "not TestCurrentTimestamp and not TestProcedureTERAInANSI and not TestProcedureANSIInANSI and not TestSafeCast" --ignore=tests/functional/adapter/test_otf_integration.py tests/functional --cov-append env: DBT_TERADATA_SERVER_NAME: ${{ steps.create-csae-environments.outputs.teradata-server-name }} DBT_TERADATA_USERNAME: dbc @@ -203,14 +196,12 @@ jobs: run: | cd $GITHUB_WORKSPACE/test/performance ./run.sh - - name: Run catalog tests with ANSI run: | # cd $GITHUB_WORKSPACE/test/catalog/with_qvci # ./run.sh cd $GITHUB_WORKSPACE/test/catalog/without_qvci ./run.sh - - name: Setup environment with TERA for performance, catalog and valid history tests run: | rm ~/.dbt/profiles.yml @@ -290,14 +281,12 @@ jobs: run: | cd $GITHUB_WORKSPACE/test/performance ./run.sh - - name: Run catalog tests with TERA run: | # cd $GITHUB_WORKSPACE/test/catalog/with_qvci # ./run.sh cd $GITHUB_WORKSPACE/test/catalog/without_qvci ./run.sh - - name: Run valid history tests run: | cd $GITHUB_WORKSPACE/test/valid_history_test @@ -309,7 +298,6 @@ jobs: cd $GITHUB_WORKSPACE/test/dbt_external_tables_test/integration_tests chmod 777 run.sh ./run.sh - - name: Run nopi tests run: | sed -i "s/hostname=dbt.*/hostname='$DBT_TERADATA_SERVER_NAME'/g" tests/conftest.py @@ -324,7 +312,6 @@ jobs: - name: Print Code Coverage Report run: | coverage report -m - - name: Delete CSAE environment if: always() run: | diff --git a/.github/workflows/ci-unit-tests.yml b/.github/workflows/ci-unit-tests.yml new file mode 100644 index 00000000..5fa3df91 --- /dev/null +++ b/.github/workflows/ci-unit-tests.yml @@ -0,0 +1,40 @@ +name: CI Unit Tests + +on: + push: + branches: + - main + - 'releases/**' + pull_request: + branches: + - main + - 'releases/**' + workflow_dispatch: + +jobs: + unit-tests: + name: Unit tests (Python ${{ matrix.python }}) + strategy: + fail-fast: false + matrix: + python: ['3.10', '3.11', '3.12', '3.13'] + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + + - name: Install dependencies + shell: bash + run: | + python -m pip install --upgrade pip + pip install -r requirements_dev.txt + + - name: Run unit tests + shell: bash + run: | + pytest tests/unit/ -v diff --git a/CHANGELOG.md b/CHANGELOG.md index a5cd4776..f101f5da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,7 @@ ## dbt-teradata 1.0.0a ### Features - -### Fixes - -### Docs +- Open Table Format (OTF) support via Teradata DATALAKE objects — create and read Iceberg/Delta Lake tables (External OTF / JOTF) with `table` and `incremental` (`append`) materializations. See the "Open Table Format (OTF) support" section in the README for configuration, the supported/unsupported feature matrix, and limitations. ### Under the hood +- README updated with a restructured OTF section: External JOTF vs Managed OTF clarification, "what is / isn't supported" lists, consolidated incremental documentation, and limitations. diff --git a/README.md b/README.md index 4a891973..6ab5f819 100644 --- a/README.md +++ b/README.md @@ -851,6 +851,226 @@ sources: data_type: CHAR(1) ``` +## Open Table Format (OTF) support + +dbt-teradata can create and read Iceberg / Delta Lake tables via Teradata's native Open Table Format support. OTF tables live in an external object store (S3, Azure, GCS) and are registered with an external catalog (AWS Glue, Unity Catalog, etc.); Teradata accesses them through a pre-created `DATALAKE` object that encapsulates the catalog type, authentication, and object-store path. + +### Which OTF flavor is supported + +Teradata offers two ways to work with open table formats. **dbt-teradata supports only External OTF (also called JOTF).** + +| OTF flavor | DDL emitted | Naming | Supported by dbt-teradata? | +| ---------- | ----------- | ------ | -------------------------- | +| **External OTF (JOTF)** | `CREATE TABLE "".""."" ... AS ...` | 3-part | ✅ **Yes** — this is what the adapter generates | +| **Teradata Managed OTF (MOTF)** | `CREATE MANAGED TABLE .
, DATALAKE=
...` | 2-part | ❌ **No** — not implemented | + +> **⚠️ Managed OTF (MOTF) is not supported.** The adapter only emits `CREATE TABLE` with 3-part naming against a pre-created `DATALAKE` object (External OTF). It does not emit `CREATE MANAGED TABLE`, the `DATALAKE=` table clause, `RETENTIONDAYS`, or MOTF primary-index options. External OTF is copy-on-write and does not support `MERGE`, so dbt-teradata only offers append-style writes on OTF tables — `merge`-based upserts would require MOTF, which is not implemented. See [Limitations](#limitations-and-trade-offs) below. + +#### What is supported (External OTF / JOTF) + +* **Table formats:** Apache Iceberg (verified) and Delta Lake. The adapter defaults to `iceberg` / `parquet`. +* **Materializations:** `table` and `incremental` (with `incremental_strategy='append'` only). +* **Catalogs:** any catalog your `DATALAKE` object is configured for (AWS Glue, Hive, Unity Catalog, REST). Schema evolution is verified on Iceberg (Glue/Hive). +* **Cross-model references:** `ref()` and OTF tables declared as `sources` (3-part naming). +* **Model config:** `partitioned_by`, `sorted_by`, `tblproperties`, `purge_mode`, `alias`, `on_schema_change`, `persist_docs`. + +#### What is not supported + +* **Managed OTF (MOTF)** — `CREATE MANAGED TABLE` and 2-part managed tables. +* **Incremental strategies other than `append`** — `merge`, `delete+insert`, `valid_history`, `microbatch` raise a compile-time error (External OTF is copy-on-write and has no `MERGE`). +* **`snapshot` materialization** on OTF tables. +* **Model contracts** (`contract.enforced: true`) on the OTF path. +* **Teradata-native table options** — `table_kind`, `table_option`, `with_statistics`, `index` (these describe native Teradata storage and don't apply to Iceberg/Delta). +* **`grants`** on OTF tables (Teradata does not allow `GRANT` on DATALAKE objects). +* **Catalog types other than `datalake`.** + +A simple OTF table model looks like this: + +```sql +-- models/sales_iceberg.sql +{{ config( + materialized='table', + catalog_name='my_otf_catalog', -- from catalogs.yml + partitioned_by='YEAR(order_date), country' +) }} +select order_id, customer_id, country, order_date, amount +from {{ ref('stg_orders') }} +``` + +### Pre-requisites + +The following must exist on the Teradata side **before** running dbt: + +* A `DATALAKE` object created in Teradata (e.g. via `CREATE DATALAKE my_lake ...`). dbt does not create DATALAKEs. +* A database inside that DATALAKE (the "OTF database") that will hold the OTF tables. dbt does not create this either. +* The dbt user must have permission to `CREATE TABLE` / `DROP TABLE` within the OTF database, and `SELECT` permission to read OTF tables defined elsewhere. + +Refer to the Teradata documentation for `CREATE DATALAKE` syntax and the specific permissions required for your catalog backend. + +### Configuration + +Register the catalog integration in a `catalogs.yml` file at your dbt project root: + +```yaml +catalogs: + - name: my_otf_catalog + active_write_integration: td_datalake + write_integrations: + - name: td_datalake + catalog_type: datalake + adapter_properties: + datalake_name: my_lake # the pre-created DATALAKE object + otf_database: my_otf_db # the pre-created OTF database within it +``` + +`catalog_type` must be `datalake`. `datalake_name` and `otf_database` are both required and validated at integration registration time. + +Reference the catalog from a model via `catalog_name`: + +```sql +-- models/sales_iceberg.sql +{{ config( + materialized='table', + catalog_name='my_otf_catalog', + partitioned_by='YEAR(order_date), country', + sorted_by='customer_id ASC', + tblproperties="'write.format.default'='parquet', 'gc.enabled'='true'", + purge_mode='NO PURGE' +) }} +select + order_id, + customer_id, + country, + order_date, + amount +from {{ ref('stg_orders') }} +``` + +### Naming conventions: 2-part vs 3-part + +Teradata's native objects use **2-part** naming (`database.object`); in dbt-teradata, the `database` field is unused and the `schema` field carries the Teradata database name. OTF tables are the **only** Teradata objects that use **3-part** naming (`"".""."
"`). + +For OTF tables, dbt-teradata maps: + +| dbt field | Teradata concept | +| ------------ | --------------------------------- | +| `database` | DATALAKE name (quoted) | +| `schema` | OTF database name (quoted) | +| `identifier` | OTF table name (quoted) | + +When you set `catalog_name` on a model, dbt-teradata pulls `database` and `schema` from the registered catalog integration automatically. For an OTF table defined as a **source** (where there is no `catalog_name` model config), declare the `database` and `schema` explicitly in `sources.yml` — the adapter detects the 3-part shape (database ≠ schema) and renders it correctly: + +```yaml +version: 2 +sources: + - name: customer_otf + database: my_lake # DATALAKE name + schema: my_otf_db # OTF database name + tables: + - name: customer_iceberg +``` + +A `ref()` from another model then compiles to `"my_lake"."my_otf_db"."customer_iceberg"`. + +> **⚠️ Important: `database` and `schema` must be different values in `sources.yml`.** +> +> The adapter uses the heuristic `database ≠ schema` to auto-detect that a source is an OTF table and should use 3-part naming. If your DATALAKE object and OTF database happen to share the same name (e.g. both are `my_lake`), the adapter will treat the source as a regular 2-part Teradata relation and generate incorrect SQL. +> +> To avoid this, ensure that your DATALAKE name and OTF database name are always different. This is only a constraint for the `sources.yml` path — models that use `catalog_name` in their config are not affected, because OTF detection is explicit rather than heuristic. + +### Supported model config options + +| Option | Type | Description | +| ---------------------- | ------- | ---------------------------------------------------------------------------------------------------------- | +| `catalog_name` | string | Name of the catalog integration from `catalogs.yml`. Required to mark a model as OTF. | +| `partitioned_by` | string | Iceberg/Delta partition expression, e.g. `'YEAR(dt), country'`. | +| `sorted_by` | string | Sort order, e.g. `'id ASC'`. | +| `tblproperties` | string | Iceberg/Delta table properties, e.g. `"'gc.enabled'='true'"`. | +| `purge_mode` | string | DROP behavior. `'NO PURGE'` (default; removes catalog entry only) or `'PURGE ALL'` (also deletes data files on the object store). Case-insensitive. | +| `incremental_strategy` | string | For `materialized='incremental'` only. **Only `'append'` is supported** on OTF (see [Incremental materialization](#incremental-materialization-otf)). | +| `on_schema_change` | string | For `materialized='incremental'` only. `'ignore'` (default), `'fail'`, `'append_new_columns'`, or `'sync_all_columns'` (best-effort on OTF — see below). | +| `alias` | string | Overrides the physical OTF table name in the catalog. The OTF object is created under the alias; the model file name is not used. Works for both `table` and `incremental` OTF models. | + +`persist_docs` and standard dbt cache management work on OTF models the same way they do on native tables. **`grants` is not supported on OTF tables** — Teradata does not allow `GRANT` on DATALAKE objects (access control is managed via AUTHORIZATION objects and external IAM/OAuth policies). Setting `grants` on an OTF model emits a warning and is otherwise ignored. + +### Incremental materialization (OTF) + +OTF tables can be materialized incrementally with `materialized='incremental'` and a `catalog_name`: + +```sql +-- models/orders_otf_incremental.sql +{{ config( + materialized='incremental', + catalog_name='my_otf_catalog', + incremental_strategy='append', + partitioned_by='YEAR(order_date)', + on_schema_change='append_new_columns' +) }} +select order_id, customer_id, order_date, amount +from {{ ref('stg_orders') }} +{% if is_incremental() %} + where order_date > (select max(order_date) from {{ this }}) +{% endif %} +``` + +How it runs: + +* **First run** creates the OTF table (`CREATE TABLE ... AS ... WITH DATA`). +* **Subsequent runs** load new rows into a regular Teradata staging table, then `INSERT ... SELECT` into the OTF table (positional insert — OTF does not accept a target column list). +* **`--full-refresh`** drops and re-creates the table from scratch. +* Existence is detected by probing the 3-part name with `SELECT ... SAMPLE 0` (OTF tables are not registered in `DBC.TablesV`/`DBC.ColumnsV` under the dbt schema), treating Teradata errors **7825** and **6321** ("OTF table does not exist") as "not found". + +> **Only `incremental_strategy='append'` is supported on OTF.** `merge`, `delete+insert`, `valid_history`, and `microbatch` raise a compile-time error. Teradata External OTF does not support `MERGE` and is copy-on-write only, so the upsert-style strategies cannot be honored. Use `append` (optionally with an `is_incremental()` filter to bound the rows appended). + +#### `on_schema_change` on OTF + +dbt's [`on_schema_change`](https://docs.getdbt.com/docs/build/incremental-models#what-if-the-columns-of-my-incremental-model-change) is supported on OTF incremental models with these values: + +| Value | OTF behavior | +| ----- | ------------ | +| `ignore` (default) | No schema reconciliation. The append assumes the source and the existing OTF table have the same columns in the same order. | +| `fail` | Compares the incoming (source) columns with the existing OTF columns and raises a clear error if any column was added or removed. | +| `append_new_columns` | For each column present in the source but not yet in the OTF table, issues a separate `ALTER TABLE ... ADD `; the new columns are added at the end of the table. Pre-existing rows get `NULL` for the new columns; rows inserted on this run carry the new values. The `INSERT` is reordered to match the resulting OTF column layout. | +| `sync_all_columns` | **Best-effort on OTF.** Makes the OTF table match the source: **adds** new columns, **drops** columns no longer in the source (destructive), and applies **type changes** OTF/Iceberg permits (e.g. `int → bigint`, decimal precision widening). A type change OTF cannot apply in place raises a clear error directing you to `--full-refresh`. | + +**Limitations of `on_schema_change` on OTF:** + +* **`sync_all_columns` is best-effort, and type changes are limited.** Type comparison is done at OTF/Iceberg granularity (via `HELP TABLE`'s `OTF Type`), so `VARCHAR` length and `SMALLINT`-vs-`INTEGER` differences are **not** treated as changes (OTF doesn't preserve them). Only OTF-permitted promotions are applied in place — verified: `int → bigint` ✅ and decimal **precision** widening (same scale) ✅; scale changes, narrowing, and cross-family changes (e.g. `decimal → string`) raise a clear error → use `--full-refresh`. `sync_all_columns` also performs **destructive, irreversible column drops** (OTF has no rollback). If you only ever add columns, prefer `append_new_columns`. +* **`append_new_columns` is additive only.** New source columns are added; columns removed from the source are **kept** on the OTF table (and back-filled with `NULL` for subsequent rows). Existing column **types are never changed**. +* **Column *order* in the model `SELECT` is handled automatically under `append_new_columns`.** OTF inserts are positional (no target column list), but `append_new_columns` realigns the `INSERT ... SELECT` to the table's column layout *by name*, so you do **not** need to place new columns at the end of the `SELECT`, and reordering existing columns is safe. (`ALTER ... ADD` does physically append new columns to the end of the table; the realignment is what keeps the data correct.) Under `on_schema_change='ignore'`, by contrast, **no realignment happens** — the model `SELECT` must produce columns in the same order as the existing OTF table. +* **Each schema change is a separate `ALTER` statement.** OTF cannot combine multiple alter operations into one statement, and External OTF does not allow multi-statement requests, so `N` new columns produce `N` separate `ALTER TABLE ... ADD` statements. There is no rollback if one of them fails midway. +* **Catalog/format support varies.** Schema evolution is verified on **Iceberg** (AWS Glue / Hive). **Unity Catalog does not support schema evolution at all** — any `ALTER` (including `append_new_columns`) will fail at the database. **Delta Lake** may require `delta.columnMapping.mode='name'` for column changes. On unsupported catalogs the `ALTER` surfaces the underlying Teradata error. + +To apply a schema change that neither `append_new_columns` nor `sync_all_columns` can do in place (e.g. a `VARCHAR`→numeric change, a decimal scale change, or a type narrowing), run the model with `--full-refresh`. + +#### `alias` on incremental OTF models + +The `alias` config sets the physical OTF table name in the catalog (the model file name is not used). It works for both `table` and `incremental` OTF models — for incremental models, both the initial CREATE and all subsequent INSERT operations target the alias-named OTF object: + +```sql +{{ config( + materialized='incremental', + catalog_name='my_otf_catalog', + incremental_strategy='append', + alias='orders_iceberg' -- physical OTF table name +) }} +``` + +### Error 7825 / 6321 suppression + +Teradata raises error **7825** ("OTF table not found in external catalog") — or **6321** ("OTF Error: Table does not exist") on newer OTF engines (e.g. 20.0.0.61) — when a `DROP TABLE` (or existence probe) targets an OTF table that no longer exists in the external catalog (e.g. Glue). dbt-teradata treats both codes the same way it treats native errors 3807/3853/3854 — suppressed under `IF EXISTS` semantics — so re-running a dbt project after an OTF table has been deleted externally does not fail, and first-run incremental existence checks correctly detect a missing table. + +### Limitations and trade-offs + +* **Only External OTF (JOTF) is supported; Managed OTF (MOTF) is not.** The adapter emits `CREATE TABLE` against a pre-created `DATALAKE` (3-part naming). It does not emit `CREATE MANAGED TABLE`. External OTF is copy-on-write and has no `MERGE`, so dbt-teradata only performs append-style writes (`incremental_strategy='append'`); `merge`-based upserts would require MOTF. +* **Non-atomic re-materialization.** OTF tables use 3-part naming that cannot be renamed via standard DDL, so the adapter cannot use the build-tmp-then-rename pattern that protects native tables on a failed CREATE. An OTF model is dropped before it is re-created — if the CREATE fails, the table is gone. Plan for `--full-refresh` workflows accordingly. +* **Model contracts are not supported on the OTF path.** Setting `contract.enforced: true` together with `catalog_name` raises a compile-time error. +* **Teradata-native table options are not supported.** Setting any of `table_kind`, `table_option`, `with_statistics`, or `index` together with `catalog_name` raises a compile-time error — these options describe native Teradata table storage and do not apply to Iceberg/Delta tables. +* **Only `catalog_type: datalake` is supported.** Other catalog types are rejected with a compile-time error. +* **Incremental: only the `append` strategy is supported.** `merge`/`delete+insert`/`valid_history`/`microbatch` raise compile-time errors. All four `on_schema_change` values are supported, with `sync_all_columns` being best-effort (limited type changes). See [Incremental materialization (OTF)](#incremental-materialization-otf). +* **OTF cannot be used with the `snapshot` materialization.** Setting `catalog_name` on a snapshot raises a compile-time error (snapshots require update/merge semantics OTF does not provide). +* **Column metadata is not available for OTF tables in dbt docs.** OTF tables are not registered in Teradata's `DBC.ColumnsV` view (only native tables are), so column-level metadata (descriptions, data types, constraints) will not appear in dbt docs when you run `dbt docs generate` and serve with `dbt docs serve`. Table-level metadata is still available and functional. + ## temporary_metadata_generation_schema (earlier fallback_schema) dbt-teradata internally created temporary tables to fetch the metadata of views for manifest and catalog creation. In case if user does not have permission to create tables on the schema they are working on, they can define a temporary_metadata_generation_schema(to which they have proper create and drop privileges) in dbt_project.yml as variable. diff --git a/dbt/adapters/teradata/catalogs.py b/dbt/adapters/teradata/catalogs.py new file mode 100644 index 00000000..86b6b708 --- /dev/null +++ b/dbt/adapters/teradata/catalogs.py @@ -0,0 +1,146 @@ +"""Catalog integrations for Teradata Open Table Format (OTF) support. + +This module provides the DATALAKE catalog integration that enables dbt to +create and manage Iceberg/Delta Lake tables via Teradata's native OTF support. +Tables are addressed using 3-part naming: .""."
". + +Configuration in catalogs.yml: + + catalogs: + - name: my_catalog + write_integrations: + - name: iceberg_glue + catalog_type: datalake + adapter_properties: + datalake_name: MyOTFLake # pre-created DATALAKE object + otf_database: my_otf_db # pre-created database within the DATALAKE + +Model-level config options (set via {{ config(...) }} in .sql files): + + partitioned_by -- Iceberg partition expression, e.g. 'YEAR(dt), country' + sorted_by -- Iceberg sort order, e.g. 'id ASC' + tblproperties -- Iceberg table properties, e.g. "'gc.enabled'='true'" + purge_mode -- DROP behavior: 'NO PURGE' (default) or 'PURGE ALL' + NO PURGE = remove catalog entry only, keep data files (safe default) + PURGE ALL = remove catalog entry AND delete data files on object store +""" + +from dataclasses import dataclass +from typing import Optional + +from dbt.adapters.catalogs import ( + CatalogIntegration, + CatalogIntegrationConfig, + CatalogRelation, + InvalidCatalogIntegrationConfigError, +) +from dbt.adapters.contracts.relation import RelationConfig + + +# --------------------------------------------------------------------------- +# Relation dataclass -- carries catalog metadata into Jinja macros +# --------------------------------------------------------------------------- + +@dataclass +class TeradataCatalogRelation(CatalogRelation): + """Relation metadata for DATALAKE-based OTF tables. + + Fields populated from catalogs.yml (via the integration): + catalog_type, catalog_name, table_format, file_format, + external_volume, datalake_name, otf_database + + Fields populated from model config (via build_relation): + partitioned_by, sorted_by, tblproperties, purge_mode + """ + catalog_type: Optional[str] = None + catalog_name: Optional[str] = None + table_format: Optional[str] = None + file_format: Optional[str] = None + external_volume: Optional[str] = None + datalake_name: Optional[str] = None + otf_database: Optional[str] = None + partitioned_by: Optional[str] = None + sorted_by: Optional[str] = None + tblproperties: Optional[str] = None + # Controls DROP TABLE behavior for OTF tables: + # 'NO PURGE' -- removes catalog entry only, data files remain (default; safe) + # 'PURGE ALL' -- removes catalog entry AND deletes data files on object store + purge_mode: Optional[str] = None + + +# --------------------------------------------------------------------------- +# DATALAKE integration -- 3-part naming (datalake."otf_db"."table") +# --------------------------------------------------------------------------- + +class TeradataDatalakeCatalogIntegration(CatalogIntegration): + """Catalog integration for Teradata DATALAKE objects. + + Works with any external catalog (AWS Glue, Unity Catalog, etc.) because + Teradata's DATALAKE object encapsulates the catalog type, auth, and + object store path. dbt only needs the datalake_name and otf_database + for 3-part naming. + + Generated SQL example: + DROP TABLE /*+ IF EXISTS */ .""."
" PURGE ALL; + CREATE TABLE .""."
" + PARTITIONED BY (...) + SORTED BY ... + TBLPROPERTIES(...) + AS (...) WITH DATA; + + Required adapter_properties in catalogs.yml: + datalake_name -- name of the pre-created DATALAKE object in Teradata + otf_database -- name of the pre-created database within the DATALAKE + """ + + catalog_type = "datalake" + allows_writes = True + table_format = "iceberg" + file_format = "parquet" + + def __init__(self, config: CatalogIntegrationConfig) -> None: + super().__init__(config) + # Restore class-level defaults if not provided in config + if config.file_format is None: + self.file_format = "parquet" + adapter_props = config.adapter_properties or {} + self.datalake_name = (adapter_props.get("datalake_name") or "").strip() or None + self.otf_database = (adapter_props.get("otf_database") or "").strip() or None + if not self.datalake_name: + raise InvalidCatalogIntegrationConfigError( + config.name, + "adapter_properties.datalake_name is required -- " + "it must match the pre-created DATALAKE object in Teradata", + ) + if not self.otf_database: + raise InvalidCatalogIntegrationConfigError( + config.name, + "adapter_properties.otf_database is required -- " + "it must match the pre-created OTF database within the DATALAKE", + ) + + def build_relation(self, config: RelationConfig) -> TeradataCatalogRelation: + """Build a TeradataCatalogRelation from integration + model config. + + Integration-level fields (datalake_name, otf_database, etc.) come from + catalogs.yml. Model-level fields (partitioned_by, sorted_by, + tblproperties, purge_mode) come from the model's {{ config(...) }}. + """ + raw_config = config.config if hasattr(config, 'config') and config.config else {} + def _get(key: str): + if hasattr(raw_config, "get"): + return raw_config.get(key) + return getattr(raw_config, key, None) + return TeradataCatalogRelation( + catalog_type=self.catalog_type, + catalog_name=self.catalog_name, + table_format=self.table_format, + file_format=self.file_format, + external_volume=self.external_volume, + datalake_name=self.datalake_name, + otf_database=self.otf_database, + partitioned_by=_get("partitioned_by"), + sorted_by=_get("sorted_by"), + tblproperties=_get("tblproperties"), + purge_mode=_get("purge_mode"), + ) diff --git a/dbt/adapters/teradata/connections.py b/dbt/adapters/teradata/connections.py index 6de4fb1c..1e749081 100644 --- a/dbt/adapters/teradata/connections.py +++ b/dbt/adapters/teradata/connections.py @@ -431,20 +431,24 @@ def add_query( try: return SQLConnectionManager.add_query(self, sql, auto_begin, bindings, abridge_sql_log) except Exception as ex: - ignored = False - query = sql.strip() - if ("DROP view /*+ IF EXISTS */" in query) or ("DROP table /*+ IF EXISTS */" in query): - for error_number in [3807, 3854, 3853]: - if f"[Error {error_number}]" in str (ex): - ignored = True + query_upper = sql.strip().upper() + if ("DROP VIEW /*+ IF EXISTS */" in query_upper) or ("DROP TABLE /*+ IF EXISTS */" in query_upper): + # 3807 = object does not exist (standard Teradata) + # 3854 = table does not exist (standard Teradata) + # 3853 = view does not exist (standard Teradata) + # 7825 = OTF table not found in external catalog (e.g. Glue/Unity) + # raised by ICEBERG_EXPORT UDF when dropping a nonexistent + # DATALAKE table; safe to ignore under IF EXISTS semantics + # 6321 = "OTF Error: Table does not exist" raised by newer OTF + # engines (e.g. TD 20.0.0.61) instead of 7825; same intent + for error_number in [3807, 3854, 3853, 7825, 6321]: + if f"[Error {error_number}]" in str(ex): return None, None - if ("DELETE DATABASE /*+ IF EXISTS */" in query) or ("DROP DATABASE /*+ IF EXISTS */" in query): + if ("DELETE DATABASE /*+ IF EXISTS */" in query_upper) or ("DROP DATABASE /*+ IF EXISTS */" in query_upper): for error_number in [3802]: - if f"[Error {error_number}]" in str (ex): - ignored = True + if f"[Error {error_number}]" in str(ex): return None, None - if not ignored: - raise # rethrow + raise # rethrow # this method will return the datatype as string @classmethod diff --git a/dbt/adapters/teradata/impl.py b/dbt/adapters/teradata/impl.py index 3b52b5b1..634736f3 100644 --- a/dbt/adapters/teradata/impl.py +++ b/dbt/adapters/teradata/impl.py @@ -1,6 +1,7 @@ from concurrent.futures import Future from dataclasses import dataclass, asdict from typing import Optional, List, Dict, Any, Union, Iterable, Callable, Set, FrozenSet, Tuple +import re import agate import dbt @@ -13,6 +14,7 @@ from dbt.adapters.teradata import TeradataConnectionManager from dbt.adapters.teradata import TeradataRelation from dbt.adapters.teradata import TeradataColumn +from dbt.adapters.teradata.catalogs import TeradataDatalakeCatalogIntegration from dbt.adapters.capability import CapabilityDict, CapabilitySupport, Support, Capability from dbt.adapters.base.meta import available from dbt.adapters.base import BaseRelation @@ -55,6 +57,10 @@ class TeradataAdapter(SQLAdapter): Column = TeradataColumn ConnectionManager = TeradataConnectionManager + CATALOG_INTEGRATIONS = [ + TeradataDatalakeCatalogIntegration, + ] + CONSTRAINT_SUPPORT = { ConstraintType.check: ConstraintSupport.ENFORCED, ConstraintType.not_null: ConstraintSupport.ENFORCED, @@ -160,10 +166,50 @@ def list_relations_without_caching( def get_relation( self, database: str, schema: str, identifier: str ) -> Optional[BaseRelation]: + # Preserve the original database before nulling it out — for OTF models, + # this holds the datalake name (e.g. "MyOTFLake"), which we need for the + # SAMPLE 0 fallback below. + original_database = database if not self.Relation.get_default_include_policy().database: database = None - return super().get_relation(database, schema, identifier) + # Standard lookup: queries DBC.TablesV. Works for all regular Teradata + # tables and views. Returns immediately when the relation is found. + relation = super().get_relation(database, schema, identifier) + if relation is not None: + return relation + + # OTF fallback: OTF (Iceberg/Delta) tables are stored in a DATALAKE + # catalog and are NOT registered in DBC.TablesV, so the standard lookup + # above always returns None for them. If we have a non-None database + # (the datalake name, preserved above), probe via SAMPLE 0. + # + # This makes is_incremental() evaluate correctly for OTF models: without + # this probe it would always return False because get_relation returns None, + # causing the model SQL to always compile without its incremental WHERE + # filter and resulting in duplicate rows on every incremental run. + # + # Regular Teradata tables/views are unaffected: + # - Existing ones are returned by super().get_relation() above. + # - For non-existing ones, original_database is typically None (Teradata + # profiles rarely set a database), so the probe is skipped entirely. + # - Guard original_database != schema: for native Teradata, database is + # often set to the same value as schema. For OTF, the datalake name + # is always distinct from the otf_database. This prevents unnecessary + # SAMPLE 0 probes (and potential errors) against non-DATALAKE 3-part names. + if original_database and schema and identifier and original_database != schema: + if self.otf_relation_exists(original_database, schema, identifier): + return self.Relation.create( + database=original_database, + schema=schema, + identifier=identifier, + type='table', + is_otf=True, + quote_policy={"database": True, "schema": True, "identifier": True}, + include_policy={"database": True, "schema": True, "identifier": True}, + ) + + return None def get_catalog( self, @@ -351,4 +397,170 @@ def valid_incremental_strategies(self): Not used to validate custom strategies defined by end users. """ return ["delete+insert","append","merge", "valid_history", "microbatch"] - + + @available + def otf_relation_exists(self, datalake_name: str, otf_database: str, identifier: str) -> bool: + """Check whether an OTF (DATALAKE) table exists on the server. + + OTF tables are not registered in DBC.TablesV under the dbt target + schema, so load_relation() / list_relations_without_caching() cannot + be used. Instead, probe the 3-part DATALAKE name with SAMPLE 0: + + SELECT * FROM "".""."" SAMPLE 0 + + This succeeds (returning 0 rows) when the table exists, and raises a + "table does not exist" error when it does not. The exact error code + depends on the Teradata / OTF engine version: + - Error 7825 = ICEBERG_EXPORT "Table does not exist" (external catalog) + - Error 6321 = "OTF Error: Table does not exist" (newer OTF engines, + e.g. 20.0.0.61) + Both indicate a missing OTF table and are treated as "does not exist". + + Returns True if the table exists, False otherwise. + """ + otf_name = self.Relation.create( + database=datalake_name, + schema=otf_database, + identifier=identifier, + is_otf=True, + ).render() + try: + self.connections.execute( + f"SELECT * FROM {otf_name} SAMPLE 0", + auto_begin=False, + fetch=False, + ) + return True + except dbt_common.exceptions.DbtDatabaseError as ex: + # "Table does not exist" for an OTF table surfaces under different + # error codes across Teradata/OTF versions (7825 on older external + # catalog paths, 6321 on newer OTF engines). Swallow only these + # specific "not found" codes; re-raise auth/network/syntax failures. + msg = str(ex) + if "[Error 7825]" in msg or "[Error 6321]" in msg: + return False + raise + + @available + def get_otf_columns_in_relation(self, datalake_name: str, otf_database: str, identifier: str) -> List[str]: + """Return the column NAMES of an OTF (DATALAKE) table. + + OTF tables are not registered in DBC.ColumnsV, and HELP COLUMN raises + Error 7825 for them, so the normal get_columns_in_relation() path cannot + read an OTF table's schema. Instead, read the result-set metadata from a + zero-row probe: + + SELECT * FROM "".""."
" SAMPLE 0 + + The agate result carries the column names even with no rows. Column + *types* are not reliably recoverable this way, so only names are + returned; callers that need a new column's DDL type source it from the + (regular Teradata) staging table instead. Used by on_schema_change + handling to diff the OTF target against the incoming staging schema. + """ + otf_name = self.Relation.create( + database=datalake_name, + schema=otf_database, + identifier=identifier, + is_otf=True, + ).render() + _, table = self.connections.execute( + f"SELECT * FROM {otf_name} SAMPLE 0", + auto_begin=False, + fetch=True, + ) + return list(table.column_names) + + @available + def get_otf_column_types(self, datalake_name: str, otf_database: str, identifier: str) -> List[Dict[str, str]]: + """Return the columns of an OTF table as ``[{'name', 'otf_type'}]``. + + Uses ``HELP TABLE "".""."
"``, which (unlike + ``HELP COLUMN`` and ``DBC.ColumnsV``) works for OTF tables and exposes an + ``OTF Type`` column carrying the canonical Iceberg/Delta type -- + ``int``, ``long``, ``string`` (no length), ``decimal(p, s)``, ``double``, + ``date``, ``timestamp``, etc. + + Comparing on the OTF type (rather than the Teradata DDL type) is what + makes ``sync_all_columns`` correct: OTF does not preserve VARCHAR length + or SMALLINT vs INTEGER, so those never appear as spurious type changes. + Names and types are lower-cased and whitespace-normalised. + """ + otf_name = self.Relation.create( + database=datalake_name, + schema=otf_database, + identifier=identifier, + is_otf=True, + ).render() + _, table = self.connections.execute( + f"HELP TABLE {otf_name}", + auto_begin=False, + fetch=True, + ) + + def _norm(value: Any) -> str: + return re.sub(r"\s+", " ", str(value).strip()).lower() if value is not None else "" + + columns: List[Dict[str, str]] = [] + for row in table.rows: + name = _norm(row["Column Name"]) + otf_type = _norm(row["OTF Type"]) + if name: + columns.append({"name": name, "otf_type": otf_type}) + return columns + + @available + def teradata_type_to_otf_type(self, data_type: Optional[str]) -> str: + """Map a Teradata column DDL type (e.g. from a staging table) to the + canonical OTF/Iceberg type reported by ``HELP TABLE`` ``OTF Type``. + + Lets ``sync_all_columns`` compare the incoming (staging) schema against + the OTF target on the same vocabulary, so that differences OTF does not + represent (VARCHAR length, SMALLINT vs INTEGER) are not flagged as type + changes. Best-effort: unknown types fall back to their normalised string. + """ + t = re.sub(r"\s+", " ", (data_type or "").strip()).lower() + m = re.match(r"^(?:decimal|numeric)\s*\(\s*(\d+)\s*,\s*(\d+)\s*\)", t) + if m: + return f"decimal({m.group(1)}, {m.group(2)})" + if t.startswith(("decimal", "numeric")): + return "decimal(38, 0)" + if t.startswith(("varchar", "char", "long varchar", "clob", "character", "vargraphic", "graphic")): + return "string" + if t.startswith("bigint"): + return "long" + if t.startswith(("integer", "int", "smallint", "byteint")): + return "int" + if t.startswith(("float", "real", "double", "number")): + return "double" + if t.startswith("timestamp"): + return "timestamp" + if t.startswith("date"): + return "date" + if t.startswith("time"): + return "time" + if t.startswith(("byte", "varbyte", "blob", "binary")): + return "binary" + return t + + @available + def otf_type_promotion_allowed(self, old_otf_type: Optional[str], new_otf_type: Optional[str]) -> bool: + """Whether changing a column from ``old_otf_type`` to ``new_otf_type`` is + a type promotion OTF/Iceberg permits via ``ALTER ... MODIFY``. + + Verified against the engine: ``int -> long`` and decimal *precision* + widening (same scale) are allowed; scale changes, narrowing, and string + changes are not. Equal types are trivially allowed (no-op). + """ + old = (old_otf_type or "").strip().lower() + new = (new_otf_type or "").strip().lower() + if old == new: + return True + if old == "int" and new == "long": + return True + mo = re.match(r"^decimal\(\s*(\d+)\s*,\s*(\d+)\s*\)$", old) + mn = re.match(r"^decimal\(\s*(\d+)\s*,\s*(\d+)\s*\)$", new) + if mo and mn and mo.group(2) == mn.group(2) and int(mn.group(1)) >= int(mo.group(1)): + return True + return False + diff --git a/dbt/adapters/teradata/relation.py b/dbt/adapters/teradata/relation.py index 0fc6146b..ad000f95 100644 --- a/dbt/adapters/teradata/relation.py +++ b/dbt/adapters/teradata/relation.py @@ -1,8 +1,13 @@ from dataclasses import dataclass, field +from typing import Any, Type, TypeVar from dbt.adapters.base.relation import BaseRelation, Policy +from dbt.adapters.contracts.relation import HasQuoting, RelationConfig from dbt_common.exceptions import DbtRuntimeError +Self = TypeVar("Self", bound="TeradataRelation") + + @dataclass class TeradataQuotePolicy(Policy): database: bool = False @@ -22,8 +27,105 @@ class TeradataRelation(BaseRelation): quote_policy: Policy = field(default_factory=lambda: TeradataQuotePolicy()) include_policy: Policy = field(default_factory=lambda: TeradataIncludePolicy()) quote_character: str = '"' + is_otf: bool = False + + @classmethod + def create_from( + cls: Type[Self], + quoting: HasQuoting, + relation_config: RelationConfig, + **kwargs: Any, + ) -> Self: + # Check if the referenced model has catalog_name (i.e. it's an OTF table) + catalog_name = None + if hasattr(relation_config, "config") and relation_config.config: + catalog_name = ( + relation_config.config.get("catalog_name") + if hasattr(relation_config.config, "get") + else getattr(relation_config.config, "catalog_name", None) + ) + + if catalog_name: + # Lazy import to avoid a circular dependency: dbt.adapters.factory + # imports adapter classes during registration, which would re-enter + # this module if imported at module load time. + from dbt.adapters.factory import get_adapter + adapter = get_adapter(quoting) + catalog_integration = adapter.get_catalog_integration(catalog_name) + + if catalog_integration.catalog_type == "datalake": + # Build a clean kwargs dict — quote_policy/include_policy are + # set explicitly below, so any caller-supplied versions must + # be discarded to avoid "multiple values for keyword argument". + forwarded_kwargs = { + k: v for k, v in kwargs.items() + if k not in ("quote_policy", "include_policy") + } + + # Build an OTF relation with 3-part DATALAKE naming: + # "".""."
" + return cls.create( + database=catalog_integration.datalake_name, + schema=catalog_integration.otf_database, + identifier=relation_config.identifier, + quote_policy={ + "database": True, # DATALAKE name is quoted + "schema": True, # otf_database is quoted + "identifier": True, # table name is quoted + }, + include_policy={ + "database": True, # include all 3 parts + "schema": True, + "identifier": True, + }, + is_otf=True, + **forwarded_kwargs, + ) + + # Standard Teradata relation (non-OTF) + relation = super().create_from(quoting, relation_config, **kwargs) + + # In Teradata, normal objects use 2-part naming (database.object, where + # dbt's `database` and `schema` collapse to the same Teradata database). + # Only OTF objects use 3-part naming (catalog.schema.object). Therefore, + # if a relation arrives with both `database` and `schema` set to + # *different* values, it can only be an OTF reference (typically + # declared in sources.yml without a `catalog_name` model config). + # + # This heuristic is safe because TeradataCredentials.__post_init__ + # (connections.py) raises a DbtRuntimeError when the profile sets + # `database` to a value different from `schema`. Native cross-database + # references with database != schema cannot reach this point. + if relation.database and relation.schema and relation.database != relation.schema: + return relation.replace( + include_policy=Policy(database=True, schema=True, identifier=True), + is_otf=True, + ) + + return relation def render(self): + if self.is_otf: + # OTF relations use 3-part naming: "".""."
". + # Quote and escape all three parts explicitly without relying on private APIs. + # NOTE: this quoting (wrap in quote_character, double any embedded + # quote) is mirrored by the Jinja macro teradata__quote_otf_part in + # macros/materializations/otf/create_otf_table_as.sql. Keep the two in + # sync so OTF DDL/DML and relation rendering produce identical names. + def _quote(part: str) -> str: + qc = self.quote_character or '"' + return qc + part.replace(qc, qc * 2) + qc + + if self.database is None or self.schema is None: + raise DbtRuntimeError( + f"OTF relation is missing required part(s): " + f"database={self.database!r}, schema={self.schema!r}, " + f"identifier={self.identifier!r}" + ) + if self.identifier is None: + # Schema-only OTF relation (e.g. cache warming via .without_identifier()). + return f"{_quote(self.database)}.{_quote(self.schema)}" + return f"{_quote(self.database)}.{_quote(self.schema)}.{_quote(self.identifier)}" if self.include_policy.database and self.include_policy.schema: raise DbtRuntimeError( f"Got a teradata relation with schema and database set to " diff --git a/dbt/include/teradata/macros/adapters.sql b/dbt/include/teradata/macros/adapters.sql index 8e8b3593..8a895c3a 100644 --- a/dbt/include/teradata/macros/adapters.sql +++ b/dbt/include/teradata/macros/adapters.sql @@ -9,9 +9,32 @@ {% endmacro %} {% macro teradata__drop_relation(relation) -%} - {% call statement('drop_relation', auto_begin=False) -%} - DROP {{ relation.type }} /*+ IF EXISTS */ {{ relation }}; - {%- endcall %} + {%- set catalog_name = config.get('catalog_name', none) -%} + {% if catalog_name is not none %} + {#- OTF is only valid for tables. Reject catalog_name on non-table relations + to prevent accidentally running OTF DROP TABLE against a view. -#} + {% if relation.type != 'table' %} + {{ exceptions.raise_compiler_error( + "catalog_name is only supported on table relations, but got relation type '" + ~ relation.type ~ "' for " ~ relation ~ "." + ) }} + {% endif %} + {#- OTF table: use 3-part naming. The purge clause is mandatory for OTF + DROP TABLE; the shared validator normalises and rejects bad values. -#} + {% set catalog_integration = adapter.get_catalog_integration(catalog_name) %} + {% if catalog_integration.catalog_type == 'datalake' %} + {{ teradata__drop_otf_table(catalog_integration, relation.identifier, config.get('purge_mode')) }} + {% else %} + {{ exceptions.raise_compiler_error( + "Unsupported catalog_type '" ~ catalog_integration.catalog_type ~ "' for drop_relation." + ) }} + {% endif %} + {% else %} + {#- Standard Teradata table/view drop -#} + {% call statement('drop_relation', auto_begin=False) -%} + DROP {{ relation.type }} /*+ IF EXISTS */ {{ relation }}; + {%- endcall %} + {% endif %} {% endmacro %} {% macro teradata__truncate_relation(relation) -%} @@ -21,6 +44,32 @@ {% endmacro %} {% macro teradata__create_table_as(temporary, relation, sql) -%} + {%- set catalog_name = config.get('catalog_name', none) -%} + + {% if catalog_name is not none %} + {#- Guard against config combinations that are not supported on the OTF path. + Teradata-native options (table_kind, table_option, with_statistics, index) + do not apply to Iceberg/Delta tables and would be silently ignored if + permitted. Contract enforcement is not yet implemented for OTF. -#} + {%- set unsupported = [] -%} + {%- if config.get('table_kind') -%}{%- do unsupported.append('table_kind') -%}{%- endif -%} + {%- if config.get('table_option') -%}{%- do unsupported.append('table_option') -%}{%- endif -%} + {%- if config.get('with_statistics', default=False) | as_bool -%}{%- do unsupported.append('with_statistics') -%}{%- endif -%} + {%- if config.get('index') -%}{%- do unsupported.append('index') -%}{%- endif -%} + {%- if unsupported | length > 0 -%} + {{ exceptions.raise_compiler_error( + "The following config option(s) are not supported with catalog_name (OTF): " + ~ unsupported | join(', ') + ) }} + {%- endif -%} + {%- set contract_config = config.get('contract') -%} + {%- if contract_config is not none and contract_config.enforced -%} + {{ exceptions.raise_compiler_error( + "Model contracts (contract.enforced=true) are not yet supported with catalog_name (OTF)." + ) }} + {%- endif -%} + {{ teradata__create_otf_table_as(relation, sql, catalog_name) }} + {% else %} {%- set sql_header = config.get('sql_header', none) -%} {%- set table_kind = config.get('table_kind', default='') -%} {%- set table_option = config.get('table_option', default='') -%} @@ -81,6 +130,7 @@ {{ sql }} ; {% endif %} + {% endif %} {% endmacro %} {% macro teradata__create_view_as(relation, sql) -%} @@ -302,6 +352,10 @@ {%- endmacro %} {% macro teradata__create_schema(relation) -%} + {%- if relation.is_otf -%} + {#- OTF schemas live inside a DATALAKE and cannot be created via standard + Teradata DDL. They must be pre-created outside of dbt. Skip silently. -#} + {%- else -%} {%- call statement('create_schema') -%} CREATE DATABASE {{ relation.without_identifier().include(database=False) }} -- Teradata expects db sizing params on creation. This macro is probably @@ -310,10 +364,14 @@ AS PERMANENT = 60e6, -- 60MB SPOOL = 120e6; -- 120MB {%- endcall -%} + {%- endif -%} {% endmacro %} {% macro teradata__drop_schema(relation) -%} - {% if relation.schema -%} + {%- if relation.is_otf -%} + {#- OTF schemas live inside a DATALAKE and cannot be dropped via standard + Teradata DDL. Skip silently. -#} + {%- elif relation.schema -%} {{ adapter.verify_database(relation.schema) }} {%- call statement('drop_schema_delete_database') -%} DELETE DATABASE /*+ IF EXISTS */ {{ relation.without_identifier().include(database=False) }} ALL; diff --git a/dbt/include/teradata/macros/materializations/incremental/incremental.sql b/dbt/include/teradata/macros/materializations/incremental/incremental.sql index 7900696e..92332412 100644 --- a/dbt/include/teradata/macros/materializations/incremental/incremental.sql +++ b/dbt/include/teradata/macros/materializations/incremental/incremental.sql @@ -5,6 +5,13 @@ -- calling the macro set_query_band() which will set the query_band for this materialization as per the user_configuration {% do set_query_band() %} +{%- set catalog_name = config.get('catalog_name', none) -%} +{% if catalog_name is not none %} + {#-- OTF (Iceberg / Delta Lake) incremental path --#} + {% set otf_result = teradata__incremental_otf(catalog_name, sql) %} + {{ return(otf_result) }} +{% else %} + {% set unique_key = config.get('unique_key') %} -- Start: Below are the configuration options for the valid_history strategy @@ -90,4 +97,6 @@ {{ return({'relations': [target_relation]}) }} +{% endif %} + {%- endmaterialization %} diff --git a/dbt/include/teradata/macros/materializations/otf/create_otf_table_as.sql b/dbt/include/teradata/macros/materializations/otf/create_otf_table_as.sql new file mode 100644 index 00000000..1eae3186 --- /dev/null +++ b/dbt/include/teradata/macros/materializations/otf/create_otf_table_as.sql @@ -0,0 +1,124 @@ +{# + OTF (Open Table Format) table creation macros for Teradata DATALAKE objects. + + These macros generate CREATE TABLE AS statements targeting Iceberg/Delta Lake + tables via Teradata's native OTF support using 3-part naming: + + "".""."
" + + Supported model config options: + - catalog_name : name of the catalog integration (from catalogs.yml) + - partitioned_by : partition expression, e.g. 'YEAR(dt), country' + - sorted_by : sort order, e.g. 'id ASC' + - tblproperties : Iceberg table properties, e.g. "'gc.enabled'='true'" + - purge_mode : DROP behavior -- 'NO PURGE' (default, safe) or 'PURGE ALL' +#} + + +{% macro teradata__validate_purge_mode(value) %} + {#- Normalise to upper-case and validate. Returns the validated value so + callers can use the normalised form directly. -#} + {%- set normalized = (value or 'NO PURGE') | trim | upper -%} + {%- if normalized not in ('PURGE ALL', 'NO PURGE') -%} + {{ exceptions.raise_compiler_error( + "Invalid purge_mode '" ~ value ~ "'. Must be 'PURGE ALL' or 'NO PURGE'." + ) }} + {%- endif -%} + {{ return(normalized) }} +{% endmacro %} + + +{% macro teradata__quote_otf_part(part) %} + {#- Quote a single OTF name part, doubling any embedded quote character. + This is the single quoting primitive for External OTF identifiers and + intentionally mirrors TeradataRelation.render() in relation.py, so that + OTF names produced by DDL/DML macros and by relation rendering are always + identical (including names that contain a double-quote). -#} + {%- set qc = '"' -%} + {{ return(qc ~ (part | string | replace(qc, qc ~ qc)) ~ qc) }} +{% endmacro %} + + +{% macro teradata__build_otf_relation_name(catalog_integration, identifier) %} + {#- Canonical builder for the 3-part External OTF relation string: + "".""."" + This is the single source of truth reused by the OTF create, drop, and + incremental macros. Quoting is delegated to teradata__quote_otf_part so it + stays consistent with TeradataRelation.render(). -#} + {%- set datalake = teradata__quote_otf_part(catalog_integration.datalake_name) -%} + {%- set otf_database = teradata__quote_otf_part(catalog_integration.otf_database) -%} + {%- set table_name = teradata__quote_otf_part(identifier) -%} + {{ return(datalake ~ '.' ~ otf_database ~ '.' ~ table_name) }} +{% endmacro %} + + +{% macro teradata__drop_otf_table(catalog_integration, identifier, purge_mode) %} + {#- Drop an OTF table. The purge clause is mandatory for OTF DROP TABLE; + the validator normalises and rejects unsupported values. -#} + {%- set validated_purge_mode = teradata__validate_purge_mode(purge_mode) -%} + {%- set otf_relation = teradata__build_otf_relation_name(catalog_integration, identifier) -%} + {% call statement('drop_otf_table', auto_begin=False) -%} + {#- /*+ IF EXISTS */ is not native OTF DDL syntax — it is a sentinel token + recognised by add_query() in connections.py to suppress error 7825 + (table not found in external catalog) for nonexistent OTF tables. -#} + DROP TABLE /*+ IF EXISTS */ {{ otf_relation }} {{ validated_purge_mode }}; + {%- endcall %} +{% endmacro %} + + +{% macro teradata__create_otf_table_as(relation, sql, catalog_name) %} + {#- Router: dispatch to the catalog-type-specific create macro. -#} + {% set catalog_integration = adapter.get_catalog_integration(catalog_name) %} + + {% if catalog_integration.catalog_type == 'datalake' %} + {{ teradata__create_datalake_table_as(relation, sql, catalog_integration) }} + {% else %} + {{ exceptions.raise_compiler_error( + "Unsupported catalog_type '" ~ catalog_integration.catalog_type ~ "'." + ) }} + {% endif %} + +{% endmacro %} + + +{# Native OTF via DATALAKE: 3-part naming + ----------------------------------------------------------------- + datalake_name -> from catalog_integration (catalogs.yml) + otf_database -> from catalog_integration (catalogs.yml) + table name -> relation.identifier (the dbt model name) + relation.schema is NOT used -- it is the Teradata database, not the OTF database + + Generated SQL pattern: + DROP TABLE /*+ IF EXISTS */ "dl"."db"."tbl" {NO PURGE|PURGE ALL}; + CREATE TABLE "dl"."db"."tbl" + [PARTITIONED BY (...)] + [SORTED BY ...] + [TBLPROPERTIES(...)] + AS (...) WITH DATA; +#} + +{% macro teradata__create_datalake_table_as(relation, sql, catalog_integration) %} + {%- set sql_header = config.get('sql_header', none) -%} + {%- set otf_relation = teradata__build_otf_relation_name(catalog_integration, relation.identifier) -%} + + {# Model-level DDL config options #} + {%- set partitioned_by = config.get('partitioned_by', none) -%} + {%- set sorted_by = config.get('sorted_by', none) -%} + {%- set tblproperties = config.get('tblproperties', none) -%} + {%- set purge_mode = config.get('purge_mode') -%} + + {# Drop existing table before re-creation (idempotent via IF EXISTS hint). + Error 7825 (table not found in external catalog) is suppressed by the + adapter's add_query() error handler in connections.py. Trade-off: this + is non-atomic -- if the subsequent CREATE fails, the table is gone. #} + {{ teradata__drop_otf_table(catalog_integration, relation.identifier, purge_mode) }} + + {{ sql_header if sql_header is not none }} + {% call statement('main') %} + CREATE TABLE {{ otf_relation }} + {% if partitioned_by %}PARTITIONED BY ({{ partitioned_by }}){% endif %} + {% if sorted_by %}SORTED BY {{ sorted_by }}{% endif %} + {% if tblproperties %}TBLPROPERTIES({{ tblproperties }}){% endif %} + AS ({{ sql }}) WITH DATA; + {% endcall %} +{% endmacro %} diff --git a/dbt/include/teradata/macros/materializations/otf/incremental_otf.sql b/dbt/include/teradata/macros/materializations/otf/incremental_otf.sql new file mode 100644 index 00000000..6a702b97 --- /dev/null +++ b/dbt/include/teradata/macros/materializations/otf/incremental_otf.sql @@ -0,0 +1,425 @@ +{# + OTF (Open Table Format) incremental materialization support. + + Handles incremental builds for Iceberg/Delta Lake tables via Teradata's + native OTF support using DATALAKE 3-part naming. + + Supported strategies: + - append : INSERT new rows into the existing OTF table + + Staging uses a REGULAR Teradata table (__dbt_tmp suffix) in the dbt target + schema. We cannot use create_table_as() because teradata__create_table_as + checks config.get('catalog_name') and routes to teradata__create_otf_table_as + whenever catalog_name is set — regardless of the relation type. An OTF + staging table is invisible to DBC.ColumnsV, so get_columns_in_relation() + returns an empty list and the INSERT fails with Error 3706. Instead we issue + native Teradata CREATE TABLE AS ... WITH NO DATA / INSERT SQL directly, + bypassing all config-based routing. The staging table is explicitly dropped + after use. +#} + + +{% macro teradata__validate_get_otf_incremental_strategy(config) %} + {#-- Validate and return the incremental strategy for OTF models. + Only 'append' is currently supported. --#} + {%- set strategy = config.get("incremental_strategy", "append") -%} + + {% if strategy not in ['append'] %} + {{ exceptions.raise_compiler_error( + "Invalid incremental strategy '" ~ strategy ~ "' for OTF models. " + "Only 'append' is currently supported for OTF (Iceberg/Delta Lake) tables. " + "'delete+insert', 'merge', 'valid_history', and 'microbatch' are not yet supported." + ) }} + {% endif %} + + {% do return(strategy) %} +{% endmacro %} + + +{% macro teradata__get_otf_incremental_append_sql(otf_relation_name, tmp_relation, dest_columns) %} + {#-- Append strategy: INSERT INTO OTF table from staging table. + OTF tables require positional inserts (no column list on the target); + specifying column names raises Error 3706 "OTF tables do not support + non-positional inserts". --#} + {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute="name")) -%} + + insert into {{ otf_relation_name }} + select {{ dest_cols_csv }} + from {{ tmp_relation }} + ; +{% endmacro %} + + +{% macro teradata__otf_reconcile_schema(on_schema_change, otf_relation_name, target_columns, staging_columns) %} + {#-- on_schema_change reconciliation for OTF incremental models (Phase 1). + + Args: + on_schema_change : 'ignore' | 'fail' | 'append_new_columns' + otf_relation_name: rendered 3-part OTF name + target_columns : list of column NAME strings currently on the OTF table + (from adapter.get_otf_columns_in_relation) + staging_columns : list of Column objects from the staging table -- the + incoming schema, carrying name + DDL type + + Returns: + none -> caller should use the default positional append + list -> positional SELECT expressions aligned to the final OTF column + order (used when columns were added under append_new_columns) + + Comparison is case-insensitive. New columns are added one ALTER at a time + because OTF cannot combine ALTER operations and External OTF disallows + multi-statement requests. --#} + + {%- set target_lower = [] -%} + {%- for t in target_columns -%}{%- do target_lower.append(t | lower) -%}{%- endfor -%} + {%- set staging_lower = [] -%} + {%- for s in staging_columns -%}{%- do staging_lower.append(s.name | lower) -%}{%- endfor -%} + + {#-- new_columns: in staging but not in target; removed_columns: target not in staging --#} + {%- set new_columns = [] -%} + {%- for s in staging_columns -%} + {%- if (s.name | lower) not in target_lower -%}{%- do new_columns.append(s) -%}{%- endif -%} + {%- endfor -%} + {%- set removed_columns = [] -%} + {%- for t in target_columns -%} + {%- if (t | lower) not in staging_lower -%}{%- do removed_columns.append(t) -%}{%- endif -%} + {%- endfor -%} + + {%- if on_schema_change == 'ignore' -%} + {{ return(none) }} + + {%- elif on_schema_change == 'fail' -%} + {%- if (new_columns | length > 0) or (removed_columns | length > 0) -%} + {{ exceptions.raise_compiler_error( + "Schema change detected on OTF incremental model with on_schema_change='fail'. " + ~ "Columns added in source: [" ~ (new_columns | map(attribute='name') | join(', ')) ~ "]; " + ~ "columns missing from source: [" ~ (removed_columns | join(', ')) ~ "]. " + ~ "Reconcile the model, switch to on_schema_change='append_new_columns', " + ~ "or run with --full-refresh." + ) }} + {%- endif -%} + {#-- No drift: still build a name-aligned positional SELECT so a reorder of + existing columns in the model SELECT cannot misalign the OTF insert. --#} + {%- set exprs = [] -%} + {%- for cname in target_columns -%} + {%- set ns = namespace(match=none) -%} + {%- for s in staging_columns -%} + {%- if (s.name | lower) == (cname | lower) -%}{%- set ns.match = s -%}{%- endif -%} + {%- endfor -%} + {%- do exprs.append(adapter.quote(ns.match.name)) -%} + {%- endfor -%} + {{ return(exprs) }} + + {%- elif on_schema_change == 'append_new_columns' -%} + {#-- Add each new source column with its own ALTER (this loop is a no-op when + there are no new columns). New columns land at the end of the OTF table. + Type is sourced from the staging column. --#} + {%- for col in new_columns -%} + {% call statement('otf_add_column_' ~ loop.index, auto_begin=False) -%} + ALTER TABLE {{ otf_relation_name }} ADD {{ adapter.quote(col.name) }} {{ col.data_type }}; + {%- endcall %} + {%- endfor -%} + + {#-- Always build a positional SELECT aligned to the final OTF column order -- + existing target columns in their current order, then the newly added + columns -- even when the column set did not change. OTF inserts are + column-list-less/positional, so aligning by name here makes a *reorder* + of existing columns in the model SELECT safe (it cannot misalign data). + A target column no longer present in the source (append_new_columns keeps + such columns) is set to NULL. --#} + {%- set final_order = target_columns + (new_columns | map(attribute='name') | list) -%} + {%- set exprs = [] -%} + {%- for cname in final_order -%} + {%- set ns = namespace(match=none) -%} + {%- for s in staging_columns -%} + {%- if (s.name | lower) == (cname | lower) -%}{%- set ns.match = s -%}{%- endif -%} + {%- endfor -%} + {%- if ns.match is not none -%} + {%- do exprs.append(adapter.quote(ns.match.name)) -%} + {%- else -%} + {%- do exprs.append('NULL') -%} + {%- endif -%} + {%- endfor -%} + {{ return(exprs) }} + {%- endif -%} +{% endmacro %} + + +{% macro teradata__otf_sync_all_columns(otf_relation_name, target_columns, staging_columns) %} + {#-- sync_all_columns reconciliation for OTF incremental models (best-effort). + + Args: + otf_relation_name: rendered 3-part OTF name + target_columns : list of {name, otf_type} from get_otf_column_types + (names lower-cased; otf_type is the Iceberg type) + staging_columns : list of Column objects -- the incoming schema + + Makes the OTF table match the source: ADD new columns, DROP columns no + longer in the source (destructive), and MODIFY columns whose OTF/Iceberg + type changed *and* is a promotion OTF allows. A type change OTF cannot + apply in place raises a clear error (use --full-refresh). + + Type comparison is at OTF/Iceberg granularity, so VARCHAR length and + SMALLINT-vs-INTEGER differences are not treated as changes. Each change is + its own ALTER (OTF cannot combine alter ops / no multi-statement requests; + no rollback if one fails midway). + + Returns the positional SELECT expressions aligned to the final OTF column + order (surviving target columns in order, then newly added columns). --#} + + {%- set target_names = [] -%} + {%- set target_type_by_name = {} -%} + {%- for c in target_columns -%} + {%- do target_names.append(c.name) -%} + {%- do target_type_by_name.update({c.name | lower: c.otf_type}) -%} + {%- endfor -%} + {%- set target_lower = [] -%} + {%- for n in target_names -%}{%- do target_lower.append(n | lower) -%}{%- endfor -%} + {%- set staging_lower = [] -%} + {%- for s in staging_columns -%}{%- do staging_lower.append(s.name | lower) -%}{%- endfor -%} + + {#-- Compute the full diff FIRST, without running any DDL: columns to add, to + drop, and to retype. Type changes are validated here so an unsupported one + raises BEFORE any ALTER executes -- otherwise a partial ADD/DROP could be + left behind, and OTF has no rollback. --#} + {%- set new_columns = [] -%} + {%- for s in staging_columns -%} + {%- if (s.name | lower) not in target_lower -%}{%- do new_columns.append(s) -%}{%- endif -%} + {%- endfor -%} + + {%- set removed_columns = [] -%} + {%- for n in target_names -%} + {%- if (n | lower) not in staging_lower -%}{%- do removed_columns.append(n) -%}{%- endif -%} + {%- endfor -%} + + {%- set modify_columns = [] -%} + {%- for s in staging_columns -%} + {%- if (s.name | lower) in target_lower -%} + {%- set old_type = target_type_by_name[s.name | lower] -%} + {%- set new_type = adapter.teradata_type_to_otf_type(s.data_type) -%} + {%- if old_type != new_type -%} + {%- if adapter.otf_type_promotion_allowed(old_type, new_type) -%} + {%- do modify_columns.append(s) -%} + {%- else -%} + {{ exceptions.raise_compiler_error( + "on_schema_change='sync_all_columns': column '" ~ s.name ~ "' changed type from '" + ~ old_type ~ "' to '" ~ new_type ~ "', which OTF/Iceberg cannot apply in place. " + ~ "Run the model with --full-refresh to rebuild it." + ) }} + {%- endif -%} + {%- endif -%} + {%- endif -%} + {%- endfor -%} + + {#-- Apply the changes, one ALTER each (OTF cannot combine alter ops / no + multi-statement requests; no rollback if one fails midway). DROP is + destructive -- the column and its data are removed. --#} + {%- for col in new_columns -%} + {% call statement('otf_sync_add_' ~ loop.index, auto_begin=False) -%} + ALTER TABLE {{ otf_relation_name }} ADD {{ adapter.quote(col.name) }} {{ col.data_type }}; + {%- endcall %} + {%- endfor -%} + {%- for col in modify_columns -%} + {% call statement('otf_sync_modify_' ~ loop.index, auto_begin=False) -%} + ALTER TABLE {{ otf_relation_name }} MODIFY {{ adapter.quote(col.name) }} {{ col.data_type }}; + {%- endcall %} + {%- endfor -%} + {%- for col in removed_columns -%} + {% call statement('otf_sync_drop_' ~ loop.index, auto_begin=False) -%} + ALTER TABLE {{ otf_relation_name }} DROP {{ adapter.quote(col) }}; + {%- endcall %} + {%- endfor -%} + + {#-- Final OTF column order = surviving target columns (original order, minus + dropped) then the newly added columns. Build a positional SELECT in that + order; every column is present in the source (dropped ones are gone). --#} + {%- set final_order = [] -%} + {%- for n in target_names -%} + {%- if (n | lower) in staging_lower -%}{%- do final_order.append(n) -%}{%- endif -%} + {%- endfor -%} + {%- for col in new_columns -%}{%- do final_order.append(col.name) -%}{%- endfor -%} + + {%- set exprs = [] -%} + {%- for cname in final_order -%} + {%- set ns = namespace(match=none) -%} + {%- for s in staging_columns -%} + {%- if (s.name | lower) == (cname | lower) -%}{%- set ns.match = s -%}{%- endif -%} + {%- endfor -%} + {%- do exprs.append(adapter.quote(ns.match.name)) -%} + {%- endfor -%} + {{ return(exprs) }} +{% endmacro %} + + +{% macro teradata__incremental_otf(catalog_name, sql) %} + {#-- Main entry point for OTF incremental materialization. --#} + + {#-- Validate strategy (currently only 'append' is supported) --#} + {%- do teradata__validate_get_otf_incremental_strategy(config) -%} + + {#-- Guardrails: reject Teradata-native options that do not apply to OTF --#} + {%- set unsupported = [] -%} + {%- if config.get('table_kind') -%}{%- do unsupported.append('table_kind') -%}{%- endif -%} + {%- if config.get('table_option') -%}{%- do unsupported.append('table_option') -%}{%- endif -%} + {%- if config.get('with_statistics', default=False) | as_bool -%}{%- do unsupported.append('with_statistics') -%}{%- endif -%} + {%- if config.get('index') -%}{%- do unsupported.append('index') -%}{%- endif -%} + {%- if unsupported | length > 0 -%} + {{ exceptions.raise_compiler_error( + "The following config option(s) are not supported with catalog_name (OTF): " + ~ unsupported | join(', ') + ) }} + {%- endif -%} + + {#-- on_schema_change handling for OTF incremental. + Supported: 'ignore' (default), 'fail', 'append_new_columns', 'sync_all_columns'. + + 'sync_all_columns' is best-effort on OTF: it ADDs new source columns, + DROPs columns missing from the source (destructive -- OTF has no + rollback), and applies column TYPE changes that OTF/Iceberg permits + (e.g. int->long, decimal precision widening). A type change OTF cannot + apply in place raises a clear error directing the user to --full-refresh. + Type comparison is done at OTF/Iceberg type granularity (via HELP TABLE + 'OTF Type'), so VARCHAR length and SMALLINT-vs-INTEGER -- which OTF does + not preserve -- are never treated as changes. + Reconciliation runs in the incremental (append) branch below via + teradata__otf_reconcile_schema() (ignore/fail/append_new_columns) or + teradata__otf_sync_all_columns() (sync_all_columns). --#} + {%- set on_schema_change = config.get('on_schema_change', 'ignore') or 'ignore' -%} + {%- set _otf_supported_osc = ['ignore', 'fail', 'append_new_columns', 'sync_all_columns'] -%} + {%- if on_schema_change not in _otf_supported_osc -%} + {{ exceptions.raise_compiler_error( + "Invalid on_schema_change='" ~ on_schema_change ~ "' for OTF incremental models. " + ~ "Supported values: " ~ (_otf_supported_osc | join(', ')) ~ "." + ) }} + {%- endif -%} + + {%- set contract_config = config.get('contract') -%} + {%- if contract_config is not none and contract_config.enforced -%} + {{ exceptions.raise_compiler_error( + "Model contracts (contract.enforced=true) are not yet supported with catalog_name (OTF)." + ) }} + {%- endif -%} + + {#-- Resolve target and check existence --#} + {%- set catalog_integration = adapter.get_catalog_integration(catalog_name) -%} + {%- set target_relation = this.incorporate(type='table') -%} + + {#-- OTF tables are not registered in DBC.TablesV under the dbt target schema, + so load_relation(this) always returns none and would cause every run to + take the full-refresh (CREATE) path. Instead, probe via SAMPLE 0 against + the 3-part DATALAKE name: succeeds when the table exists, raises + Teradata Error 7825 (ICEBERG_EXPORT "Table does not exist") when it does + not. The adapter method catches the exception and returns True/False. --#} + {%- set otf_exists = adapter.otf_relation_exists( + catalog_integration.datalake_name, + catalog_integration.otf_database, + target_relation.identifier) -%} + + {{ run_hooks(pre_hooks, inside_transaction=False) }} + {{ run_hooks(pre_hooks, inside_transaction=True) }} + + {% if not otf_exists or should_full_refresh() %} + {#-- FIRST RUN or --full-refresh: create OTF table from scratch + (DROP + CREATE AS via the existing table-materialization macro). --#} + {{ teradata__create_otf_table_as(target_relation, sql, catalog_name) }} + + {% else %} + {#-- INCREMENTAL RUN --#} + + {#-- Step 1: Create a REGULAR (non-OTF) staging table in the dbt target schema. + We cannot call create_table_as(True, tmp_relation, sql) because + teradata__create_table_as checks config.get('catalog_name') and always + routes to teradata__create_otf_table_as when catalog_name is set — + regardless of the relation type. An OTF staging table is invisible to + DBC.ColumnsV, so adapter.get_columns_in_relation() returns [] and the + INSERT into the OTF target fails with Error 3706 (empty column list). + Instead, issue native Teradata DDL directly into target.schema (a regular + Teradata database) to bypass config-based routing entirely. The staging + data is just SQL rows; Parquet serialisation only happens at the final + INSERT INTO the OTF target. --#} + {%- set tmp_relation = make_temp_relation(api.Relation.create( + schema=target.schema, + identifier=target_relation.identifier, + type='table' + )) -%} + {% call statement('drop_preexisting_staging', auto_begin=False) %} + DROP TABLE /*+ IF EXISTS */ {{ tmp_relation }}; + {% endcall %} + {% call statement('create_staging', auto_begin=False) %} + CREATE MULTISET TABLE {{ tmp_relation }} AS ({{ sql }}) WITH NO DATA; + {% endcall %} + {% call statement('populate_staging') %} + INSERT INTO {{ tmp_relation }} + {{ sql }} + ; + {% endcall %} + + {#-- Step 2: Get destination columns from the staging table. + The staging table is a regular Teradata table so DBC.ColumnsV has its schema. --#} + {%- set dest_columns = adapter.get_columns_in_relation(tmp_relation) -%} + + {#-- Step 3: Build the 3-part OTF relation name --#} + {%- set otf_relation_name = teradata__build_otf_relation_name(catalog_integration, target_relation.identifier) -%} + + {#-- Step 3a: on_schema_change reconciliation. + For the default 'ignore' there is nothing to reconcile, so we skip the + OTF column probe entirely (it is an extra SAMPLE 0 round-trip against + the DATALAKE). For 'fail'/'append_new_columns', read the OTF target's + current columns via SAMPLE 0 metadata (DBC.ColumnsV does not see OTF + tables), diff against the staging schema, and for 'append_new_columns' + issue one ALTER TABLE ADD per new column. teradata__otf_reconcile_schema + returns the positional SELECT expression list aligned to the final OTF + column order, or none to use the default positional append. --#} + {%- if on_schema_change == 'ignore' -%} + {%- set insert_exprs = none -%} + {%- elif on_schema_change == 'sync_all_columns' -%} + {#-- sync needs column TYPES, read from HELP TABLE (OTF Type). --#} + {%- set target_typed_columns = adapter.get_otf_column_types( + catalog_integration.datalake_name, + catalog_integration.otf_database, + target_relation.identifier) -%} + {%- set insert_exprs = teradata__otf_sync_all_columns( + otf_relation_name, target_typed_columns, dest_columns) -%} + {%- else -%} + {#-- fail / append_new_columns need column NAMES only. --#} + {%- set target_otf_columns = adapter.get_otf_columns_in_relation( + catalog_integration.datalake_name, + catalog_integration.otf_database, + target_relation.identifier) -%} + {%- set insert_exprs = teradata__otf_reconcile_schema( + on_schema_change, otf_relation_name, target_otf_columns, dest_columns) -%} + {%- endif -%} + + {% call statement('main') %} + {%- if insert_exprs is none -%} + {{ teradata__get_otf_incremental_append_sql(otf_relation_name, tmp_relation, dest_columns) }} + {%- else -%} + insert into {{ otf_relation_name }} + select {{ insert_exprs | join(', ') }} + from {{ tmp_relation }} + ; + {%- endif -%} + {% endcall %} + + {#-- Step 4: Cleanup staging table --#} + {% call statement('drop_staging', auto_begin=False) %} + DROP TABLE /*+ IF EXISTS */ {{ tmp_relation }}; + {% endcall %} + + {% endif %} + + {#-- Grants warning — not supported on OTF --#} + {%- if config.get('grants') -%} + {{ exceptions.warn("grants config is ignored for OTF models — Teradata does not support GRANT on DATALAKE tables.") }} + {%- endif -%} + + {% do persist_docs(target_relation, model) %} + + {{ run_hooks(post_hooks, inside_transaction=True) }} + {% do adapter.commit() %} + {% do adapter.cache_added(target_relation) %} + {{ run_hooks(post_hooks, inside_transaction=False) }} + + {{ return({'relations': [target_relation]}) }} + +{% endmacro %} diff --git a/dbt/include/teradata/macros/materializations/snapshot/snapshot.sql b/dbt/include/teradata/macros/materializations/snapshot/snapshot.sql index bb2f7c00..dae95e8d 100644 --- a/dbt/include/teradata/macros/materializations/snapshot/snapshot.sql +++ b/dbt/include/teradata/macros/materializations/snapshot/snapshot.sql @@ -3,6 +3,13 @@ -- calling the macro set_query_band() which will set the query_band for this materialization as per the user_configuration {% do set_query_band() %} + {%- if config.get('catalog_name') -%} + {{ exceptions.raise_compiler_error( + "catalog_name (OTF) is not supported with the 'snapshot' materialization. " + "Only the 'table' materialization supports OTF tables." + ) }} + {%- endif -%} + {%- set target_table = model.get('alias', model.get('name')) -%} {%- set strategy_name = config.get('strategy') -%} diff --git a/dbt/include/teradata/macros/materializations/table/table.sql b/dbt/include/teradata/macros/materializations/table/table.sql index 228640c2..3ed0fcd6 100644 --- a/dbt/include/teradata/macros/materializations/table/table.sql +++ b/dbt/include/teradata/macros/materializations/table/table.sql @@ -2,7 +2,62 @@ -- calling the macro set_query_band() which will set the query_band for this materialization as per the user_configuration {% do set_query_band() %} - {% set relations = materialization_table_default() %} -- calling the default table materialization from dbt-core - {{ return(relations) }} -{%- endmaterialization -%} \ No newline at end of file + {%- set catalog_name = config.get('catalog_name', none) -%} + {% if catalog_name is not none %} + {#-- OTF models: create directly at target, skip intermediate/rename pattern. + DATALAKE tables use 3-part naming that can't be renamed via standard DDL, + so we cannot use the build-tmp-then-rename pattern from the default + table materialization. Trade-off: a failed CREATE leaves the target + dropped (non-atomic re-materialization). --#} + {#- Guard against config combinations that are not supported on the OTF path. + Teradata-native options do not apply to Iceberg/Delta tables. -#} + {%- set unsupported = [] -%} + {%- if config.get('table_kind') -%}{%- do unsupported.append('table_kind') -%}{%- endif -%} + {%- if config.get('table_option') -%}{%- do unsupported.append('table_option') -%}{%- endif -%} + {%- if config.get('with_statistics', default=False) | as_bool -%}{%- do unsupported.append('with_statistics') -%}{%- endif -%} + {%- if config.get('index') -%}{%- do unsupported.append('index') -%}{%- endif -%} + {%- if unsupported | length > 0 -%} + {{ exceptions.raise_compiler_error( + "The following config option(s) are not supported with catalog_name (OTF): " + ~ unsupported | join(', ') + ) }} + {%- endif -%} + {%- set contract_config = config.get('contract') -%} + {%- if contract_config is not none and contract_config.enforced -%} + {{ exceptions.raise_compiler_error( + "Model contracts (contract.enforced=true) are not yet supported with catalog_name (OTF)." + ) }} + {%- endif -%} + + {%- set target_relation = this.incorporate(type='table') -%} + + {{ run_hooks(pre_hooks, inside_transaction=False) }} + {{ run_hooks(pre_hooks, inside_transaction=True) }} + + {{ teradata__create_otf_table_as(target_relation, sql, catalog_name) }} + + {{ run_hooks(post_hooks, inside_transaction=True) }} + + {% do persist_docs(target_relation, model) %} + + {#- Teradata does not support GRANT on OTF tables (3-part names are + invalid in GRANT syntax, and OTF objects are not in DBC.AllRights). + Access control for OTF tables is managed via AUTHORIZATION objects + and external IAM/OAuth policies. -#} + {%- if config.get('grants') -%} + {{ exceptions.warn("grants config is ignored for OTF models — Teradata does not support GRANT on DATALAKE tables.") }} + {%- endif -%} + + {% do adapter.commit() %} + {% do adapter.cache_added(target_relation) %} + + {{ run_hooks(post_hooks, inside_transaction=False) }} + + {{ return({'relations': [target_relation]}) }} + {% else %} + {% set relations = materialization_table_default() %} -- calling the default table materialization from dbt-core + {{ return(relations) }} + {% endif %} + +{%- endmaterialization -%} diff --git a/pytest.ini b/pytest.ini index d01ba123..57075fd3 100644 --- a/pytest.ini +++ b/pytest.ini @@ -7,5 +7,7 @@ filterwarnings = # this allows you to store env vars for database connection in a file named test.env # rather than passing them in every CLI command, or setting in `PYTEST_ADDOPTS` # be sure to add "test.env" to .gitignore as well! +# test paths: unit tests (no database) and functional tests testpaths = - tests/functional # name per convention \ No newline at end of file + tests/unit + tests/functional diff --git a/requirements_dev.txt b/requirements_dev.txt index 50e8d8cc..569f1ea8 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -10,6 +10,7 @@ pylava~=0.3.0 teradatasql>=20.00.00.10 dbt-adapters>=1.17.2 dbt-common>=1.13,<2.0 +dbt-core>=1.10.0,<2.0 MarkupSafe==2.0.1 pytest-dotenv pytest-cov diff --git a/tests/conftest.py b/tests/conftest.py index efc446be..35a1d383 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,6 +8,24 @@ #load_dotenv("../test.env") +# Environment variables read by the test suite: +# +# Connection (with defaults for Vantage Express running on localhost): +# DBT_TERADATA_SERVER_NAME -- hostname (default: 'localhost') +# DBT_TERADATA_USERNAME -- username (default: 'dbc') +# DBT_TERADATA_PASSWORD -- password (default: 'dbc') +# DBT_TERADATA_TMODE -- transaction mode (default: 'ANSI') +# +# Grants tests (created automatically if set): +# DBT_TEST_USER_1 +# DBT_TEST_USER_2 +# DBT_TEST_USER_3 +# +# OTF / DATALAKE tests (tests/functional/adapter/test_otf_integration.py). +# When BOTH are set, OTF integration tests run; otherwise they are skipped: +# DBT_TERADATA_DATALAKE -- pre-created DATALAKE object name +# DBT_TERADATA_OTF_DATABASE -- pre-created OTF database within the DATALAKE + # Import the standard functional fixtures as a plugin # Note: fixtures with session scope need to be locals pytest_plugins = ["dbt.tests.fixtures.project"] diff --git a/tests/functional/adapter/test_otf_guardrails.py b/tests/functional/adapter/test_otf_guardrails.py new file mode 100644 index 00000000..7db013d7 --- /dev/null +++ b/tests/functional/adapter/test_otf_guardrails.py @@ -0,0 +1,462 @@ +"""Compile-time guardrail tests for OTF config validation. + +These tests verify that unsupported config combinations on the OTF path +are rejected at compile time. They do NOT require a real DATALAKE or OTF +database and are never skipped. + +Scenarios covered: + 1. table_kind with catalog_name is rejected + 2. Invalid purge_mode is rejected + 3. table_option with catalog_name is rejected + 4. with_statistics with catalog_name is rejected + 5. index with catalog_name is rejected + 6. Multiple unsupported options reported together + 7. OTF incremental: unsupported strategy (merge) is rejected + 8. OTF incremental: delete+insert is rejected (not yet supported) + 9. OTF incremental: native-only options (table_kind) are rejected + 10. OTF incremental: contract.enforced is rejected + 11. OTF incremental: valid_history strategy is rejected + 12. OTF incremental: microbatch strategy is rejected + 13. snapshot materialization with catalog_name is rejected + 14. contract.enforced with catalog_name on table materialization is rejected +""" + +import pytest + +from dbt.tests.adapter.catalog_integrations.test_catalog_integration import ( + BaseCatalogIntegrationValidation, +) +from dbt.tests.util import run_dbt + + +CATALOG_NAME = "test_catalog" + +# Placeholder values — no real DATALAKE is needed because the errors fire +# before any SQL is sent to the database. +CATALOGS_CONFIG = { + "catalogs": [ + { + "name": CATALOG_NAME, + "active_write_integration": "td_datalake", + "write_integrations": [ + { + "name": "td_datalake", + "catalog_type": "datalake", + "adapter_properties": { + "datalake_name": "placeholder_datalake", + "otf_database": "placeholder_otf_db", + }, + } + ], + } + ] +} + + +# --------------------------------------------------------------------------- +# Model SQL fixtures for unsupported config combos +# --------------------------------------------------------------------------- + +unsupported_table_kind_sql = f""" +{{{{ config( + materialized='table', + catalog_name='{CATALOG_NAME}', + table_kind='SET' +) }}}} +select 1 as id +""" + +invalid_purge_mode_sql = f""" +{{{{ config( + materialized='table', + catalog_name='{CATALOG_NAME}', + purge_mode='SOFT' +) }}}} +select 1 as id +""" + +unsupported_table_option_sql = f""" +{{{{ config( + materialized='table', + catalog_name='{CATALOG_NAME}', + table_option='NO FALLBACK' +) }}}} +select 1 as id +""" + +unsupported_with_statistics_sql = f""" +{{{{ config( + materialized='table', + catalog_name='{CATALOG_NAME}', + with_statistics='true' +) }}}} +select 1 as id +""" + +unsupported_index_sql = f""" +{{{{ config( + materialized='table', + catalog_name='{CATALOG_NAME}', + index='PRIMARY INDEX (id)' +) }}}} +select 1 as id +""" + +unsupported_multiple_options_sql = f""" +{{{{ config( + materialized='table', + catalog_name='{CATALOG_NAME}', + table_kind='MULTISET', + index='PRIMARY INDEX (id)', + with_statistics='true' +) }}}} +select 1 as id +""" + +incremental_with_catalog_sql = f""" +{{{{ config( + materialized='incremental', + catalog_name='{CATALOG_NAME}', + unique_key='id' +) }}}} +select 1 as id +""" + +incremental_otf_merge_strategy_sql = f""" +{{{{ config( + materialized='incremental', + catalog_name='{CATALOG_NAME}', + incremental_strategy='merge', + unique_key='id' +) }}}} +select 1 as id +""" + +incremental_otf_delete_insert_strategy_sql = f""" +{{{{ config( + materialized='incremental', + catalog_name='{CATALOG_NAME}', + incremental_strategy='delete+insert', + unique_key='id' +) }}}} +select 1 as id +""" + +incremental_otf_with_table_kind_sql = f""" +{{{{ config( + materialized='incremental', + catalog_name='{CATALOG_NAME}', + table_kind='SET' +) }}}} +select 1 as id +""" + +incremental_otf_with_contract_sql = f""" +{{{{ config( + materialized='incremental', + catalog_name='{CATALOG_NAME}', + contract={{'enforced': true}}, + on_schema_change='fail' +) }}}} +select 1 as id +""" + +incremental_otf_valid_history_strategy_sql = f""" +{{{{ config( + materialized='incremental', + catalog_name='{CATALOG_NAME}', + incremental_strategy='valid_history', + unique_key='id' +) }}}} +select 1 as id +""" + +incremental_otf_microbatch_strategy_sql = f""" +{{{{ config( + materialized='incremental', + catalog_name='{CATALOG_NAME}', + incremental_strategy='microbatch', + unique_key='id', + event_time='created_at', + batch_size='day', + begin='2020-01-01' +) }}}} +select 1 as id, current_timestamp as created_at +""" + +snapshot_with_catalog_sql = f""" +{{% snapshot snapshot_otf %}} +{{{{ config( + catalog_name='{CATALOG_NAME}', + strategy='timestamp', + unique_key='id', + updated_at='updated_at' +) }}}} +select 1 as id, current_timestamp as updated_at +{{% endsnapshot %}} +""" + +contract_with_catalog_sql = f""" +{{{{ config( + materialized='table', + catalog_name='{CATALOG_NAME}', + contract={{'enforced': true}} +) }}}} +select 1 as id +""" + + +# =================================================================== +# Original guardrail tests: table_kind and purge_mode +# =================================================================== + +class TestOTFCompileTimeErrors(BaseCatalogIntegrationValidation): + """These exercise guardrails for unsupported config combinations on the OTF + path. The checks fire inside the materialization macro, so they require + ``dbt run`` (not ``compile``). A real DATALAKE does not need to exist + because the error is raised before any SQL is sent to the database. + """ + + @pytest.fixture(scope="class") + def catalogs(self): + return CATALOGS_CONFIG + + @pytest.fixture(scope="class") + def models(self): + return { + "bad_table_kind.sql": unsupported_table_kind_sql, + "bad_purge_mode.sql": invalid_purge_mode_sql, + } + + def test_table_kind_with_catalog_name_fails(self, project): + results = run_dbt( + ["run", "--select", "bad_table_kind"], expect_pass=False + ) + assert any("table_kind" in str(r.message or "") for r in results) + + def test_invalid_purge_mode_fails(self, project): + results = run_dbt( + ["run", "--select", "bad_purge_mode"], expect_pass=False + ) + assert any("purge_mode" in str(r.message or "").lower() for r in results) + + +# =================================================================== +# Extended guardrails: table_option, with_statistics, index +# =================================================================== + +class TestOTFUnsupportedTableOptions(BaseCatalogIntegrationValidation): + """Verify that table_option, with_statistics, and index configs are + all rejected when catalog_name is set (OTF path). + """ + + @pytest.fixture(scope="class") + def catalogs(self): + return CATALOGS_CONFIG + + @pytest.fixture(scope="class") + def models(self): + return { + "bad_table_option.sql": unsupported_table_option_sql, + "bad_with_statistics.sql": unsupported_with_statistics_sql, + "bad_index.sql": unsupported_index_sql, + "bad_multiple.sql": unsupported_multiple_options_sql, + } + + def test_table_option_with_catalog_name_fails(self, project): + results = run_dbt( + ["run", "--select", "bad_table_option"], expect_pass=False + ) + assert any("table_option" in str(r.message or "") for r in results) + + def test_with_statistics_with_catalog_name_fails(self, project): + results = run_dbt( + ["run", "--select", "bad_with_statistics"], expect_pass=False + ) + assert any("with_statistics" in str(r.message or "") for r in results) + + def test_index_with_catalog_name_fails(self, project): + results = run_dbt( + ["run", "--select", "bad_index"], expect_pass=False + ) + assert any("index" in str(r.message or "") for r in results) + + def test_multiple_unsupported_options_reported(self, project): + """When multiple unsupported options are set, the error message + includes all of them so users can fix everything in one pass.""" + results = run_dbt( + ["run", "--select", "bad_multiple"], expect_pass=False + ) + msg = str(results[0].message or "") + assert "table_kind" in msg + assert "index" in msg + assert "with_statistics" in msg + + +# =================================================================== +# Materialization-level guardrails: incremental OTF and snapshot +# =================================================================== + +class TestOTFIncrementalUnsupportedStrategy(BaseCatalogIntegrationValidation): + """Verify that unsupported incremental strategies are rejected for OTF.""" + + @pytest.fixture(scope="class") + def catalogs(self): + return CATALOGS_CONFIG + + @pytest.fixture(scope="class") + def models(self): + return {"incremental_otf_merge.sql": incremental_otf_merge_strategy_sql} + + def test_merge_strategy_with_otf_fails(self, project): + results = run_dbt( + ["run", "--select", "incremental_otf_merge"], expect_pass=False + ) + msg = " ".join(str(r.message or "") for r in results) + assert "merge" in msg.lower() or "invalid incremental strategy" in msg.lower() + + +class TestOTFIncrementalDeleteInsertBlocked(BaseCatalogIntegrationValidation): + """Verify that delete+insert strategy is rejected for OTF (not yet supported).""" + + @pytest.fixture(scope="class") + def catalogs(self): + return CATALOGS_CONFIG + + @pytest.fixture(scope="class") + def models(self): + return {"incremental_otf_di.sql": incremental_otf_delete_insert_strategy_sql} + + def test_delete_insert_strategy_with_otf_fails(self, project): + results = run_dbt( + ["run", "--select", "incremental_otf_di"], expect_pass=False + ) + msg = " ".join(str(r.message or "") for r in results) + assert "delete+insert" in msg.lower() or "invalid incremental strategy" in msg.lower() + + +class TestOTFIncrementalUnsupportedOptions(BaseCatalogIntegrationValidation): + """Verify that native-only config options (table_kind, etc.) are rejected + for OTF incremental models.""" + + @pytest.fixture(scope="class") + def catalogs(self): + return CATALOGS_CONFIG + + @pytest.fixture(scope="class") + def models(self): + return {"incremental_otf_table_kind.sql": incremental_otf_with_table_kind_sql} + + def test_table_kind_with_otf_incremental_fails(self, project): + results = run_dbt( + ["run", "--select", "incremental_otf_table_kind"], expect_pass=False + ) + msg = " ".join(str(r.message or "") for r in results) + assert "table_kind" in msg.lower() + + +class TestOTFIncrementalContractBlocked(BaseCatalogIntegrationValidation): + """Verify that contract.enforced=true is rejected on OTF incremental models.""" + + @pytest.fixture(scope="class") + def catalogs(self): + return CATALOGS_CONFIG + + @pytest.fixture(scope="class") + def models(self): + return {"incremental_otf_contract.sql": incremental_otf_with_contract_sql} + + def test_contract_with_otf_incremental_fails(self, project): + results = run_dbt( + ["run", "--select", "incremental_otf_contract"], expect_pass=False + ) + msg = " ".join(str(r.message or "") for r in results) + assert "contract" in msg.lower() + + +class TestOTFIncrementalValidHistoryBlocked(BaseCatalogIntegrationValidation): + """Verify that valid_history strategy is rejected for OTF.""" + + @pytest.fixture(scope="class") + def catalogs(self): + return CATALOGS_CONFIG + + @pytest.fixture(scope="class") + def models(self): + return {"incremental_otf_vh.sql": incremental_otf_valid_history_strategy_sql} + + def test_valid_history_strategy_with_otf_fails(self, project): + results = run_dbt( + ["run", "--select", "incremental_otf_vh"], expect_pass=False + ) + msg = " ".join(str(r.message or "") for r in results) + assert "valid_history" in msg.lower() or "invalid incremental strategy" in msg.lower() + + +class TestOTFIncrementalMicrobatchBlocked(BaseCatalogIntegrationValidation): + """Verify that microbatch strategy is rejected for OTF.""" + + @pytest.fixture(scope="class") + def catalogs(self): + return CATALOGS_CONFIG + + @pytest.fixture(scope="class") + def models(self): + return {"incremental_otf_mb.sql": incremental_otf_microbatch_strategy_sql} + + def test_microbatch_strategy_with_otf_fails(self, project): + results = run_dbt( + ["run", "--select", "incremental_otf_mb"], expect_pass=False + ) + # Microbatch wraps execution in batches — the compilation error from + # our OTF guardrail fires inside the first batch, so r.message is just + # "ERROR" rather than the full text. Asserting on error status is + # sufficient; the guardrail message is visible in the dbt log output. + assert len(results) == 1 + assert results[0].status == "error" + + +class TestOTFSnapshotBlocked(BaseCatalogIntegrationValidation): + """Verify that the snapshot materialization rejects catalog_name.""" + + @pytest.fixture(scope="class") + def catalogs(self): + return CATALOGS_CONFIG + + @pytest.fixture(scope="class") + def snapshots(self): + return {"snapshot_otf.sql": snapshot_with_catalog_sql} + + @pytest.fixture(scope="class") + def models(self): + return {} + + def test_snapshot_with_catalog_name_fails(self, project): + results = run_dbt( + ["snapshot"], expect_pass=False + ) + msg = " ".join(str(r.message or "") for r in results) + assert "catalog_name" in msg.lower() or "otf" in msg.lower() + + +# =================================================================== +# Contract guardrail: contract.enforced with OTF +# =================================================================== + +class TestOTFContractBlocked(BaseCatalogIntegrationValidation): + """Verify that contract.enforced=true is rejected on OTF models.""" + + @pytest.fixture(scope="class") + def catalogs(self): + return CATALOGS_CONFIG + + @pytest.fixture(scope="class") + def models(self): + return {"contract_otf.sql": contract_with_catalog_sql} + + def test_contract_enforced_with_catalog_name_fails(self, project): + results = run_dbt( + ["run", "--select", "contract_otf"], expect_pass=False + ) + msg = " ".join(str(r.message or "") for r in results) + assert "contract" in msg.lower() diff --git a/tests/functional/adapter/test_otf_integration.py b/tests/functional/adapter/test_otf_integration.py new file mode 100644 index 00000000..edb08bb0 --- /dev/null +++ b/tests/functional/adapter/test_otf_integration.py @@ -0,0 +1,1726 @@ +"""End-to-end functional tests for OTF (Iceberg/Delta Lake) materialization. + +These tests require a Teradata instance with a pre-created DATALAKE object +and OTF database. They are gated on env vars and skipped otherwise so the +file is safe to include in the standard pytest run. + +Required env vars: + DBT_TERADATA_DATALAKE -- name of the pre-created DATALAKE object + DBT_TERADATA_OTF_DATABASE -- name of the pre-created OTF database within + that DATALAKE + +The standard DBT_TERADATA_* connection env vars (or their defaults from +tests/conftest.py) are used for the Teradata connection. + +Scenarios covered: + 1. Basic OTF table create + 2. Idempotency (DROP+CREATE cycle survives re-runs) + 3. Cross-model ref() to an OTF model produces 3-part compiled SQL + 4. Cross-model source() with database/schema set produces 3-part SQL + (verifies the database != schema auto-OTF heuristic) + 5. purge_mode: 'NO PURGE' end-to-end + 6. PARTITIONED BY with transform functions (YEAR, BUCKET) + 7. SORTED BY with multiple columns + 8. All DDL options combined (partition + sort + tblproperties + purge) + 9. View materialization reading from OTF source (compile check) + 10. CTAS from OTF to native table (cross-ref compile check) + 11. CTAS from native to OTF (cross-ref compile check) + 12. OTF model with sql_header config + 13. PURGE ALL end-to-end + 14. Multiple file formats via tblproperties + 15. OTF incremental append: first run creates, second run appends (proves + is_incremental() fired via source-row deletion + survival assertion) + 16. OTF incremental append with PARTITIONED BY (same incremental proof) + 17. OTF incremental with --full-refresh (non-alias) + 18. dbt `alias` config on OTF table materialization + 19. dbt `alias` config on OTF incremental: create + true append proof + 20. dbt `alias` config + --full-refresh on OTF incremental + 21. Long alias name (64 chars): identifier plumbing does not truncate or + mangle the alias; staging table name stays within Teradata limits + 22. ref() to an aliased OTF incremental model compiles to the alias-named + 3-part DATALAKE identifier (compile-only check) + 23. dbt `alias` + `partitioned_by` on OTF incremental: create + true append + proof across two year-partitions +""" + +import os + +import pytest + +from dbt.tests.adapter.catalog_integrations.test_catalog_integration import ( + BaseCatalogIntegrationValidation, +) +from dbt.tests.util import run_dbt + + +DATALAKE_NAME = os.getenv("DBT_TERADATA_DATALAKE") +OTF_DATABASE = os.getenv("DBT_TERADATA_OTF_DATABASE") + +# 64-character alias used by TestOTFLongAlias (Scenario 21). +# Staging name = OTF_LONG_ALIAS_NAME + "__dbt_tmp" = 73 chars — within +# Teradata's 128-char identifier limit and well under OTF catalog limits. +OTF_LONG_ALIAS_NAME = "otf_long_alias_sixty_four_chars_to_test_identifier_length_limits" + +pytestmark = pytest.mark.skipif( + not (DATALAKE_NAME and OTF_DATABASE), + reason="requires DBT_TERADATA_DATALAKE and DBT_TERADATA_OTF_DATABASE env vars", +) + +# OTF tables are created in the shared external OTF database (not the per-test +# `{schema}`), so they are not cleaned up by the standard dbt test harness. +_OTF_TEST_TABLES = [ + "basic_otf", + "otf_no_purge", + "otf_partitioned_transforms", + "otf_partitioned_month", + "otf_sorted_multi", + "otf_all_options", + "otf_base_for_view", + "otf_source_model", + "otf_with_header", + "otf_purge_all", + "otf_write_orc", + "otf_write_avro", + "otf_write_parquet_gzip", + "otf_inc_append", + "otf_inc_append_partitioned", + "otf_inc_full_refresh", + # dbt `alias` config: physical OTF objects carry the *alias*, not the + # model file name, so register the alias names for cleanup (not the file + # names otf_alias_model / otf_alias_inc). + "otf_aliased_object", + "otf_alias_inc_object", + # Scenario 20: --full-refresh with alias + "otf_alias_fr_object", + # Scenario 21: long alias + OTF_LONG_ALIAS_NAME, + # Scenario 23: alias + partitioned_by incremental + "otf_alias_partitioned_object", + # on_schema_change Phase 1 + "otf_osc", + # on_schema_change sync_all_columns + "otf_sync", +] + + +def _is_otf_table_not_found(exc: Exception) -> bool: + """True if the exception is an OTF 'table does not exist' error. + + project.run_sql() calls cursor.execute() directly, bypassing the adapter's + add_query() where /*+ IF EXISTS */ suppression lives. Error 7825 (older + engines) and 6321 (TD 20.0.0.61+) must be swallowed during cleanup so that + one missing table does not abort cleanup of the rest. + """ + msg = str(exc) + return "[Error 7825]" in msg or "[Error 6321]" in msg + + +def _drop_otf_test_tables(project) -> None: + for name in _OTF_TEST_TABLES: + try: + project.run_sql( + f'DROP TABLE /*+ IF EXISTS */ "{DATALAKE_NAME}"."{OTF_DATABASE}"."{name}" NO PURGE;' + ) + except Exception as exc: + if not _is_otf_table_not_found(exc): + raise + + +@pytest.fixture(autouse=True) +def _cleanup_otf_tables(project): + _drop_otf_test_tables(project) # pre-test: clear any leftovers from prior runs + yield + _drop_otf_test_tables(project) # post-test: clean up what this test created + + +CATALOG_NAME = "test_catalog" + +CATALOGS_CONFIG = { + "catalogs": [ + { + "name": CATALOG_NAME, + "active_write_integration": "td_datalake", + "write_integrations": [ + { + "name": "td_datalake", + "catalog_type": "datalake", + "adapter_properties": { + "datalake_name": DATALAKE_NAME, + "otf_database": OTF_DATABASE, + }, + } + ], + } + ] +} + + +# --------------------------------------------------------------------------- +# Model SQL fixtures +# --------------------------------------------------------------------------- + +basic_otf_model_sql = f""" +{{{{ config( + materialized='table', + catalog_name='{CATALOG_NAME}' +) }}}} +select id, name from {{{{ target.schema }}}}.otf_src +""" + +otf_with_purge_mode_sql = f""" +{{{{ config( + materialized='table', + catalog_name='{CATALOG_NAME}', + purge_mode='NO PURGE' +) }}}} +select id from {{{{ target.schema }}}}.otf_src +""" + +downstream_of_otf_sql = """ +{{ config(materialized='table') }} +select * from {{ ref('basic_otf') }} +""" + +source_referencing_otf_sql = """ +{{ config(materialized='table') }} +select * from {{ source('otf_source', 'external_otf_table') }} +""" + +sources_yml = f""" +version: 2 +sources: + - name: otf_source + database: {DATALAKE_NAME or 'placeholder'} + schema: {OTF_DATABASE or 'placeholder'} + tables: + - name: external_otf_table +""" + + +# =================================================================== +# Scenarios 1 & 2: basic create + idempotency +# =================================================================== + +class TestOTFBasicAndIdempotent(BaseCatalogIntegrationValidation): + @pytest.fixture(scope="class") + def catalogs(self): + return CATALOGS_CONFIG + + @pytest.fixture(scope="class") + def models(self): + return {"basic_otf.sql": basic_otf_model_sql} + + def test_basic_create_and_rerun(self, project): + project.run_sql( + "CREATE TABLE {schema}.otf_src (id INTEGER, name VARCHAR(100))" + ) + project.run_sql("INSERT INTO {schema}.otf_src VALUES (1, 'a')") + project.run_sql("INSERT INTO {schema}.otf_src VALUES (2, 'b')") + try: + # First run: creates the OTF table. + results = run_dbt(["run", "--select", "basic_otf"]) + assert len(results) == 1 + assert results[0].status == "success" + + # Second run: must succeed idempotently (DROP+CREATE cycle). + results = run_dbt(["run", "--select", "basic_otf"]) + assert len(results) == 1 + assert results[0].status == "success" + finally: + project.run_sql("DROP TABLE {schema}.otf_src") + + +# =================================================================== +# Scenario 3: cross-model ref() produces 3-part name in compiled SQL +# =================================================================== + +class TestOTFRefRendersThreePartName(BaseCatalogIntegrationValidation): + @pytest.fixture(scope="class") + def catalogs(self): + return CATALOGS_CONFIG + + @pytest.fixture(scope="class") + def models(self): + return { + "basic_otf.sql": basic_otf_model_sql, + "downstream_of_otf.sql": downstream_of_otf_sql, + } + + def test_compiled_sql_contains_three_part_name(self, project): + run_dbt(["compile", "--select", "downstream_of_otf"]) + compiled_path = os.path.join( + str(project.project_root), + "target", "compiled", "test", "models", "downstream_of_otf.sql", + ) + with open(compiled_path, "r", encoding="utf-8") as f: + compiled = f.read() + # 3-part: "".""."
" + expected = f'"{DATALAKE_NAME}"."{OTF_DATABASE}"."basic_otf"' + assert expected in compiled, ( + f"Expected 3-part OTF name {expected!r} in compiled SQL, got:\n{compiled}" + ) + + +# =================================================================== +# Scenario 4: source() with database != schema triggers OTF heuristic +# =================================================================== + +class TestOTFSourceHeuristic(BaseCatalogIntegrationValidation): + @pytest.fixture(scope="class") + def catalogs(self): + return CATALOGS_CONFIG + + @pytest.fixture(scope="class") + def models(self): + return {"source_referencing_otf.sql": source_referencing_otf_sql} + + @pytest.fixture(scope="class") + def properties(self): + return {"sources.yml": sources_yml} + + def test_source_compiles_to_three_part_name(self, project): + # Compile only -- we don't require the source to actually exist. + run_dbt(["compile", "--select", "source_referencing_otf"]) + compiled_path = os.path.join( + str(project.project_root), + "target", "compiled", "test", "models", "source_referencing_otf.sql", + ) + with open(compiled_path, "r", encoding="utf-8") as f: + compiled = f.read() + expected = f'"{DATALAKE_NAME}"."{OTF_DATABASE}"."external_otf_table"' + assert expected in compiled, ( + f"Expected 3-part OTF source name {expected!r} in compiled SQL, " + f"got:\n{compiled}" + ) + + +# =================================================================== +# Scenario 5: purge_mode: 'NO PURGE' end-to-end +# =================================================================== + +class TestOTFNoPurge(BaseCatalogIntegrationValidation): + @pytest.fixture(scope="class") + def catalogs(self): + return CATALOGS_CONFIG + + @pytest.fixture(scope="class") + def models(self): + return {"otf_no_purge.sql": otf_with_purge_mode_sql} + + def test_no_purge_run_succeeds(self, project): + project.run_sql( + "CREATE TABLE {schema}.otf_src (id INTEGER, name VARCHAR(100))" + ) + project.run_sql("INSERT INTO {schema}.otf_src VALUES (1, 'a')") + try: + results = run_dbt(["run", "--select", "otf_no_purge"]) + assert len(results) == 1 + assert results[0].status == "success" + # Re-run to exercise the DROP path with NO PURGE. + results = run_dbt(["run", "--select", "otf_no_purge"]) + assert results[0].status == "success" + finally: + project.run_sql("DROP TABLE {schema}.otf_src") + + +# =================================================================== +# Scenario 6: PARTITIONED BY with transform functions +# =================================================================== + +otf_partitioned_transforms_sql = f""" +{{{{ config( + materialized='table', + catalog_name='{CATALOG_NAME}', + partitioned_by='YEAR(order_date), BUCKET(16, customer_id)' +) }}}} +select id, order_date, customer_id, amount from {{{{ target.schema }}}}.otf_src_orders +""" + +otf_partitioned_month_sql = f""" +{{{{ config( + materialized='table', + catalog_name='{CATALOG_NAME}', + partitioned_by='MONTH(created_at)' +) }}}} +select id, created_at from {{{{ target.schema }}}}.otf_src_orders +""" + + +class TestOTFPartitionTransforms(BaseCatalogIntegrationValidation): + """Verify PARTITIONED BY with transform functions like YEAR, BUCKET, MONTH. + Exercises partition transforms on the External OTF (CREATE TABLE) path. + """ + + @pytest.fixture(scope="class") + def catalogs(self): + return CATALOGS_CONFIG + + @pytest.fixture(scope="class") + def models(self): + return { + "otf_partitioned_transforms.sql": otf_partitioned_transforms_sql, + "otf_partitioned_month.sql": otf_partitioned_month_sql, + } + + def test_partition_transforms_run_succeeds(self, project): + project.run_sql( + "CREATE TABLE {schema}.otf_src_orders " + "(id INTEGER, order_date DATE, customer_id INTEGER, " + "amount DECIMAL(10,2), created_at TIMESTAMP)" + ) + project.run_sql( + "INSERT INTO {schema}.otf_src_orders " + "VALUES (1, DATE '2024-03-15', 100, 99.99, CURRENT_TIMESTAMP)" + ) + try: + results = run_dbt(["run", "--select", "otf_partitioned_transforms"]) + assert len(results) == 1 + assert results[0].status == "success" + finally: + project.run_sql("DROP TABLE {schema}.otf_src_orders") + + def test_partition_month_run_succeeds(self, project): + project.run_sql( + "CREATE TABLE {schema}.otf_src_orders " + "(id INTEGER, order_date DATE, customer_id INTEGER, " + "amount DECIMAL(10,2), created_at TIMESTAMP)" + ) + project.run_sql( + "INSERT INTO {schema}.otf_src_orders " + "VALUES (1, DATE '2024-06-01', 200, 50.00, CURRENT_TIMESTAMP)" + ) + try: + results = run_dbt(["run", "--select", "otf_partitioned_month"]) + assert len(results) == 1 + assert results[0].status == "success" + finally: + project.run_sql("DROP TABLE {schema}.otf_src_orders") + + +# =================================================================== +# Scenario 7: SORTED BY with multiple columns +# =================================================================== + +otf_sorted_multi_col_sql = f""" +{{{{ config( + materialized='table', + catalog_name='{CATALOG_NAME}', + sorted_by='region ASC, order_date DESC, customer_id ASC' +) }}}} +select id, region, order_date, customer_id +from {{{{ target.schema }}}}.otf_src_orders +""" + + +class TestOTFSortedByMultiColumn(BaseCatalogIntegrationValidation): + """Verify SORTED BY with multiple columns and directions on the External + OTF (CREATE TABLE) path. + """ + + @pytest.fixture(scope="class") + def catalogs(self): + return CATALOGS_CONFIG + + @pytest.fixture(scope="class") + def models(self): + return {"otf_sorted_multi.sql": otf_sorted_multi_col_sql} + + def test_sorted_by_multi_column_succeeds(self, project): + project.run_sql( + "CREATE TABLE {schema}.otf_src_orders " + "(id INTEGER, region VARCHAR(50), order_date DATE, customer_id INTEGER)" + ) + project.run_sql( + "INSERT INTO {schema}.otf_src_orders " + "VALUES (1, 'US-EAST', DATE '2024-01-01', 100)" + ) + try: + results = run_dbt(["run", "--select", "otf_sorted_multi"]) + assert len(results) == 1 + assert results[0].status == "success" + finally: + project.run_sql("DROP TABLE {schema}.otf_src_orders") + + +# =================================================================== +# Scenario 8: All DDL options combined +# =================================================================== + +otf_all_options_sql = f""" +{{{{ config( + materialized='table', + catalog_name='{CATALOG_NAME}', + partitioned_by='YEAR(order_date)', + sorted_by='customer_id ASC', + tblproperties="'write.format.default'='parquet', 'gc.enabled'='true'", + purge_mode='NO PURGE' +) }}}} +select id, order_date, customer_id, amount +from {{{{ target.schema }}}}.otf_src_orders +""" + + +class TestOTFAllDDLOptions(BaseCatalogIntegrationValidation): + """Verify combined DDL options in a single External OTF model. + + Exercises PARTITIONED BY + SORTED BY + TBLPROPERTIES together on the + External OTF (CREATE TABLE) path. Note: this does NOT cover primary-index + combinations -- INDEX is not allowed for External OTF, and the model below + sets no index. Index/PI support is a Managed OTF concern tracked separately. + """ + + @pytest.fixture(scope="class") + def catalogs(self): + return CATALOGS_CONFIG + + @pytest.fixture(scope="class") + def models(self): + return {"otf_all_options.sql": otf_all_options_sql} + + def test_all_ddl_options_run_succeeds(self, project): + project.run_sql( + "CREATE TABLE {schema}.otf_src_orders " + "(id INTEGER, order_date DATE, customer_id INTEGER, " + "amount DECIMAL(10,2))" + ) + project.run_sql( + "INSERT INTO {schema}.otf_src_orders " + "VALUES (1, DATE '2024-07-04', 42, 199.99)" + ) + try: + results = run_dbt(["run", "--select", "otf_all_options"]) + assert len(results) == 1 + assert results[0].status == "success" + + # Idempotency: second run also succeeds + results = run_dbt(["run", "--select", "otf_all_options"]) + assert results[0].status == "success" + finally: + project.run_sql("DROP TABLE {schema}.otf_src_orders") + + +# =================================================================== +# Scenario 9: View materialization reading from OTF source +# =================================================================== + +otf_base_for_view_sql = f""" +{{{{ config( + materialized='table', + catalog_name='{CATALOG_NAME}' +) }}}} +select id, name from {{{{ target.schema }}}}.otf_src +""" + +view_from_otf_sql = """ +{{ config(materialized='view') }} +select id, name from {{ ref('otf_base_for_view') }} +""" + + +class TestOTFViewFromOTFSource(BaseCatalogIntegrationValidation): + """Verify a view materialization can reference an OTF model. + The compiled SQL should contain the 3-part OTF name in the view definition. + Covers: Native OTF scenario #20 (Create Views reading from OTF tables). + """ + + @pytest.fixture(scope="class") + def catalogs(self): + return CATALOGS_CONFIG + + @pytest.fixture(scope="class") + def models(self): + return { + "otf_base_for_view.sql": otf_base_for_view_sql, + "view_from_otf.sql": view_from_otf_sql, + } + + def test_view_from_otf_compiles_with_three_part_name(self, project): + run_dbt(["compile", "--select", "view_from_otf"]) + compiled_path = os.path.join( + str(project.project_root), + "target", "compiled", "test", "models", "view_from_otf.sql", + ) + with open(compiled_path, "r", encoding="utf-8") as f: + compiled = f.read() + expected = f'"{DATALAKE_NAME}"."{OTF_DATABASE}"."otf_base_for_view"' + assert expected in compiled, ( + f"Expected 3-part OTF name {expected!r} in compiled view SQL, " + f"got:\n{compiled}" + ) + + +# =================================================================== +# Scenario 10: CTAS from OTF to native table (cross-ref) +# =================================================================== + +otf_source_model_sql = f""" +{{{{ config( + materialized='table', + catalog_name='{CATALOG_NAME}' +) }}}} +select id, name from {{{{ target.schema }}}}.otf_src +""" + +native_from_otf_ref_sql = """ +{{ config(materialized='table') }} +select id, name from {{ ref('otf_source_model') }} +""" + + +class TestOTFCrossRefOTFToNative(BaseCatalogIntegrationValidation): + """Verify a native table can select from an OTF model via ref(). + Compiled SQL should use 3-part name for the OTF source. + Exercises CREATE TABLE ... AS with an External OTF table as source. + """ + + @pytest.fixture(scope="class") + def catalogs(self): + return CATALOGS_CONFIG + + @pytest.fixture(scope="class") + def models(self): + return { + "otf_source_model.sql": otf_source_model_sql, + "native_from_otf_ref.sql": native_from_otf_ref_sql, + } + + def test_native_ctas_from_otf_compiles(self, project): + run_dbt(["compile", "--select", "native_from_otf_ref"]) + compiled_path = os.path.join( + str(project.project_root), + "target", "compiled", "test", "models", "native_from_otf_ref.sql", + ) + with open(compiled_path, "r", encoding="utf-8") as f: + compiled = f.read() + expected = f'"{DATALAKE_NAME}"."{OTF_DATABASE}"."otf_source_model"' + assert expected in compiled, ( + f"Expected 3-part OTF ref {expected!r} in native model SQL, " + f"got:\n{compiled}" + ) + + +# =================================================================== +# Scenario 11: CTAS from native to OTF (cross-ref) +# =================================================================== + +native_source_model_sql = """ +{{ config(materialized='table') }} +select 1 as id, 'product_a' as name +""" + +otf_from_native_ref_sql = f""" +{{{{ config( + materialized='table', + catalog_name='{CATALOG_NAME}' +) }}}} +select id, name from {{{{ ref('native_source_model') }}}} +""" + + +class TestOTFCrossRefNativeToOTF(BaseCatalogIntegrationValidation): + """Verify an OTF model can select from a native model via ref(). + The compiled SQL for the OTF model should reference the native table + using standard 2-part naming (schema.table). + Covers: Hive Catalog scenario #4 (CTAS from BFS/OFS into OTF). + """ + + @pytest.fixture(scope="class") + def catalogs(self): + return CATALOGS_CONFIG + + @pytest.fixture(scope="class") + def models(self): + return { + "native_source_model.sql": native_source_model_sql, + "otf_from_native_ref.sql": otf_from_native_ref_sql, + } + + def test_otf_ctas_from_native_compiles(self, project): + run_dbt(["compile", "--select", "otf_from_native_ref"]) + compiled_path = os.path.join( + str(project.project_root), + "target", "compiled", "test", "models", "otf_from_native_ref.sql", + ) + with open(compiled_path, "r", encoding="utf-8") as f: + compiled = f.read() + # The native model should be referenced with 2-part name (schema.table) + assert "native_source_model" in compiled + # The OTF model target is handled by the macro, not in compiled SQL body + + +# =================================================================== +# Scenario 12: OTF model with sql_header config +# =================================================================== + +otf_with_sql_header_sql = f""" +{{{{ config( + materialized='table', + catalog_name='{CATALOG_NAME}', + sql_header='SET QUERY_BAND = \\'app=dbt;model=otf_header;\\' FOR SESSION;' +) }}}} +select id, name from {{{{ target.schema }}}}.otf_src +""" + + +class TestOTFWithSqlHeader(BaseCatalogIntegrationValidation): + """Verify OTF models support sql_header config. + Covers: custom session settings before OTF DDL execution. + """ + + @pytest.fixture(scope="class") + def catalogs(self): + return CATALOGS_CONFIG + + @pytest.fixture(scope="class") + def models(self): + return {"otf_with_header.sql": otf_with_sql_header_sql} + + def test_otf_with_sql_header_succeeds(self, project): + project.run_sql( + "CREATE TABLE {schema}.otf_src (id INTEGER, name VARCHAR(100))" + ) + project.run_sql("INSERT INTO {schema}.otf_src VALUES (1, 'header_test')") + try: + results = run_dbt(["run", "--select", "otf_with_header"]) + assert len(results) == 1 + assert results[0].status == "success" + finally: + project.run_sql("DROP TABLE {schema}.otf_src") + + +# =================================================================== +# Scenario 13: PURGE ALL end-to-end +# =================================================================== + +otf_purge_all_sql = f""" +{{{{ config( + materialized='table', + catalog_name='{CATALOG_NAME}', + purge_mode='PURGE ALL' +) }}}} +select id, name from {{{{ target.schema }}}}.otf_src +""" + + +class TestOTFPurgeAll(BaseCatalogIntegrationValidation): + """Verify purge_mode='PURGE ALL' works end-to-end. + Covers: explicit PURGE ALL in DROP TABLE for OTF tables. + """ + + @pytest.fixture(scope="class") + def catalogs(self): + return CATALOGS_CONFIG + + @pytest.fixture(scope="class") + def models(self): + return {"otf_purge_all.sql": otf_purge_all_sql} + + def test_purge_all_run_succeeds(self, project): + project.run_sql( + "CREATE TABLE {schema}.otf_src (id INTEGER, name VARCHAR(100))" + ) + project.run_sql("INSERT INTO {schema}.otf_src VALUES (1, 'purge_test')") + try: + # First run: create the table + results = run_dbt(["run", "--select", "otf_purge_all"]) + assert len(results) == 1 + assert results[0].status == "success" + # Second run: exercises DROP with PURGE ALL + re-create + results = run_dbt(["run", "--select", "otf_purge_all"]) + assert results[0].status == "success" + finally: + project.run_sql("DROP TABLE {schema}.otf_src") + + +# =================================================================== +# Scenario 14: Different file formats via tblproperties +# =================================================================== + +otf_write_orc_sql = f""" +{{{{ config( + materialized='table', + catalog_name='{CATALOG_NAME}', + tblproperties="'write.format.default'='orc'" +) }}}} +select id, name from {{{{ target.schema }}}}.otf_src +""" + +otf_write_avro_sql = f""" +{{{{ config( + materialized='table', + catalog_name='{CATALOG_NAME}', + tblproperties="'write.format.default'='avro'" +) }}}} +select id, name from {{{{ target.schema }}}}.otf_src +""" + +otf_write_parquet_gzip_sql = f""" +{{{{ config( + materialized='table', + catalog_name='{CATALOG_NAME}', + tblproperties="'write.format.default'='parquet', 'write.parquet.compression-codec'='gzip'" +) }}}} +select id, name from {{{{ target.schema }}}}.otf_src +""" + + +class TestOTFFileFormats(BaseCatalogIntegrationValidation): + """Verify write data-file format configuration via tblproperties. + + Avro and Parquet (incl. gzip compression) are supported write formats for + External OTF (Iceberg). Writing ORC data files in Iceberg is NOT supported + by Teradata OTF, so that case is asserted as a failure rather than a success. + """ + + @pytest.fixture(scope="class") + def catalogs(self): + return CATALOGS_CONFIG + + @pytest.fixture(scope="class") + def models(self): + return { + "otf_write_orc.sql": otf_write_orc_sql, + "otf_write_avro.sql": otf_write_avro_sql, + "otf_write_parquet_gzip.sql": otf_write_parquet_gzip_sql, + } + + def test_orc_format_fails(self, project): + """Writing ORC data files in Iceberg is not supported, so materializing + a model with write.format.default='orc' must fail with an ORC-specific + error (e.g. TD_ICEBERG_WRITE: Cannot write ORC file).""" + project.run_sql( + "CREATE TABLE {schema}.otf_src (id INTEGER, name VARCHAR(100))" + ) + project.run_sql("INSERT INTO {schema}.otf_src VALUES (1, 'orc_test')") + try: + results = run_dbt( + ["run", "--select", "otf_write_orc"], expect_pass=False + ) + assert len(results) == 1 + assert results[0].status != "success" + # Be specific: the failure must be about the unsupported ORC write, + # not an unrelated error (e.g. connection/auth). The Teradata engine + # reports "Cannot write ORC file" from TD_ICEBERG_WRITE. + message = str(results[0].message or "").lower() + assert "orc" in message, ( + "Expected an ORC-related failure message, got: " + f"{results[0].message!r}" + ) + finally: + project.run_sql("DROP TABLE {schema}.otf_src") + + def test_avro_format_succeeds(self, project): + project.run_sql( + "CREATE TABLE {schema}.otf_src (id INTEGER, name VARCHAR(100))" + ) + project.run_sql("INSERT INTO {schema}.otf_src VALUES (1, 'avro_test')") + try: + results = run_dbt(["run", "--select", "otf_write_avro"]) + assert len(results) == 1 + assert results[0].status == "success" + finally: + project.run_sql("DROP TABLE {schema}.otf_src") + + def test_parquet_gzip_format_succeeds(self, project): + project.run_sql( + "CREATE TABLE {schema}.otf_src (id INTEGER, name VARCHAR(100))" + ) + project.run_sql("INSERT INTO {schema}.otf_src VALUES (1, 'gzip_test')") + try: + results = run_dbt(["run", "--select", "otf_write_parquet_gzip"]) + assert len(results) == 1 + assert results[0].status == "success" + finally: + project.run_sql("DROP TABLE {schema}.otf_src") + + +# =================================================================== +# Scenario 15: OTF incremental append +# =================================================================== + +otf_inc_append_sql = f""" +{{{{ config( + materialized='incremental', + catalog_name='{CATALOG_NAME}', + incremental_strategy='append' +) }}}} +select id, name from {{{{ target.schema }}}}.otf_inc_src + +{{% if is_incremental() %}} + where id > (select max(id) from {{{{ this }}}}) +{{% endif %}} +""" + + +class TestOTFIncrementalAppend(BaseCatalogIntegrationValidation): + """Verify OTF incremental append: first run creates, second run appends. + Covers: append strategy (default) for OTF tables. + """ + + @pytest.fixture(scope="class") + def catalogs(self): + return CATALOGS_CONFIG + + @pytest.fixture(scope="class") + def models(self): + return {"otf_inc_append.sql": otf_inc_append_sql} + + def test_incremental_append_first_and_second_run(self, project): + project.run_sql( + "CREATE TABLE {schema}.otf_inc_src (id INTEGER, name VARCHAR(100))" + ) + project.run_sql("INSERT INTO {schema}.otf_inc_src VALUES (1, 'alice')") + project.run_sql("INSERT INTO {schema}.otf_inc_src VALUES (2, 'bob')") + try: + # First run: creates the OTF table via CREATE TABLE AS. + results = run_dbt(["run", "--select", "otf_inc_append"]) + assert len(results) == 1 + assert results[0].status == "success" + + # Verify first run produced exactly 2 rows (alice, bob). + stats = project.run_sql( + f'SELECT MIN(id), MAX(id), COUNT(*) FROM "{DATALAKE_NAME}"."{OTF_DATABASE}"."otf_inc_append"', + fetch="one", + ) + assert stats[0] == 1 + assert stats[1] == 2 + assert stats[2] == 2 + + # Delete id=1 from the source to prove the second run is truly + # append-only (existing OTF rows must survive even if removed + # from the source). + project.run_sql("DELETE FROM {schema}.otf_inc_src WHERE id = 1") + project.run_sql("INSERT INTO {schema}.otf_inc_src VALUES (3, 'charlie')") + project.run_sql("INSERT INTO {schema}.otf_inc_src VALUES (4, 'diana')") + + # Second run: should only append new rows (id > max existing = 2). + results = run_dbt(["run", "--select", "otf_inc_append"]) + assert len(results) == 1 + assert results[0].status == "success" + + # Verify: all 4 rows present (alice/bob from run 1 + charlie/diana + # from run 2). id=1 (alice) must still be present — it was deleted + # from the source but append must never remove OTF rows. + stats = project.run_sql( + f'SELECT MIN(id), MAX(id), COUNT(*) FROM "{DATALAKE_NAME}"."{OTF_DATABASE}"."otf_inc_append"', + fetch="one", + ) + assert stats[0] == 1 # alice still present (append-only) + assert stats[1] == 4 # diana is newest + assert stats[2] == 4 # 4 distinct rows, no duplicates + finally: + project.run_sql("DROP TABLE {schema}.otf_inc_src") + + +# =================================================================== +# Scenario 16: OTF incremental append with PARTITIONED BY +# =================================================================== + +otf_inc_append_partitioned_sql = f""" +{{{{ config( + materialized='incremental', + catalog_name='{CATALOG_NAME}', + incremental_strategy='append', + partitioned_by='YEAR(created_date)' +) }}}} +select id, name, created_date from {{{{ target.schema }}}}.otf_inc_src_dated + +{{% if is_incremental() %}} + where created_date > (select max(created_date) from {{{{ this }}}}) +{{% endif %}} +""" + + +class TestOTFIncrementalAppendPartitioned(BaseCatalogIntegrationValidation): + """Verify OTF incremental append with PARTITIONED BY. + The first run creates a partitioned OTF table, second run appends. + """ + + @pytest.fixture(scope="class") + def catalogs(self): + return CATALOGS_CONFIG + + @pytest.fixture(scope="class") + def models(self): + return { + "otf_inc_append_partitioned.sql": otf_inc_append_partitioned_sql, + } + + def test_incremental_append_partitioned(self, project): + project.run_sql( + "CREATE TABLE {schema}.otf_inc_src_dated " + "(id INTEGER, name VARCHAR(100), created_date DATE)" + ) + project.run_sql( + "INSERT INTO {schema}.otf_inc_src_dated " + "VALUES (1, 'alice', DATE '2024-01-15')" + ) + try: + # First run: creates partitioned OTF table. + results = run_dbt(["run", "--select", "otf_inc_append_partitioned"]) + assert len(results) == 1 + assert results[0].status == "success" + + # Verify first run produced exactly 1 row. + count1 = project.run_sql( + f'SELECT COUNT(*) FROM "{DATALAKE_NAME}"."{OTF_DATABASE}"."otf_inc_append_partitioned"', + fetch="one", + )[0] + assert count1 == 1 + + # Delete id=1 from the source to prove the second run is truly + # append-only (existing OTF rows must survive source deletions). + project.run_sql("DELETE FROM {schema}.otf_inc_src_dated WHERE id = 1") + + # Add a row in a new partition (later date → passes the incremental filter). + project.run_sql( + "INSERT INTO {schema}.otf_inc_src_dated " + "VALUES (2, 'bob', DATE '2025-03-20')" + ) + + # Second run: should only append the new partition row. + results = run_dbt(["run", "--select", "otf_inc_append_partitioned"]) + assert len(results) == 1 + assert results[0].status == "success" + + # Verify: alice (2024-01-15) still present + bob (2025-03-20) appended. + count2 = project.run_sql( + f'SELECT COUNT(*) FROM "{DATALAKE_NAME}"."{OTF_DATABASE}"."otf_inc_append_partitioned"', + fetch="one", + )[0] + assert count2 == 2 + finally: + project.run_sql("DROP TABLE {schema}.otf_inc_src_dated") + + +# =================================================================== +# Scenario 17: OTF incremental with --full-refresh +# =================================================================== + +otf_inc_full_refresh_sql = f""" +{{{{ config( + materialized='incremental', + catalog_name='{CATALOG_NAME}', + incremental_strategy='append' +) }}}} +select id, name from {{{{ target.schema }}}}.otf_inc_src +""" + + +class TestOTFIncrementalFullRefresh(BaseCatalogIntegrationValidation): + """Verify --full-refresh drops and recreates OTF incremental table. + Covers: full refresh override on existing incremental OTF model. + """ + + @pytest.fixture(scope="class") + def catalogs(self): + return CATALOGS_CONFIG + + @pytest.fixture(scope="class") + def models(self): + return {"otf_inc_full_refresh.sql": otf_inc_full_refresh_sql} + + def test_full_refresh_recreates_otf_table(self, project): + project.run_sql( + "CREATE TABLE {schema}.otf_inc_src (id INTEGER, name VARCHAR(100))" + ) + project.run_sql("INSERT INTO {schema}.otf_inc_src VALUES (1, 'alice')") + try: + # First run: creates. + results = run_dbt(["run", "--select", "otf_inc_full_refresh"]) + assert len(results) == 1 + assert results[0].status == "success" + + # Full refresh: should DROP + CREATE from scratch. + results = run_dbt([ + "run", "--select", "otf_inc_full_refresh", "--full-refresh" + ]) + assert len(results) == 1 + assert results[0].status == "success" + finally: + project.run_sql("DROP TABLE {schema}.otf_inc_src") + + +# =================================================================== +# Scenario 18: dbt `alias` config on an OTF table materialization +# +# These exercise dbt's own `alias` resource config (NOT Teradata's +# CREATE ALIAS TABLE). The expectation is that the physical OTF object is +# named after the *alias*, not the model file name, and that ref()/this +# resolve to the alias-named 3-part DATALAKE object. +# =================================================================== + +# Model file is `otf_alias_model.sql`; the configured alias is a different name. +otf_alias_model_sql = f""" +{{{{ config( + materialized='table', + catalog_name='{CATALOG_NAME}', + alias='otf_aliased_object' +) }}}} +select id, name from {{{{ target.schema }}}}.otf_src +""" + +# Downstream model refs by MODEL NAME (otf_alias_model); dbt must resolve that +# ref to the ALIAS-named OTF object in the compiled 3-part name. +downstream_of_alias_sql = """ +{{ config(materialized='view') }} +select id, name from {{ ref('otf_alias_model') }} +""" + + +class TestOTFTableAlias(BaseCatalogIntegrationValidation): + """dbt `alias` on an OTF `table` model: the OTF object is created under the + alias, and the model-file name is NOT used as the object name. + """ + + @pytest.fixture(scope="class") + def catalogs(self): + return CATALOGS_CONFIG + + @pytest.fixture(scope="class") + def models(self): + return { + "otf_alias_model.sql": otf_alias_model_sql, + "downstream_of_alias.sql": downstream_of_alias_sql, + } + + def test_alias_drives_otf_object_name(self, project): + project.run_sql( + "CREATE TABLE {schema}.otf_src (id INTEGER, name VARCHAR(100))" + ) + project.run_sql("INSERT INTO {schema}.otf_src VALUES (1, 'a')") + project.run_sql("INSERT INTO {schema}.otf_src VALUES (2, 'b')") + try: + results = run_dbt(["run", "--select", "otf_alias_model"]) + assert len(results) == 1 + assert results[0].status == "success" + + # The OTF object must exist under the ALIAS name with 2 rows. + count = project.run_sql( + f'SELECT COUNT(*) FROM "{DATALAKE_NAME}"."{OTF_DATABASE}"."otf_aliased_object"', + fetch="one", + )[0] + assert count == 2 + + # The model-FILE name must NOT exist as an OTF object (Error 7825). + with pytest.raises(Exception, match=r"\[Error (7825|6321)\]"): + project.run_sql( + f'SELECT COUNT(*) FROM "{DATALAKE_NAME}"."{OTF_DATABASE}"."otf_alias_model"', + fetch="one", + ) + finally: + project.run_sql("DROP TABLE {schema}.otf_src") + + def test_ref_to_aliased_otf_model_uses_alias_name(self, project): + # ref('otf_alias_model') resolves by model name but must compile to the + # ALIAS-named 3-part OTF object. + run_dbt(["compile", "--select", "downstream_of_alias"]) + compiled_path = os.path.join( + str(project.project_root), + "target", "compiled", "test", "models", "downstream_of_alias.sql", + ) + with open(compiled_path, "r", encoding="utf-8") as f: + compiled = f.read() + expected = f'"{DATALAKE_NAME}"."{OTF_DATABASE}"."otf_aliased_object"' + assert expected in compiled, ( + f"Expected alias-named 3-part OTF ref {expected!r} in compiled SQL, " + f"got:\n{compiled}" + ) + # The model-file name must not leak into the compiled relation name. + assert f'"{OTF_DATABASE}"."otf_alias_model"' not in compiled + + +# =================================================================== +# Scenario 19: dbt `alias` config on an OTF incremental materialization +# =================================================================== + +# Model file is `otf_alias_inc.sql`; alias renames the physical OTF object. +otf_alias_inc_sql = f""" +{{{{ config( + materialized='incremental', + catalog_name='{CATALOG_NAME}', + incremental_strategy='append', + alias='otf_alias_inc_object' +) }}}} +select id, name from {{{{ target.schema }}}}.otf_inc_src + +{{% if is_incremental() %}} + where id > (select max(id) from {{{{ this }}}}) +{{% endif %}} +""" + + +class TestOTFIncrementalAlias(BaseCatalogIntegrationValidation): + """dbt `alias` on an OTF incremental model: create + append must both target + the alias-named object, and `this` (used by the is_incremental filter and the + existence probe) must resolve to the alias name. + """ + + @pytest.fixture(scope="class") + def catalogs(self): + return CATALOGS_CONFIG + + @pytest.fixture(scope="class") + def models(self): + return {"otf_alias_inc.sql": otf_alias_inc_sql} + + def test_incremental_alias_create_then_append(self, project): + project.run_sql( + "CREATE TABLE {schema}.otf_inc_src (id INTEGER, name VARCHAR(100))" + ) + project.run_sql("INSERT INTO {schema}.otf_inc_src VALUES (1, 'alice')") + project.run_sql("INSERT INTO {schema}.otf_inc_src VALUES (2, 'bob')") + try: + # First run: creates the alias-named OTF object. + results = run_dbt(["run", "--select", "otf_alias_inc"]) + assert len(results) == 1 + assert results[0].status == "success" + + stats = project.run_sql( + f'SELECT MIN(id), MAX(id), COUNT(*) FROM "{DATALAKE_NAME}"."{OTF_DATABASE}"."otf_alias_inc_object"', + fetch="one", + ) + assert stats[0] == 1 + assert stats[1] == 2 + assert stats[2] == 2 + + # The model-file name must NOT exist as an OTF object. + with pytest.raises(Exception, match=r"\[Error (7825|6321)\]"): + project.run_sql( + f'SELECT COUNT(*) FROM "{DATALAKE_NAME}"."{OTF_DATABASE}"."otf_alias_inc"', + fetch="one", + ) + + # Delete id=1 from source to prove the second run is a true + # incremental append, not a full rebuild. If is_incremental() fires + # correctly, the WHERE id > 2 filter excludes id=1 from the staging + # payload, so id=1 (alice) must still be present in the OTF table + # even though it no longer exists in the source. A full rebuild + # would produce OTF = [2, 3] (alice gone); a true append produces + # OTF = [1, 2, 3] (alice preserved). + project.run_sql("DELETE FROM {schema}.otf_inc_src WHERE id = 1") + project.run_sql("INSERT INTO {schema}.otf_inc_src VALUES (3, 'charlie')") + + # Second run: append path must target the alias-named object via `this`. + results = run_dbt(["run", "--select", "otf_alias_inc"]) + assert len(results) == 1 + assert results[0].status == "success" + + stats = project.run_sql( + f'SELECT MIN(id), MAX(id), COUNT(*) FROM "{DATALAKE_NAME}"."{OTF_DATABASE}"."otf_alias_inc_object"', + fetch="one", + ) + assert stats[0] == 1 # alice still present (append-only, not rebuilt) + assert stats[1] == 3 # charlie is newest + assert stats[2] == 3 # 3 rows: alice (run 1) + bob (run 1) + charlie (run 2) + finally: + project.run_sql("DROP TABLE {schema}.otf_inc_src") + + +# =================================================================== +# Scenario 20: dbt `alias` config + --full-refresh on OTF incremental +# +# Proves that --full-refresh drops and recreates the *alias-named* OTF +# object, not the model-file-named one. +# =================================================================== + +otf_alias_fr_sql = f""" +{{{{ config( + materialized='incremental', + catalog_name='{CATALOG_NAME}', + incremental_strategy='append', + alias='otf_alias_fr_object' +) }}}} +select id, name from {{{{ target.schema }}}}.otf_fr_src +""" + + +class TestOTFIncrementalAliasFullRefresh(BaseCatalogIntegrationValidation): + """--full-refresh on an aliased OTF incremental model must DROP+CREATE the + alias-named OTF object, not the model-file-named one. + """ + + @pytest.fixture(scope="class") + def catalogs(self): + return CATALOGS_CONFIG + + @pytest.fixture(scope="class") + def models(self): + return {"otf_alias_fr_model.sql": otf_alias_fr_sql} + + def test_alias_full_refresh_recreates_alias_object(self, project): + project.run_sql( + "CREATE TABLE {schema}.otf_fr_src (id INTEGER, name VARCHAR(100))" + ) + project.run_sql("INSERT INTO {schema}.otf_fr_src VALUES (1, 'alice')") + project.run_sql("INSERT INTO {schema}.otf_fr_src VALUES (2, 'bob')") + try: + # First run: creates alias-named OTF object with [alice, bob]. + results = run_dbt(["run", "--select", "otf_alias_fr_model"]) + assert len(results) == 1 + assert results[0].status == "success" + + count = project.run_sql( + f'SELECT COUNT(*) FROM "{DATALAKE_NAME}"."{OTF_DATABASE}"."otf_alias_fr_object"', + fetch="one", + )[0] + assert count == 2 + + # Mutate source: remove alice, add charlie. An incremental run + # would leave alice in OTF (append-only). A full-refresh must + # rebuild from the *current* source, so alice must disappear. + project.run_sql("DELETE FROM {schema}.otf_fr_src WHERE id = 1") + project.run_sql("INSERT INTO {schema}.otf_fr_src VALUES (3, 'charlie')") + + # --full-refresh: DROP + CREATE from current source [bob, charlie]. + results = run_dbt([ + "run", "--select", "otf_alias_fr_model", "--full-refresh" + ]) + assert len(results) == 1 + assert results[0].status == "success" + + # OTF table must now contain exactly 2 rows: bob and charlie. + # alice (id=1) must be gone — proves the table was rebuilt, not appended. + stats = project.run_sql( + f'SELECT MIN(id), MAX(id), COUNT(*) FROM "{DATALAKE_NAME}"."{OTF_DATABASE}"."otf_alias_fr_object"', + fetch="one", + ) + assert stats[2] == 2 # exactly 2 rows after full-refresh + assert stats[0] == 2 # bob is the minimum (alice gone) + assert stats[1] == 3 # charlie is the maximum + + # The model-FILE name must NOT exist as an OTF object. + with pytest.raises(Exception, match=r"\[Error (7825|6321)\]"): + project.run_sql( + f'SELECT COUNT(*) FROM "{DATALAKE_NAME}"."{OTF_DATABASE}"."otf_alias_fr_model"', + fetch="one", + ) + finally: + project.run_sql("DROP TABLE {schema}.otf_fr_src") + + +# =================================================================== +# Scenario 21: Long alias name (64 chars) +# +# Verifies that the identifier plumbing does not truncate or mangle a +# long alias. The staging table name (__dbt_tmp = 73 chars) must +# also stay within Teradata's 128-char identifier limit. +# =================================================================== + +otf_long_alias_sql = f""" +{{{{ config( + materialized='incremental', + catalog_name='{CATALOG_NAME}', + incremental_strategy='append', + alias='{OTF_LONG_ALIAS_NAME}' +) }}}} +select id, name from {{{{ target.schema }}}}.otf_long_alias_src + +{{% if is_incremental() %}} + where id > (select max(id) from {{{{ this }}}}) +{{% endif %}} +""" + + +class TestOTFLongAlias(BaseCatalogIntegrationValidation): + """A 64-character alias must be written verbatim to the OTF catalog — the + adapter must not truncate or mangle long-but-valid identifiers. The + staging table name (alias + '__dbt_tmp' = 73 chars) must also stay + within Teradata's 128-char limit across both create and append runs. + """ + + @pytest.fixture(scope="class") + def catalogs(self): + return CATALOGS_CONFIG + + @pytest.fixture(scope="class") + def models(self): + return {"otf_long_alias_model.sql": otf_long_alias_sql} + + def test_long_alias_create_and_append(self, project): + project.run_sql( + "CREATE TABLE {schema}.otf_long_alias_src (id INTEGER, name VARCHAR(100))" + ) + project.run_sql("INSERT INTO {schema}.otf_long_alias_src VALUES (1, 'alice')") + project.run_sql("INSERT INTO {schema}.otf_long_alias_src VALUES (2, 'bob')") + try: + # First run: creates the OTF table under the 64-char alias name. + results = run_dbt(["run", "--select", "otf_long_alias_model"]) + assert len(results) == 1 + assert results[0].status == "success" + + count = project.run_sql( + f'SELECT COUNT(*) FROM "{DATALAKE_NAME}"."{OTF_DATABASE}"."{OTF_LONG_ALIAS_NAME}"', + fetch="one", + )[0] + assert count == 2 + + # The model-file name must NOT exist as an OTF object. + with pytest.raises(Exception, match=r"\[Error (7825|6321)\]"): + project.run_sql( + f'SELECT COUNT(*) FROM "{DATALAKE_NAME}"."{OTF_DATABASE}"."otf_long_alias_model"', + fetch="one", + ) + + # Delete id=1 to prove the incremental path fires correctly with the + # long alias (staging table construction + existence probe both use it). + project.run_sql("DELETE FROM {schema}.otf_long_alias_src WHERE id = 1") + project.run_sql("INSERT INTO {schema}.otf_long_alias_src VALUES (3, 'charlie')") + + # Second run: append only id=3 via WHERE id > 2. + results = run_dbt(["run", "--select", "otf_long_alias_model"]) + assert len(results) == 1 + assert results[0].status == "success" + + stats = project.run_sql( + f'SELECT MIN(id), MAX(id), COUNT(*) FROM "{DATALAKE_NAME}"."{OTF_DATABASE}"."{OTF_LONG_ALIAS_NAME}"', + fetch="one", + ) + assert stats[0] == 1 # alice preserved (append-only, not rebuilt) + assert stats[1] == 3 # charlie appended + assert stats[2] == 3 + finally: + project.run_sql("DROP TABLE {schema}.otf_long_alias_src") + + +# =================================================================== +# Scenario 22: ref() to an aliased OTF incremental model +# +# Compile-only check: a downstream model that ref()s an aliased OTF +# incremental model must compile to the alias-named 3-part identifier, +# not the model file name. +# =================================================================== + +otf_alias_inc_ref_sql = f""" +{{{{ config( + materialized='incremental', + catalog_name='{CATALOG_NAME}', + incremental_strategy='append', + alias='otf_alias_inc_ref_object' +) }}}} +select id, name from {{{{ target.schema }}}}.otf_inc_ref_src +{{% if is_incremental() %}} + where id > (select max(id) from {{{{ this }}}}) +{{% endif %}} +""" + +downstream_of_alias_inc_sql = """ +{{ config(materialized='view') }} +select id, name from {{ ref('otf_alias_inc_ref_model') }} +""" + + +class TestOTFRefToAliasedIncrementalModel(BaseCatalogIntegrationValidation): + """ref() to an aliased OTF incremental model must compile to the + alias-named 3-part DATALAKE object, not the model file name. + """ + + @pytest.fixture(scope="class") + def catalogs(self): + return CATALOGS_CONFIG + + @pytest.fixture(scope="class") + def models(self): + return { + "otf_alias_inc_ref_model.sql": otf_alias_inc_ref_sql, + "downstream_of_alias_inc.sql": downstream_of_alias_inc_sql, + } + + def test_ref_to_aliased_incremental_uses_alias_name(self, project): + run_dbt(["compile", "--select", "downstream_of_alias_inc"]) + compiled_path = os.path.join( + str(project.project_root), + "target", "compiled", "test", "models", "downstream_of_alias_inc.sql", + ) + with open(compiled_path, "r", encoding="utf-8") as f: + compiled = f.read() + expected = f'"{DATALAKE_NAME}"."{OTF_DATABASE}"."otf_alias_inc_ref_object"' + assert expected in compiled, ( + f"Expected alias-named 3-part OTF ref {expected!r} in compiled SQL, " + f"got:\n{compiled}" + ) + assert f'"{OTF_DATABASE}"."otf_alias_inc_ref_model"' not in compiled + + +# =================================================================== +# Scenario 23: dbt `alias` + `partitioned_by` on OTF incremental +# +# Verifies that alias and partitioned_by work together end-to-end: +# the alias-named OTF table is created with the partition spec, and +# the incremental append targets the alias-named object correctly. +# =================================================================== + +otf_alias_partitioned_sql = f""" +{{{{ config( + materialized='incremental', + catalog_name='{CATALOG_NAME}', + incremental_strategy='append', + alias='otf_alias_partitioned_object', + partitioned_by='YEAR(event_date)' +) }}}} +select id, name, event_date +from {{{{ target.schema }}}}.otf_alias_partitioned_src +{{% if is_incremental() %}} + where event_date > (select max(event_date) from {{{{ this }}}}) +{{% endif %}} +""" + + +class TestOTFAliasWithPartition(BaseCatalogIntegrationValidation): + """alias + partitioned_by on OTF incremental: the alias-named table is + created with the partition spec and the incremental append targets the + correct alias-named 3-part object across two year-partitions. + """ + + @pytest.fixture(scope="class") + def catalogs(self): + return CATALOGS_CONFIG + + @pytest.fixture(scope="class") + def models(self): + return {"otf_alias_partitioned_model.sql": otf_alias_partitioned_sql} + + def test_alias_partitioned_create_and_append(self, project): + project.run_sql( + "CREATE TABLE {schema}.otf_alias_partitioned_src " + "(id INTEGER, name VARCHAR(100), event_date DATE)" + ) + project.run_sql( + "INSERT INTO {schema}.otf_alias_partitioned_src " + "VALUES (1, 'alice', DATE '2024-01-15')" + ) + project.run_sql( + "INSERT INTO {schema}.otf_alias_partitioned_src " + "VALUES (2, 'bob', DATE '2024-03-20')" + ) + try: + # First run: creates partitioned OTF table under the alias name. + results = run_dbt(["run", "--select", "otf_alias_partitioned_model"]) + assert len(results) == 1 + assert results[0].status == "success" + + stats = project.run_sql( + f'SELECT MIN(id), MAX(id), COUNT(*) FROM "{DATALAKE_NAME}"."{OTF_DATABASE}"."otf_alias_partitioned_object"', + fetch="one", + ) + assert stats[2] == 2 # alice and bob + + # The model-file name must NOT exist as an OTF object. + with pytest.raises(Exception, match=r"\[Error (7825|6321)\]"): + project.run_sql( + f'SELECT COUNT(*) FROM "{DATALAKE_NAME}"."{OTF_DATABASE}"."otf_alias_partitioned_model"', + fetch="one", + ) + + # Delete alice from source; add charlie in a new year-partition. + # A true incremental run must preserve alice in the OTF table. + project.run_sql( + "DELETE FROM {schema}.otf_alias_partitioned_src WHERE id = 1" + ) + project.run_sql( + "INSERT INTO {schema}.otf_alias_partitioned_src " + "VALUES (3, 'charlie', DATE '2025-06-10')" + ) + + # Second run: WHERE event_date > '2024-03-20' appends only charlie. + results = run_dbt(["run", "--select", "otf_alias_partitioned_model"]) + assert len(results) == 1 + assert results[0].status == "success" + + # OTF table must have 3 rows: alice (preserved), bob, charlie. + stats = project.run_sql( + f'SELECT MIN(id), MAX(id), COUNT(*) FROM "{DATALAKE_NAME}"."{OTF_DATABASE}"."otf_alias_partitioned_object"', + fetch="one", + ) + assert stats[0] == 1 # alice preserved (append-only, not rebuilt) + assert stats[1] == 3 # charlie appended + assert stats[2] == 3 # 3 rows: alice + bob + charlie + finally: + project.run_sql("DROP TABLE {schema}.otf_alias_partitioned_src") + + +# =================================================================== +# Scenario 24: on_schema_change Phase 1 for OTF incremental +# - append_new_columns: ALTER ADD a new source column, back-fill NULL for +# pre-existing rows, positional INSERT aligned to the new OTF column order +# - fail: raise a clear error on schema drift +# (sync_all_columns is covered separately in Scenario 25) +# +# A var toggles the extra column and the mode so the whole lifecycle runs +# unattended in one test. +# =================================================================== + +otf_osc_model_sql = f""" +{{{{ config( + materialized='incremental', + catalog_name='{CATALOG_NAME}', + incremental_strategy='append', + on_schema_change=var('otf_osc_mode', 'append_new_columns') +) }}}} +{{% if var('otf_reorder', false) %}} +select name, id from {{{{ target.schema }}}}.otf_osc_src +{{% else %}} +select id, name +{{% if var('otf_add_col', false) %}}, cast('x' as varchar(10)) as extra_col{{% endif %}} +from {{{{ target.schema }}}}.otf_osc_src +{{% endif %}} +""" + + +class TestOTFIncrementalOnSchemaChange(BaseCatalogIntegrationValidation): + """on_schema_change Phase 1 for OTF incremental models.""" + + @pytest.fixture(scope="class") + def catalogs(self): + return CATALOGS_CONFIG + + @pytest.fixture(scope="class") + def models(self): + return {"otf_osc.sql": otf_osc_model_sql} + + def test_append_new_columns_then_fail(self, project): + # Defensive: ensure no stale OTF table from a prior aborted run. + try: + project.run_sql( + f'DROP TABLE /*+ IF EXISTS */ "{DATALAKE_NAME}"."{OTF_DATABASE}"."otf_osc" NO PURGE;' + ) + except Exception as exc: + # Only the "table not found" case is expected here; surface anything else. + if not _is_otf_table_not_found(exc): + raise + project.run_sql( + "CREATE TABLE {schema}.otf_osc_src (id INTEGER, name VARCHAR(50))" + ) + project.run_sql("INSERT INTO {schema}.otf_osc_src VALUES (1, 'a')") + project.run_sql("INSERT INTO {schema}.otf_osc_src VALUES (2, 'b')") + try: + # 1) First run: create OTF table with (id, name). + r = run_dbt(["run", "--select", "otf_osc"]) + assert r[0].status == "success" + + # 2) append_new_columns: add extra_col -> ALTER TABLE ADD + aligned + # positional INSERT. Pre-existing rows get NULL; new rows get 'x'. + r = run_dbt(["run", "--select", "otf_osc", "--vars", "{otf_add_col: true}"]) + assert r[0].status == "success" + # Referencing extra_col proves the ALTER ran; counts prove back-fill. + cnt = project.run_sql( + f'SELECT COUNT(*), COUNT(extra_col) ' + f'FROM "{DATALAKE_NAME}"."{OTF_DATABASE}"."otf_osc"', + fetch="one", + ) + assert cnt[0] == 4 # 2 from run 1 + 2 from run 2 + assert cnt[1] == 2 # only run-2 rows carry extra_col + + # 2b) reorder-only change ('select name, id') under append_new_columns + # must NOT corrupt data: OTF inserts are positional, so the adapter + # realigns the SELECT to the OTF column order by name. The run must + # succeed and the id<->name pairing must be preserved. + r = run_dbt(["run", "--select", "otf_osc", "--vars", "{otf_reorder: true}"]) + assert r[0].status == "success" + mispaired = project.run_sql( + f'SELECT COUNT(*) FROM "{DATALAKE_NAME}"."{OTF_DATABASE}"."otf_osc" ' + f"WHERE (id = 1 AND name <> 'a') OR (id = 2 AND name <> 'b')", + fetch="one", + )[0] + assert mispaired == 0 + + # 3) fail mode on drift (extra_col now missing from source) -> error. + r = run_dbt( + ["run", "--select", "otf_osc", "--vars", "{otf_osc_mode: fail}"], + expect_pass=False, + ) + assert r[0].status == "error" + assert "on_schema_change='fail'" in r[0].message + finally: + project.run_sql("DROP TABLE {schema}.otf_osc_src") + + + +# =================================================================== +# Scenario 25: on_schema_change='sync_all_columns' for OTF incremental +# - adds new columns, drops removed columns, applies allowed type promotions +# (int -> bigint), and raises a clear error for an unsupported type change. +# Type comparison is at OTF/Iceberg granularity (HELP TABLE 'OTF Type'). +# =================================================================== + +otf_sync_model_sql = f""" +{{{{ config( + materialized='incremental', + catalog_name='{CATALOG_NAME}', + incremental_strategy='append', + on_schema_change='sync_all_columns' +) }}}} +{{% if var('otf_sync_bad', false) %}} +select id, name, cast(amount as varchar(20)) as amount from {{{{ target.schema }}}}.otf_sync_src +{{% elif var('otf_sync_v2', false) %}} +select cast(id as bigint) as id, amount, region from {{{{ target.schema }}}}.otf_sync_src +{{% else %}} +select id, name, amount from {{{{ target.schema }}}}.otf_sync_src +{{% endif %}} +""" + + +class TestOTFIncrementalSyncAllColumns(BaseCatalogIntegrationValidation): + """sync_all_columns: add + drop + allowed promotion, and a clear error on an + unsupported type change.""" + + @pytest.fixture(scope="class") + def catalogs(self): + return CATALOGS_CONFIG + + @pytest.fixture(scope="class") + def models(self): + return {"otf_sync.sql": otf_sync_model_sql} + + def test_sync_all_columns(self, project): + # Defensive: clear any stale OTF table from a prior aborted run. + try: + project.run_sql( + f'DROP TABLE /*+ IF EXISTS */ "{DATALAKE_NAME}"."{OTF_DATABASE}"."otf_sync" NO PURGE;' + ) + except Exception as exc: + if not _is_otf_table_not_found(exc): + raise + project.run_sql( + "CREATE TABLE {schema}.otf_sync_src " + "(id INTEGER, name VARCHAR(50), amount DECIMAL(10,2), region VARCHAR(20))" + ) + project.run_sql("INSERT INTO {schema}.otf_sync_src VALUES (1, 'a', 10.00, 'NORTH')") + project.run_sql("INSERT INTO {schema}.otf_sync_src VALUES (2, 'b', 20.00, 'SOUTH')") + try: + # 1) First run: create (id INT, name, amount). + r = run_dbt(["run", "--select", "otf_sync"]) + assert r[0].status == "success" + + # 2) sync_all_columns: add 'region', drop 'name', promote id INT->BIGINT. + r = run_dbt(["run", "--select", "otf_sync", "--vars", "{otf_sync_v2: true}"]) + assert r[0].status == "success" + + # 'region' now exists and is back-filled; 'name' was dropped. + cnt = project.run_sql( + f'SELECT COUNT(*), COUNT(region) ' + f'FROM "{DATALAKE_NAME}"."{OTF_DATABASE}"."otf_sync"', + fetch="one", + ) + assert cnt[0] == 4 # 2 from run 1 + 2 from run 2 + assert cnt[1] == 2 # only run-2 rows carry region (run-1 rows NULL) + # Dropped column must be gone — verify via HELP TABLE (avoids stale + # connection pool handle from project.adapter method after run_sql closes + # the _test connection). + help_rows = project.run_sql( + f'HELP TABLE "{DATALAKE_NAME}"."{OTF_DATABASE}"."otf_sync"', + fetch="all", + ) + col_names = [row[0].strip().lower() for row in help_rows] + assert "name" not in col_names, "'name' should have been dropped by sync_all_columns" + assert "region" in col_names, "'region' should have been added by sync_all_columns" + + # 3) Unsupported type change (amount DECIMAL -> VARCHAR) -> clear error. + r = run_dbt( + ["run", "--select", "otf_sync", "--vars", "{otf_sync_bad: true}"], + expect_pass=False, + ) + assert r[0].status == "error" + assert "--full-refresh" in r[0].message + finally: + project.run_sql("DROP TABLE {schema}.otf_sync_src") diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unit/test_otf_catalogs.py b/tests/unit/test_otf_catalogs.py new file mode 100644 index 00000000..9db09bea --- /dev/null +++ b/tests/unit/test_otf_catalogs.py @@ -0,0 +1,878 @@ +"""Unit tests for Teradata OTF (Open Table Format) catalog integration. + +Covers: + - TeradataCatalogRelation dataclass fields and defaults + - TeradataDatalakeCatalogIntegration initialization and validation + - build_relation() config passthrough + - Adapter CATALOG_INTEGRATIONS registration + - TeradataRelation.render() for both 2-part (native) and 3-part (OTF) names + - is_otf invariant guard in render() + +These tests are pure unit tests (no database required) and live in tests/unit/ +rather than tests/functional/ so they can run in a fast CI matrix without +provisioning Vantage Express. +""" + +import pytest +from dataclasses import asdict +from unittest.mock import MagicMock + +from dbt.adapters.catalogs import ( + CatalogIntegrationConfig, + InvalidCatalogIntegrationConfigError, +) +from dbt_common.exceptions import DbtRuntimeError, DbtDatabaseError + +from dbt.adapters.teradata.catalogs import ( + TeradataDatalakeCatalogIntegration, + TeradataCatalogRelation, +) +from dbt.adapters.teradata.relation import TeradataRelation +from dbt.adapters.teradata.impl import TeradataAdapter + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _make_config(**overrides): + """Build a spec'd MagicMock of CatalogIntegrationConfig. + + Using spec= ensures the mock fails on attribute access for anything + not on the protocol — masking-by-MagicMock would defeat the test. + """ + config = MagicMock(spec=CatalogIntegrationConfig) + config.name = overrides.get("name", "test_catalog") + config.catalog_type = overrides.get("catalog_type", "datalake") + config.catalog_name = overrides.get("catalog_name", None) + config.table_format = overrides.get("table_format", None) + config.external_volume = overrides.get("external_volume", None) + config.file_format = overrides.get("file_format", None) + config.adapter_properties = overrides.get("adapter_properties", {}) + return config + + +def _make_valid_config(**overrides): + defaults = { + "adapter_properties": { + "datalake_name": "my_datalake", + "otf_database": "my_otf_db", + }, + } + defaults.update(overrides) + return _make_config(**defaults) + + +# =================================================================== +# TeradataCatalogRelation +# =================================================================== + +class TestTeradataCatalogRelation: + def test_defaults(self): + rel = TeradataCatalogRelation() + assert rel.catalog_type is None + assert rel.catalog_name is None + assert rel.table_format is None + assert rel.file_format is None + assert rel.external_volume is None + assert rel.datalake_name is None + assert rel.otf_database is None + assert rel.partitioned_by is None + assert rel.sorted_by is None + assert rel.tblproperties is None + assert rel.purge_mode is None + + def test_all_fields(self): + rel = TeradataCatalogRelation( + catalog_type="datalake", + catalog_name="cat", + table_format="iceberg", + file_format="parquet", + external_volume="vol", + datalake_name="dl", + otf_database="otf_db", + partitioned_by="YEAR(dt)", + sorted_by="id ASC", + tblproperties="'k'='v'", + purge_mode="NO PURGE", + ) + assert rel.catalog_type == "datalake" + assert rel.datalake_name == "dl" + assert rel.otf_database == "otf_db" + assert rel.partitioned_by == "YEAR(dt)" + assert rel.sorted_by == "id ASC" + assert rel.tblproperties == "'k'='v'" + assert rel.purge_mode == "NO PURGE" + + def test_is_dataclass(self): + rel = TeradataCatalogRelation(datalake_name="dl", otf_database="db") + d = asdict(rel) + assert d["datalake_name"] == "dl" + assert d["otf_database"] == "db" + assert "partitioned_by" in d + + def test_purge_mode_no_purge(self): + rel = TeradataCatalogRelation(purge_mode="NO PURGE") + assert rel.purge_mode == "NO PURGE" + + def test_purge_mode_purge_all(self): + rel = TeradataCatalogRelation(purge_mode="PURGE ALL") + assert rel.purge_mode == "PURGE ALL" + + +# =================================================================== +# TeradataDatalakeCatalogIntegration -- __init__ +# =================================================================== + +class TestTeradataDatalakeCatalogIntegration: + # -- happy path -- + + def test_init_with_valid_config(self): + integration = TeradataDatalakeCatalogIntegration(_make_valid_config()) + assert integration.datalake_name == "my_datalake" + assert integration.otf_database == "my_otf_db" + assert integration.catalog_type == "datalake" + assert integration.allows_writes is True + assert integration.table_format == "iceberg" + assert integration.file_format == "parquet" + + def test_class_level_defaults(self): + assert TeradataDatalakeCatalogIntegration.catalog_type == "datalake" + assert TeradataDatalakeCatalogIntegration.allows_writes is True + assert TeradataDatalakeCatalogIntegration.table_format == "iceberg" + assert TeradataDatalakeCatalogIntegration.file_format == "parquet" + + def test_file_format_defaults_to_parquet_when_none(self): + integration = TeradataDatalakeCatalogIntegration(_make_valid_config(file_format=None)) + assert integration.file_format == "parquet" + + def test_file_format_override(self): + integration = TeradataDatalakeCatalogIntegration(_make_valid_config(file_format="orc")) + assert integration.file_format == "orc" + + def test_table_format_override(self): + integration = TeradataDatalakeCatalogIntegration(_make_valid_config(table_format="delta")) + assert integration.table_format == "delta" + + def test_catalog_name_passed_through(self): + integration = TeradataDatalakeCatalogIntegration(_make_valid_config(catalog_name="glue_catalog")) + assert integration.catalog_name == "glue_catalog" + + # -- validation errors -- + + def test_init_missing_datalake_name_raises(self): + config = _make_config(adapter_properties={"otf_database": "my_otf_db"}) + with pytest.raises(InvalidCatalogIntegrationConfigError): + TeradataDatalakeCatalogIntegration(config) + + def test_init_missing_otf_database_raises(self): + config = _make_config(adapter_properties={"datalake_name": "my_datalake"}) + with pytest.raises(InvalidCatalogIntegrationConfigError): + TeradataDatalakeCatalogIntegration(config) + + def test_init_empty_adapter_properties_raises(self): + with pytest.raises(InvalidCatalogIntegrationConfigError): + TeradataDatalakeCatalogIntegration(_make_config(adapter_properties={})) + + def test_init_none_adapter_properties_raises(self): + # Pinned to InvalidCatalogIntegrationConfigError only: the production + # code coerces None via `or {}`, so this is the only path that fires. + with pytest.raises(InvalidCatalogIntegrationConfigError): + TeradataDatalakeCatalogIntegration(_make_config(adapter_properties=None)) + + def test_init_empty_string_datalake_name_raises(self): + config = _make_config(adapter_properties={"datalake_name": "", "otf_database": "db"}) + with pytest.raises(InvalidCatalogIntegrationConfigError): + TeradataDatalakeCatalogIntegration(config) + + def test_init_empty_string_otf_database_raises(self): + config = _make_config(adapter_properties={"datalake_name": "dl", "otf_database": ""}) + with pytest.raises(InvalidCatalogIntegrationConfigError): + TeradataDatalakeCatalogIntegration(config) + + def test_error_message_mentions_datalake_name(self): + with pytest.raises(InvalidCatalogIntegrationConfigError, match="datalake_name"): + TeradataDatalakeCatalogIntegration(_make_config(adapter_properties={"otf_database": "db"})) + + def test_error_message_mentions_otf_database(self): + with pytest.raises(InvalidCatalogIntegrationConfigError, match="otf_database"): + TeradataDatalakeCatalogIntegration(_make_config(adapter_properties={"datalake_name": "dl"})) + + +# =================================================================== +# build_relation() +# =================================================================== + +class TestBuildRelation: + def _make_integration(self, **config_overrides): + return TeradataDatalakeCatalogIntegration(_make_valid_config(**config_overrides)) + + def test_basic_build(self): + integration = self._make_integration( + catalog_name="glue_catalog", + adapter_properties={ + "datalake_name": "my_glue_datalake", + "otf_database": "my_otf_db", + }, + ) + relation_config = MagicMock() + relation_config.config = {} + result = integration.build_relation(relation_config) + + assert isinstance(result, TeradataCatalogRelation) + assert result.datalake_name == "my_glue_datalake" + assert result.otf_database == "my_otf_db" + assert result.catalog_type == "datalake" + assert result.catalog_name == "glue_catalog" + assert result.table_format == "iceberg" + assert result.file_format == "parquet" + assert result.partitioned_by is None + assert result.sorted_by is None + assert result.tblproperties is None + assert result.purge_mode is None + + def test_with_all_ddl_options(self): + integration = self._make_integration() + relation_config = MagicMock() + relation_config.config = { + "partitioned_by": "YEAR(dt)", + "sorted_by": "id ASC", + "tblproperties": "'write.format.default'='parquet'", + "purge_mode": "NO PURGE", + } + result = integration.build_relation(relation_config) + assert result.partitioned_by == "YEAR(dt)" + assert result.sorted_by == "id ASC" + assert result.tblproperties == "'write.format.default'='parquet'" + assert result.purge_mode == "NO PURGE" + + def test_with_partitioned_by_only(self): + integration = self._make_integration() + relation_config = MagicMock() + relation_config.config = {"partitioned_by": "MONTH(created_at)"} + result = integration.build_relation(relation_config) + assert result.partitioned_by == "MONTH(created_at)" + assert result.sorted_by is None + assert result.tblproperties is None + + def test_with_sorted_by_only(self): + integration = self._make_integration() + relation_config = MagicMock() + relation_config.config = {"sorted_by": "ts DESC"} + result = integration.build_relation(relation_config) + assert result.partitioned_by is None + assert result.sorted_by == "ts DESC" + + def test_with_tblproperties_only(self): + integration = self._make_integration() + relation_config = MagicMock() + relation_config.config = {"tblproperties": "'gc.enabled'='true'"} + result = integration.build_relation(relation_config) + assert result.tblproperties == "'gc.enabled'='true'" + assert result.partitioned_by is None + + def test_with_purge_mode_only(self): + integration = self._make_integration() + relation_config = MagicMock() + relation_config.config = {"purge_mode": "NO PURGE"} + result = integration.build_relation(relation_config) + assert result.purge_mode == "NO PURGE" + assert result.partitioned_by is None + + def test_no_config_attr(self): + integration = self._make_integration() + relation_config = MagicMock(spec=[]) + result = integration.build_relation(relation_config) + assert result.partitioned_by is None + assert result.purge_mode is None + + def test_none_config_attr(self): + integration = self._make_integration() + relation_config = MagicMock() + relation_config.config = None + result = integration.build_relation(relation_config) + assert result.partitioned_by is None + assert result.purge_mode is None + + +# =================================================================== +# Adapter registration +# =================================================================== + +class TestAdapterCatalogRegistration: + def test_datalake_registered(self): + from dbt.adapters.teradata.impl import TeradataAdapter + assert ( + TeradataDatalakeCatalogIntegration + in TeradataAdapter.CATALOG_INTEGRATIONS + ) + + +# =================================================================== +# TeradataRelation.render() -- 2-part native vs 3-part OTF +# =================================================================== + +class TestTeradataRelationRender: + """Exercise render() directly to assert the actual rendered string. + + Replaces the old TestThreeDotNaming, which only re-implemented the Jinja + formula in Python and so caught nothing. + """ + + # -- 2-part native -- + + def test_native_render_two_part(self): + rel = TeradataRelation.create(schema="mydb", identifier="mytbl") + assert rel.render() == '"mydb"."mytbl"' + + def test_native_render_database_none_schema_set(self): + rel = TeradataRelation.create(database=None, schema="db", identifier="t") + assert rel.render() == '"db"."t"' + + # -- 3-part OTF (explicit is_otf=True via create()) -- + + def test_otf_render_three_part(self): + rel = TeradataRelation.create( + database="dl", + schema="db", + identifier="t", + quote_policy={"database": True, "schema": True, "identifier": True}, + include_policy={"database": True, "schema": True, "identifier": True}, + is_otf=True, + ) + assert rel.render() == '"dl"."db"."t"' + + def test_otf_render_with_underscores_and_hyphens(self): + rel = TeradataRelation.create( + database="dl_1", + schema="db-2", + identifier="tbl 3", + quote_policy={"database": True, "schema": True, "identifier": True}, + include_policy={"database": True, "schema": True, "identifier": True}, + is_otf=True, + ) + assert rel.render() == '"dl_1"."db-2"."tbl 3"' + + # -- Invariant guard: is_otf=True with a None part must raise -- + + def test_otf_render_missing_database_raises(self): + rel = TeradataRelation.create( + database=None, schema="db", identifier="t", + include_policy={"database": True, "schema": True, "identifier": True}, + is_otf=True, + ) + with pytest.raises(DbtRuntimeError, match="OTF relation is missing"): + rel.render() + + def test_otf_render_missing_schema_raises(self): + rel = TeradataRelation.create( + database="dl", schema=None, identifier="t", + include_policy={"database": True, "schema": True, "identifier": True}, + is_otf=True, + ) + with pytest.raises(DbtRuntimeError, match="OTF relation is missing"): + rel.render() + + def test_otf_render_missing_identifier_returns_two_part(self): + rel = TeradataRelation.create( + database="dl", schema="db", identifier=None, + include_policy={"database": True, "schema": True, "identifier": True}, + is_otf=True, + ) + # Schema-only OTF relation (e.g. cache warming) returns 2-part form + assert rel.render() == '"dl"."db"' + + # -- Additional OTF naming scenarios -- + + def test_otf_render_long_identifiers(self): + """OTF with long database, schema, and table names.""" + rel = TeradataRelation.create( + database="my_production_datalake_us_east_1", + schema="analytics_warehouse_iceberg_db", + identifier="fact_orders_partitioned_by_date_2024", + quote_policy={"database": True, "schema": True, "identifier": True}, + include_policy={"database": True, "schema": True, "identifier": True}, + is_otf=True, + ) + assert rel.render() == ( + '"my_production_datalake_us_east_1".' + '"analytics_warehouse_iceberg_db".' + '"fact_orders_partitioned_by_date_2024"' + ) + + def test_otf_render_numeric_start_identifiers(self): + """OTF names starting with numbers (valid when quoted).""" + rel = TeradataRelation.create( + database="datalake1", + schema="123_db", + identifier="456_table", + quote_policy={"database": True, "schema": True, "identifier": True}, + include_policy={"database": True, "schema": True, "identifier": True}, + is_otf=True, + ) + assert rel.render() == '"datalake1"."123_db"."456_table"' + + def test_otf_render_special_characters_in_schema(self): + """OTF schema with dots and special chars (quoted).""" + rel = TeradataRelation.create( + database="dl", + schema="my.otf.db", + identifier="my_table", + quote_policy={"database": True, "schema": True, "identifier": True}, + include_policy={"database": True, "schema": True, "identifier": True}, + is_otf=True, + ) + assert rel.render() == '"dl"."my.otf.db"."my_table"' + + def test_otf_render_uppercase_datalake(self): + """OTF datalake name in uppercase (quoted).""" + rel = TeradataRelation.create( + database="MY_DATALAKE", + schema="iceberg_db", + identifier="orders", + quote_policy={"database": True, "schema": True, "identifier": True}, + include_policy={"database": True, "schema": True, "identifier": True}, + is_otf=True, + ) + assert rel.render() == '"MY_DATALAKE"."iceberg_db"."orders"' + + def test_otf_render_doubles_embedded_quote(self): + """Embedded double-quotes are doubled in each part. + + This locks the quoting contract that the Jinja helper + teradata__quote_otf_part / teradata__build_otf_relation_name mirrors, + so DDL/DML macros and relation rendering produce identical names. + """ + rel = TeradataRelation.create( + database='d"l', + schema='o"db', + identifier='t"bl', + quote_policy={"database": True, "schema": True, "identifier": True}, + include_policy={"database": True, "schema": True, "identifier": True}, + is_otf=True, + ) + assert rel.render() == '"d""l"."o""db"."t""bl"' + + +# =================================================================== +# TeradataRelation.create_from() -- OTF detection via catalog_name +# =================================================================== + +class TestTeradataRelationCreateFromOTF: + """Verify create_from() detects catalog_name, calls + adapter.get_catalog_integration(), and builds an OTF 3-part relation. + """ + + def test_create_from_catalog_name_builds_otf_relation(self, monkeypatch): + from dbt.adapters import factory as adapter_factory + + cat = MagicMock(catalog_type="datalake", datalake_name="dl", otf_database="db") + adapter = MagicMock() + adapter.get_catalog_integration.return_value = cat + monkeypatch.setattr(adapter_factory, "get_adapter", lambda _: adapter) + + rc = MagicMock(identifier="tbl", config={"catalog_name": "test_catalog"}) + rel = TeradataRelation.create_from(MagicMock(), rc, type="table") + + assert rel.is_otf is True + assert rel.render() == '"dl"."db"."tbl"' + + def test_create_from_honours_dbt_alias(self, monkeypatch): + """dbt `alias` config: the OTF object name must be the *alias*, not the + model file name. + + dbt-core resolves a model's `alias` into ``node.alias`` and exposes it + as ``RelationConfig.identifier`` (artifacts/.../components.py). The OTF + path keys off ``relation_config.identifier``, so when ``alias`` differs + from the model file name the resolved identifier carries the alias and + the 3-part DATALAKE name must use it. This locks in that the existing + identifier plumbing satisfies dbt's `alias` feature for OTF tables. + """ + from dbt.adapters import factory as adapter_factory + + cat = MagicMock(catalog_type="datalake", datalake_name="dl", otf_database="db") + adapter = MagicMock() + adapter.get_catalog_integration.return_value = cat + monkeypatch.setattr(adapter_factory, "get_adapter", lambda _: adapter) + + # Model file is `sales_total.sql` but the user set alias='sales_dashboard'. + # dbt-core has already collapsed that into identifier == the alias. + rc = MagicMock(identifier="sales_dashboard", config={"catalog_name": "test_catalog"}) + rel = TeradataRelation.create_from(MagicMock(), rc, type="table") + + assert rel.is_otf is True + # The alias -- not the model file name -- is the OTF table identifier. + assert rel.identifier == "sales_dashboard" + assert rel.render() == '"dl"."db"."sales_dashboard"' + + +# =================================================================== +# Build relation -- additional DDL config scenarios +# =================================================================== + +class TestBuildRelationDDLCombinations: + """Verify build_relation() handles various combinations of DDL configs + from model config matching OTF PDF test scenarios. + """ + + def _make_integration(self, **config_overrides): + return TeradataDatalakeCatalogIntegration(_make_valid_config(**config_overrides)) + + def test_partition_with_transform_functions(self): + """Covers: partition transforms (bucket, truncate, month, year).""" + integration = self._make_integration() + relation_config = MagicMock() + relation_config.config = { + "partitioned_by": "YEAR(order_date), BUCKET(16, customer_id)", + } + result = integration.build_relation(relation_config) + assert result.partitioned_by == "YEAR(order_date), BUCKET(16, customer_id)" + + def test_partition_with_month_transform(self): + """Covers: MONTH partition transform from Native OTF scenarios.""" + integration = self._make_integration() + relation_config = MagicMock() + relation_config.config = { + "partitioned_by": "MONTH(created_at)", + } + result = integration.build_relation(relation_config) + assert result.partitioned_by == "MONTH(created_at)" + + def test_partition_with_truncate_transform(self): + """Covers: TRUNCATE partition transform.""" + integration = self._make_integration() + relation_config = MagicMock() + relation_config.config = { + "partitioned_by": "TRUNCATE(10, product_name)", + } + result = integration.build_relation(relation_config) + assert result.partitioned_by == "TRUNCATE(10, product_name)" + + def test_all_ddl_options_combined(self): + """Covers: Managed OTF Phase-1 combinations (PI+PB+SB).""" + integration = self._make_integration() + relation_config = MagicMock() + relation_config.config = { + "partitioned_by": "YEAR(dt), country", + "sorted_by": "id ASC, created_at DESC", + "tblproperties": "'write.format.default'='parquet', 'gc.enabled'='true'", + "purge_mode": "PURGE ALL", + } + result = integration.build_relation(relation_config) + assert result.partitioned_by == "YEAR(dt), country" + assert result.sorted_by == "id ASC, created_at DESC" + assert "'write.format.default'='parquet'" in result.tblproperties + assert "'gc.enabled'='true'" in result.tblproperties + assert result.purge_mode == "PURGE ALL" + + def test_tblproperties_write_format_orc(self): + """Covers: OTF write in ORC format via tblproperties.""" + integration = self._make_integration() + relation_config = MagicMock() + relation_config.config = { + "tblproperties": "'write.format.default'='orc'", + } + result = integration.build_relation(relation_config) + assert result.tblproperties == "'write.format.default'='orc'" + + def test_tblproperties_write_format_avro(self): + """Covers: OTF write in Avro format via tblproperties.""" + integration = self._make_integration() + relation_config = MagicMock() + relation_config.config = { + "tblproperties": "'write.format.default'='avro'", + } + result = integration.build_relation(relation_config) + assert result.tblproperties == "'write.format.default'='avro'" + + def test_tblproperties_catalog_database_name(self): + """Covers: Managed OTF external catalog name mapping.""" + integration = self._make_integration() + relation_config = MagicMock() + relation_config.config = { + "tblproperties": "'CATALOG_Database_NAME'='ext_db', 'CATALOG_Table_NAME'='ext_tbl'", + } + result = integration.build_relation(relation_config) + assert "CATALOG_Database_NAME" in result.tblproperties + assert "CATALOG_Table_NAME" in result.tblproperties + + def test_tblproperties_compression_snappy(self): + """Covers: Managed OTF Phase-1 compression (snappy only in Phase-1).""" + integration = self._make_integration() + relation_config = MagicMock() + relation_config.config = { + "tblproperties": "'write.parquet.compression-codec'='snappy'", + } + result = integration.build_relation(relation_config) + assert "snappy" in result.tblproperties + + def test_sorted_by_multiple_columns(self): + """Covers: SORTED BY with multiple columns.""" + integration = self._make_integration() + relation_config = MagicMock() + relation_config.config = { + "sorted_by": "region ASC, order_date DESC, customer_id ASC", + } + result = integration.build_relation(relation_config) + assert result.sorted_by == "region ASC, order_date DESC, customer_id ASC" + + def test_config_keys_case_sensitivity(self): + """Verify config keys must match exactly (case sensitive per MOTF spec).""" + integration = self._make_integration() + relation_config = MagicMock() + relation_config.config = { + "partitioned_by": "col1", + "sorted_by": "col2 ASC", + } + result = integration.build_relation(relation_config) + assert result.partitioned_by == "col1" + assert result.sorted_by == "col2 ASC" + # Uppercase keys should not be picked up + relation_config2 = MagicMock() + relation_config2.config = { + "PARTITIONED_BY": "col1", + } + result2 = integration.build_relation(relation_config2) + assert result2.partitioned_by is None + + +# =================================================================== +# Multiple catalog integrations +# =================================================================== + +class TestMultipleCatalogIntegrations: + """Verify that multiple catalog integrations can coexist with + different datalake/otf_database configurations (multi-catalog support). + """ + + def test_two_catalogs_different_datalakes(self): + """Covers: Hive Catalog + Glue Catalog in same project.""" + config_glue = _make_config( + name="glue_catalog", + adapter_properties={ + "datalake_name": "aws_glue_datalake", + "otf_database": "glue_iceberg_db", + }, + ) + config_unity = _make_config( + name="unity_catalog", + adapter_properties={ + "datalake_name": "azure_unity_datalake", + "otf_database": "unity_iceberg_db", + }, + ) + glue_integration = TeradataDatalakeCatalogIntegration(config_glue) + unity_integration = TeradataDatalakeCatalogIntegration(config_unity) + + assert glue_integration.datalake_name == "aws_glue_datalake" + assert glue_integration.otf_database == "glue_iceberg_db" + assert unity_integration.datalake_name == "azure_unity_datalake" + assert unity_integration.otf_database == "unity_iceberg_db" + + def test_two_catalogs_build_different_relations(self): + """Each catalog integration produces its own 3-part naming.""" + config_primary = _make_config( + name="primary", + adapter_properties={ + "datalake_name": "dl_primary", + "otf_database": "db_primary", + }, + ) + config_secondary = _make_config( + name="secondary", + adapter_properties={ + "datalake_name": "dl_secondary", + "otf_database": "db_secondary", + }, + ) + primary = TeradataDatalakeCatalogIntegration(config_primary) + secondary = TeradataDatalakeCatalogIntegration(config_secondary) + + relation_config = MagicMock() + relation_config.config = {"partitioned_by": "region"} + + rel_p = primary.build_relation(relation_config) + rel_s = secondary.build_relation(relation_config) + + assert rel_p.datalake_name == "dl_primary" + assert rel_p.otf_database == "db_primary" + assert rel_s.datalake_name == "dl_secondary" + assert rel_s.otf_database == "db_secondary" + # Both pass through the same model config + assert rel_p.partitioned_by == "region" + assert rel_s.partitioned_by == "region" + + +# =================================================================== +# TeradataAdapter.otf_relation_exists -- error handling +# =================================================================== + +class TestOtfRelationExists: + """Unit tests for TeradataAdapter.otf_relation_exists(). + + The method is tested by calling the unbound implementation on a + minimal MagicMock self so no database connection is required. + """ + + def _call(self, mock_self): + mock_self.Relation = TeradataRelation + return TeradataAdapter.otf_relation_exists( + mock_self, "my_datalake", "my_otf_db", "my_table" + ) + + def test_returns_true_when_execute_succeeds(self): + """Table exists: execute() returns normally → True.""" + mock_self = MagicMock() + mock_self.connections.execute.return_value = None + assert self._call(mock_self) is True + + def test_executes_correct_3part_quoted_sql(self): + """Verify the SAMPLE 0 probe uses the properly quoted 3-part OTF name.""" + mock_self = MagicMock() + mock_self.connections.execute.return_value = None + self._call(mock_self) + executed_sql = mock_self.connections.execute.call_args[0][0] + assert '"my_datalake"."my_otf_db"."my_table"' in executed_sql + assert "SAMPLE 0" in executed_sql + + def test_returns_false_for_error_7825(self): + """Table not found: DbtDatabaseError with [Error 7825] → False.""" + mock_self = MagicMock() + mock_self.connections.execute.side_effect = DbtDatabaseError( + "[Error 7825] ICEBERG_EXPORT: Table does not exist" + ) + assert self._call(mock_self) is False + + def test_returns_false_for_error_6321(self): + """Table not found on newer OTF engines (e.g. 20.0.0.61): DbtDatabaseError + with [Error 6321] "OTF Error: Table does not exist" → False.""" + mock_self = MagicMock() + mock_self.connections.execute.side_effect = DbtDatabaseError( + "[Error 6321] [SQLState HY000] OTF Error: Table does not exist: db.tbl" + ) + assert self._call(mock_self) is False + + def test_reraises_other_dbt_database_errors(self): + """Auth/permission errors must propagate, not be swallowed.""" + mock_self = MagicMock() + mock_self.connections.execute.side_effect = DbtDatabaseError( + "[Error 3524] The user does not have SELECT access to my_table" + ) + with pytest.raises(DbtDatabaseError, match="3524"): + self._call(mock_self) + + def test_reraises_non_database_exceptions(self): + """Network / unexpected errors must propagate unchanged.""" + mock_self = MagicMock() + mock_self.connections.execute.side_effect = RuntimeError("connection reset") + with pytest.raises(RuntimeError, match="connection reset"): + self._call(mock_self) + + +# =================================================================== +# TeradataAdapter.get_otf_columns_in_relation -- OTF schema read +# =================================================================== + +class TestGetOtfColumnsInRelation: + """get_otf_columns_in_relation reads an OTF table's column names from the + SAMPLE 0 result-set metadata (DBC.ColumnsV / HELP COLUMN do not work for + OTF). Used by on_schema_change reconciliation. + """ + + def test_returns_column_names_from_sample_zero(self): + mock_self = MagicMock() + # Use the real Relation so the 3-part name renders correctly. + mock_self.Relation = TeradataRelation + agate_table = MagicMock() + agate_table.column_names = ("order_id", "amount", "region") + mock_self.connections.execute.return_value = (MagicMock(), agate_table) + + result = TeradataAdapter.get_otf_columns_in_relation( + mock_self, "dl", "db", "orders" + ) + + assert result == ["order_id", "amount", "region"] + # Probes the 3-part OTF name with SAMPLE 0 and fetches metadata. + sql = mock_self.connections.execute.call_args[0][0] + assert "SAMPLE 0" in sql + assert '"dl"."db"."orders"' in sql + assert mock_self.connections.execute.call_args.kwargs.get("fetch") is True + + +# =================================================================== +# sync_all_columns support: type read + mapping + promotion rules +# =================================================================== + +class TestGetOtfColumnTypes: + """get_otf_column_types reads name + Iceberg 'OTF Type' from HELP TABLE.""" + + def test_parses_help_table_otf_type(self): + mock_self = MagicMock() + mock_self.Relation = TeradataRelation + agate_table = MagicMock() + # HELP TABLE rows expose 'Column Name' and 'OTF Type'; values may carry + # whitespace / mixed case, which the method normalises. + agate_table.rows = [ + {"Column Name": "ID ", "OTF Type": "int"}, + {"Column Name": "amt", "OTF Type": "decimal(10, 2)"}, + {"Column Name": "nm", "OTF Type": "STRING"}, + ] + mock_self.connections.execute.return_value = (MagicMock(), agate_table) + + result = TeradataAdapter.get_otf_column_types(mock_self, "dl", "db", "t") + + assert result == [ + {"name": "id", "otf_type": "int"}, + {"name": "amt", "otf_type": "decimal(10, 2)"}, + {"name": "nm", "otf_type": "string"}, + ] + sql = mock_self.connections.execute.call_args[0][0] + assert sql.strip().startswith("HELP TABLE") + assert '"dl"."db"."t"' in sql + + +class TestTeradataTypeToOtfType: + """Teradata DDL type -> canonical OTF/Iceberg type (length-agnostic).""" + + @pytest.mark.parametrize("ddl,expected", [ + ("INTEGER", "int"), + ("INT", "int"), + ("SMALLINT", "int"), + ("BYTEINT", "int"), + ("BIGINT", "long"), + ("VARCHAR(50)", "string"), + ("CHAR(10)", "string"), + ("LONG VARCHAR", "string"), + ("DECIMAL(10,2)", "decimal(10, 2)"), + ("NUMERIC(18, 4)", "decimal(18, 4)"), + ("FLOAT", "double"), + ("DOUBLE PRECISION", "double"), + ("DATE", "date"), + ("TIMESTAMP(6)", "timestamp"), + ("TIME(0)", "time"), + ]) + def test_mapping(self, ddl, expected): + assert TeradataAdapter.teradata_type_to_otf_type(MagicMock(), ddl) == expected + + def test_varchar_lengths_collapse_to_string(self): + # The whole point: VARCHAR length differences must NOT look like a change. + a = TeradataAdapter.teradata_type_to_otf_type(MagicMock(), "VARCHAR(50)") + b = TeradataAdapter.teradata_type_to_otf_type(MagicMock(), "VARCHAR(4000)") + assert a == b == "string" + + +class TestOtfTypePromotionAllowed: + """Only OTF/Iceberg-permitted promotions are allowed.""" + + @pytest.mark.parametrize("old,new,allowed", [ + ("int", "int", True), # no-op + ("int", "long", True), # widening + ("long", "int", False), # narrowing + ("decimal(10, 2)", "decimal(18, 2)", True), # precision widen, same scale + ("decimal(10, 2)", "decimal(18, 4)", False), # scale change + ("decimal(18, 2)", "decimal(10, 2)", False), # precision narrow + ("string", "int", False), + ("int", "string", False), + ]) + def test_promotions(self, old, new, allowed): + assert TeradataAdapter.otf_type_promotion_allowed(MagicMock(), old, new) is allowed diff --git a/tests/unit/test_otf_incremental.py b/tests/unit/test_otf_incremental.py new file mode 100644 index 00000000..3dc169dd --- /dev/null +++ b/tests/unit/test_otf_incremental.py @@ -0,0 +1,329 @@ +"""Unit tests for OTF incremental materialization support. + +Pure unit tests (no database required) that verify: + 1. The incremental_otf.sql macro file exists and contains expected macros + 2. The incremental.sql entry point branches to OTF when catalog_name is set + 3. OTF relation rendering works correctly for incremental SQL generation + 4. Catalog integration properties support incremental use case + 5. Strategy constants and macro structure match the design +""" + +import os +import pytest +from unittest.mock import MagicMock + +from dbt.adapters.catalogs import CatalogIntegrationConfig +from dbt.adapters.teradata.catalogs import ( + TeradataDatalakeCatalogIntegration, +) +from dbt.adapters.teradata.relation import TeradataRelation + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +_REPO_ROOT = os.path.dirname( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +) +_MACRO_DIR = os.path.join( + _REPO_ROOT, "dbt", "include", "teradata", "macros", "materializations" +) + + +def _read_macro(subdir, filename): + path = os.path.join(_MACRO_DIR, subdir, filename) + with open(path, "r", encoding="utf-8") as f: + return f.read() + + +def _make_catalog_config(**overrides): + """Build a spec'd MagicMock of CatalogIntegrationConfig.""" + config = MagicMock(spec=CatalogIntegrationConfig) + config.name = overrides.get("name", "test_catalog") + config.catalog_type = overrides.get("catalog_type", "datalake") + config.catalog_name = overrides.get("catalog_name", None) + config.table_format = overrides.get("table_format", None) + config.external_volume = overrides.get("external_volume", None) + config.file_format = overrides.get("file_format", None) + config.adapter_properties = overrides.get("adapter_properties", { + "datalake_name": "my_datalake", + "otf_database": "my_otf_db", + }) + return config + + +# =================================================================== +# 1. Macro file existence and structure +# =================================================================== + +class TestOTFIncrementalMacroFileExists: + """Verify the incremental_otf.sql file exists in the expected location.""" + + def test_file_exists(self): + path = os.path.join(_MACRO_DIR, "otf", "incremental_otf.sql") + assert os.path.isfile(path), f"Missing macro file: {path}" + + def test_file_is_non_empty(self): + content = _read_macro("otf", "incremental_otf.sql") + assert len(content.strip()) > 0 + + +class TestOTFIncrementalMacroContent: + """Verify the macro file declares all expected macros and references.""" + + @pytest.fixture(scope="class") + def content(self): + return _read_macro("otf", "incremental_otf.sql") + + # --- expected macro definitions --- + + def test_defines_strategy_validator(self, content): + assert "macro teradata__validate_get_otf_incremental_strategy" in content + + def test_defines_append_macro(self, content): + assert "macro teradata__get_otf_incremental_append_sql" in content + + def test_defines_main_entry_point(self, content): + assert "macro teradata__incremental_otf" in content + + # --- strategy validation --- + + def test_accepts_append_strategy(self, content): + assert "'append'" in content + + def test_rejects_delete_insert(self, content): + assert "'delete+insert'" in content + assert "not yet supported" in content.lower() + + def test_rejects_merge(self, content): + assert "'merge'" in content + assert "not yet supported" in content.lower() + + def test_rejects_valid_history(self, content): + assert "'valid_history'" in content + + def test_rejects_microbatch(self, content): + assert "'microbatch'" in content + + # --- required macro references --- + + def test_calls_build_otf_relation_name(self, content): + assert "teradata__build_otf_relation_name" in content + + def test_calls_create_otf_table_as_for_first_run(self, content): + assert "teradata__create_otf_table_as" in content + + def test_drops_staging_table_after_use(self, content): + assert "DROP TABLE /*+ IF EXISTS */ {{ tmp_relation }}" in content + + # --- guardrails --- + + def test_guards_table_kind(self, content): + assert "table_kind" in content + + def test_guards_table_option(self, content): + assert "table_option" in content + + def test_guards_with_statistics(self, content): + assert "with_statistics" in content + + def test_guards_index(self, content): + assert "index" in content + + def test_guards_contract(self, content): + assert "contract" in content.lower() + + def test_guards_on_schema_change(self, content): + assert "on_schema_change" in content + assert "--full-refresh" in content + + def test_warns_on_grants(self, content): + assert "grants" in content.lower() + + # --- flow control --- + + def test_handles_first_run(self, content): + assert "adapter.otf_relation_exists(" in content + assert "not otf_exists" in content + + def test_handles_full_refresh(self, content): + assert "should_full_refresh" in content + + def test_uses_native_staging_creation(self, content): + assert "api.Relation.create(" in content + assert "target.schema" in content + + def test_runs_pre_hooks(self, content): + assert "run_hooks(pre_hooks" in content + + def test_runs_post_hooks(self, content): + assert "run_hooks(post_hooks" in content + + def test_commits_transaction(self, content): + assert "adapter.commit()" in content + + def test_caches_relation(self, content): + assert "adapter.cache_added" in content + + def test_returns_relations(self, content): + assert "return({'relations':" in content + + +# =================================================================== +# 2. Incremental entry point (incremental.sql) +# =================================================================== + +class TestIncrementalEntryPointBranching: + """Verify the main incremental.sql properly branches to OTF.""" + + @pytest.fixture(scope="class") + def content(self): + return _read_macro("incremental", "incremental.sql") + + def test_checks_catalog_name(self, content): + assert "catalog_name" in content + + def test_calls_otf_incremental_macro(self, content): + assert "teradata__incremental_otf" in content + + def test_captures_otf_result(self, content): + """OTF result is captured with {% set %} and returned at materialization level.""" + assert "set otf_result" in content or "teradata__incremental_otf" in content + + def test_returns_otf_result(self, content): + assert "return(otf_result)" in content + + def test_has_else_branch_for_native(self, content): + assert "{% else %}" in content + + def test_no_longer_blocks_otf_with_compile_error(self, content): + assert "Only the 'table' materialization supports OTF tables" not in content + + def test_native_path_still_has_validate_strategy(self, content): + assert "teradata__validate_get_incremental_strategy" in content + + def test_native_path_still_has_return(self, content): + assert "return({'relations': [target_relation]})" in content + + +# =================================================================== +# 3. OTF relation rendering for incremental SQL +# =================================================================== + +class TestOTFRelationForIncrementalSQL: + """Verify OTF and staging relations render correctly for INSERT/DELETE SQL.""" + + def test_otf_target_renders_three_part_name(self): + """INSERT INTO target must use 3-part OTF name.""" + rel = TeradataRelation.create( + database="dl", + schema="otf_db", + identifier="sales_fact", + quote_policy={"database": True, "schema": True, "identifier": True}, + include_policy={"database": True, "schema": True, "identifier": True}, + is_otf=True, + ) + assert rel.render() == '"dl"."otf_db"."sales_fact"' + + def test_staging_table_renders_two_part_name(self): + """SELECT FROM staging table uses 2-part native name.""" + rel = TeradataRelation.create( + schema="test_schema", + identifier="sales_fact__dbt_tmp", + ) + assert rel.render() == '"test_schema"."sales_fact__dbt_tmp"' + + def test_otf_relation_is_otf_true(self): + rel = TeradataRelation.create( + database="dl", schema="otf_db", identifier="t", + is_otf=True, + ) + assert rel.is_otf is True + + def test_staging_relation_is_otf_false(self): + rel = TeradataRelation.create( + schema="mydb", identifier="t__dbt_tmp", + ) + assert rel.is_otf is False + + def test_otf_and_staging_can_coexist_in_sql(self): + """Verify both relations render independently for cross-table SQL.""" + otf = TeradataRelation.create( + database="prod_lake", + schema="iceberg_db", + identifier="orders", + quote_policy={"database": True, "schema": True, "identifier": True}, + include_policy={"database": True, "schema": True, "identifier": True}, + is_otf=True, + ) + staging = TeradataRelation.create( + schema="dbt_schema", + identifier="orders__dbt_tmp", + ) + insert_sql = f"INSERT INTO {otf.render()} SELECT * FROM {staging.render()}" + assert '"prod_lake"."iceberg_db"."orders"' in insert_sql + assert '"dbt_schema"."orders__dbt_tmp"' in insert_sql + + +# =================================================================== +# 4. Catalog integration for incremental +# =================================================================== + +class TestCatalogIntegrationForIncremental: + """Verify catalog integration properties needed for incremental flow.""" + + def test_has_datalake_name(self): + integration = TeradataDatalakeCatalogIntegration(_make_catalog_config()) + assert integration.datalake_name == "my_datalake" + + def test_has_otf_database(self): + integration = TeradataDatalakeCatalogIntegration(_make_catalog_config()) + assert integration.otf_database == "my_otf_db" + + def test_allows_writes_for_incremental(self): + """Incremental strategy requires write access to OTF tables.""" + integration = TeradataDatalakeCatalogIntegration(_make_catalog_config()) + assert integration.allows_writes is True + + def test_catalog_type_is_datalake(self): + integration = TeradataDatalakeCatalogIntegration(_make_catalog_config()) + assert integration.catalog_type == "datalake" + + def test_default_table_format_is_iceberg(self): + integration = TeradataDatalakeCatalogIntegration(_make_catalog_config()) + assert integration.table_format == "iceberg" + + def test_supports_delta_format(self): + integration = TeradataDatalakeCatalogIntegration( + _make_catalog_config(table_format="delta") + ) + assert integration.table_format == "delta" + + +# =================================================================== +# 5. Strategy constants +# =================================================================== + +class TestOTFIncrementalStrategyConstants: + """Verify supported vs unsupported strategy definitions.""" + + SUPPORTED = {"append"} + UNSUPPORTED = {"delete+insert", "merge", "valid_history", "microbatch"} + + def test_append_is_supported(self): + assert "append" in self.SUPPORTED + + @pytest.mark.parametrize("strategy", ["delete+insert", "merge", "valid_history", "microbatch"]) + def test_unsupported_strategy(self, strategy): + assert strategy not in self.SUPPORTED + + def test_supported_and_unsupported_are_disjoint(self): + assert self.SUPPORTED.isdisjoint(self.UNSUPPORTED) + + def test_default_strategy_convention(self): + """Default strategy (when none specified) should be 'append' + which is the safest additive operation.""" + content = _read_macro("otf", "incremental_otf.sql") + # The validator defaults to 'append' when no strategy is configured + assert '"incremental_strategy", "append"' in content