feat(ingest/snowflake): breaking change for match_fully_qualified_names (#13852)

Author: anshbansal

docs/how/updating-datahub.md

@@ -30,6 +30,8 @@ This file documents any backwards-incompatible changes in DataHub and assists pe
 
 ### Breaking Changes
 
+- #13852 `snowflake` connector has `match_fully_qualified_names` which has been present for backward compatibility. If you are using `schema_pattern` but have not set `match_fully_qualified_names: true` please change it to `match_fully_qualified_names: true` and update your `schema_pattern` to match against fully qualified schema names `<catalog_name>.<schema_name>`. We were printing warnings earlier for this. We are making this backward incompatible breaking change to make this consistent with all other sources.
+
 ### Known Issues
 
 ### Potential Downtime

metadata-ingestion/docs/sources/snowflake/snowflake_recipe.yml

@@ -4,6 +4,13 @@ source:
     # This option is recommended to be used to ingest all lineage on the first run.
     ignore_start_time_lineage: true
 
+    # This has the default of `false` for backward compatibility but we are making a breaking change so add it here
+    # in case you wish to use schema_pattern
+    match_fully_qualified_names: true
+
+    # This flag tells the snowflake ingestion to use the more advanced query parsing. This will become the default eventually.
+    use_queries_v2: true
+
     # Coordinates
     account_id: "abc48144"
     warehouse: "COMPUTE_WH"

metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_config.py

@@ -135,10 +135,12 @@ def validate_legacy_schema_pattern(cls, values: Dict) -> Dict:
             and match_fully_qualified_names is not None
             and not match_fully_qualified_names
         ):
-            logger.warning(
-                "Please update `schema_pattern` to match against fully qualified schema name `<catalog_name>.<schema_name>` and set config `match_fully_qualified_names : True`."
-                "Current default `match_fully_qualified_names: False` is only to maintain backward compatibility. "
-                "The config option `match_fully_qualified_names` will be deprecated in future and the default behavior will assume `match_fully_qualified_names: True`."
+            raise ValueError(
+                "`match_fully_qualified_names: False` is no longer supported. "
+                "Please set `match_fully_qualified_names: True` and update your `schema_pattern` "
+                "to match against fully qualified schema names `<catalog_name>.<schema_name>`. "
+                "The config option `match_fully_qualified_names` will be removed in future and the default behavior will assume `match_fully_qualified_names: True`."
+                "match_fully_qualified_names False was there to maintain backward compatibility."
             )
 
         # Always exclude reporting metadata for INFORMATION_SCHEMA schema

metadata-ingestion/tests/integration/snowflake/test_snowflake.py

@@ -284,7 +284,8 @@ def test_snowflake_private_link_and_incremental_mcps(
                         account_id="ABC12345.ap-south-1.privatelink",
                         username="TST_USR",
                         password="TST_PWD",
-                        schema_pattern=AllowDenyPattern(allow=["test_schema"]),
+                        schema_pattern=AllowDenyPattern(allow=["test_db.test_schema"]),
+                        match_fully_qualified_names=True,
                         include_technical_schema=True,
                         include_table_lineage=True,
                         include_column_lineage=False,