diff --git a/bmsdna/lakeapi/core/datasource.py b/bmsdna/lakeapi/core/datasource.py index 6152ade..9a8e278 100644 --- a/bmsdna/lakeapi/core/datasource.py +++ b/bmsdna/lakeapi/core/datasource.py @@ -174,18 +174,21 @@ def get_execution_uri(self, meta_only: bool): def file_exists(self): if self.config.file_type in ["odbc", "sqlite"]: return True # the uri is not really a file here + if self.config.file_type == "delta": + return self.get_execution_uri(meta_only=True).exists() if not self.source_uri.exists(): return False return True def get_delta_table(self, schema_only: bool): if self.config.file_type == "delta": - if self.source_uri.exists(): + if self.file_exists(): try: from bmsdna.lakeapi.utils.meta_cache import get_deltalake_meta meta = get_deltalake_meta( - self.basic_config.default_engine == "polars", self.source_uri + self.basic_config.default_engine == "polars", + self.get_execution_uri(meta_only=True), ) return meta except FileNotFoundError: diff --git a/bmsdna/lakeapi/utils/meta_cache.py b/bmsdna/lakeapi/utils/meta_cache.py index f203e6b..e7ca90b 100644 --- a/bmsdna/lakeapi/utils/meta_cache.py +++ b/bmsdna/lakeapi/utils/meta_cache.py @@ -29,7 +29,10 @@ def get_deltalake_meta(use_polars: bool, uri: SourceUri): else: if _global_duck_con is None: _global_duck_con = duckdb.connect(":memory:") - _global_duck_meta_engine = DuckDBMetaEngine(_global_duck_con) + _global_duck_meta_engine = DuckDBMetaEngine( + _global_duck_con, + use_fsspec=os.getenv("DUCKDB_DELTA_USE_FSSPEC", "0") == "1", + ) ab_uri, ab_opts = uri.get_uri_options(flavor="original") assert _global_duck_meta_engine is not None meta_engine = _global_duck_meta_engine diff --git a/pyproject.toml b/pyproject.toml index 5a30190..8ee8277 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "bmsdna-lakeapi" -version = "0.27.2" +version = "0.27.3" description = "" authors = [{ name = "DWH Team", email = "you@example.com" }] dependencies = [