fixes staging_allowed_local_path on databricks, bumps databricks connector in lockfile

Author: rudolfix

dlt/destinations/impl/databricks/databricks.py

@@ -72,10 +72,15 @@ def run(self) -> None:
         # decide if this is a local file or a staged file
         is_local_file = not ReferenceFollowupJobRequest.is_reference_job(self._file_path)
         if is_local_file:
-            # conn parameter staging_allowed_local_path must be set to use 'PUT/REMOVE volume_path' SQL statement
-            self._sql_client.native_connection.thrift_backend.staging_allowed_local_path = (
-                os.path.dirname(self._file_path)
-            )
+            # staging_allowed_local_path should be set when opening the connection but at that
+            # time we do not know this path so do it now
+            conn_ = self._sql_client.native_connection
+            file_dir = os.path.dirname(self._file_path)
+            if backend := getattr(conn_, "thrift_backend", None):
+                backend.staging_allowed_local_path = file_dir
+            else:
+                # thrift backend discontinued on newer databricks connector clients
+                conn_.staging_allowed_local_path = file_dir
             # local file by uploading to a temporary volume on Databricks
             from_clause, file_name, volume_path, volume_file_path = self._handle_local_file_upload(
                 self._file_path

docs/education/dlt-fundamentals-course/lesson_3_pagination_and_authentication_and_dlt_configuration.ipynb

@@ -560,6 +560,7 @@
     "from dlt.sources.helpers.rest_client.auth import BearerTokenAuth\n",
     "from dlt.common.typing import TDataItems\n",
     "\n",
+    "\n",
     "@dlt.source\n",
     "def github_source(\n",
     "    access_token=dlt.secrets.value,\n",

pyproject.toml

@@ -142,7 +142,7 @@ qdrant = [
     "qdrant-client[fastembed]>=1.8"
 ]
 databricks = [
-    "databricks-sql-connector>=2.9.3,<4 ; python_version <= '3.12'",
+    "databricks-sql-connector>=2.9.3 ; python_version <= '3.12'",
     "databricks-sql-connector>=3.6.0 ; python_version >= '3.13'",
     "databricks-sdk>=0.38.0",
 ]