Merge branch 'devel' into fix/3269-fixes-athena-refresh-mode

Author: rudolfix

.github/workflows/main.yml

@@ -2,7 +2,7 @@ name: dlt
 
 on:
   pull_request:
-    branches: [ master, devel ]
+    branches: [ master, devel, runtime ]
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}

.github/workflows/test_tools_dashboard.yml

@@ -96,7 +96,7 @@ jobs:
         if: matrix.python-version != '3.14.0-beta.4'
 
       - name: Install playwright & deps
-        run: playwright install && playwright install-deps
+        run: playwright install chromium && playwright install-deps
         if: matrix.python-version != '3.14.0-beta.4'
 
       # Run workspace dashboard unit tests
@@ -107,16 +107,8 @@ jobs:
       # Run workspace dashboard e2e tests (does not pass with python 3.9
       - name: Run dashboard e2e
         run: |
-          marimo run --headless dlt/_workspace/helpers/dashboard/dlt_dashboard.py -- -- --pipelines-dir _storage/.dlt/pipelines/ --with_test_identifiers true & pytest --browser chromium tests/e2e
-        if: matrix.python-version != '3.9' && matrix.python-version != '3.14.0-beta.4' && matrix.os != 'windows-latest'
-
-      # note that this test will pass only when running from cmd shell (_storage\.dlt\pipelines\ must stay)
-      - name: Run dashboard e2e windows
-        run: |
-          start marimo run --headless dlt/_workspace/helpers/dashboard/dlt_dashboard.py -- -- --pipelines-dir _storage\.dlt\pipelines\ --with_test_identifiers true
-          timeout /t 6 /nobreak
           pytest --browser chromium tests/e2e
-        if: matrix.python-version != '3.9' && matrix.python-version != '3.14.0-beta.4' && matrix.os == 'windows-latest'
+        if: matrix.python-version != '3.9' && matrix.python-version != '3.14.0-beta.4'
 
   matrix_job_required_check:
     name: common | common tests

Makefile

@@ -173,9 +173,6 @@ test-e2e-dashboard:
 test-e2e-dashboard-headed:
 	uv run pytest --headed --browser chromium tests/e2e
 
-start-dlt-dashboard-e2e:
-	uv run marimo run --headless dlt/_workspace/helpers/dashboard/dlt_dashboard.py -- -- --pipelines-dir _storage/.dlt/pipelines --with_test_identifiers true
-
 # creates the dashboard test pipelines globally for manual testing of the dashboard app and cli
 create-test-pipelines:
 	uv run python tests/workspace/helpers/dashboard/example_pipelines.py

dlt/_workspace/_templates/_single_file_templates/vibe_rest_api_pipeline.py

@@ -1,46 +1,40 @@
-"""A template that is a good start for vibe coding REST API Source. Works best with `dlt ai` command cursor rules"""
+"""Template for building a `dlt` pipeline to ingest data from a REST API."""
 
 import dlt
-from dlt.sources.rest_api import (
-    RESTAPIConfig,
-    rest_api_resources,
-)
+from dlt.sources.rest_api import rest_api_resources
+from dlt.sources.rest_api.typing import RESTAPIConfig
 
 
+# if no argument is provided, `access_token` is read from `.dlt/secrets.toml`
 @dlt.source
-def source(access_token=dlt.secrets.value):
+def rest_api_source(access_token: str = dlt.secrets.value):
+    """Define dlt resources from REST API endpoints."""
     config: RESTAPIConfig = {
         "client": {
-            # TODO: place valid base url here
+            # TODO set base URL for the REST API
             "base_url": "https://example.com/v1/",
-            # TODO: configure the right auth or remove if api does not need authentication
-            # NOTE: pass secrets and other configuration in source function signature
-            "auth": {
-                "type": "bearer",
-                "token": access_token,
-            },
+            # TODO configure the right authentication method or remove
+            "auth": {"type": "bearer", "token": access_token},
         },
         "resources": [
-            # TODO: add resource definitions here
+            # TODO define resources per endpoint
         ],
+        # set `resource_defaults` to apply configuration to all endpoints
     }
 
     yield from rest_api_resources(config)
 
 
-def get_data() -> None:
-    pipeline = dlt.pipeline(
-        pipeline_name="rest_api_github",
-        destination="duckdb",
-        dataset_name="rest_api_data",
-    )
-
-    # TODO: during debugging feel free to pass access token explicitly
-    # NOTE: use `secrets.toml` or env variables to pass configuration in production
-    access_token = "my_access_token"
-    load_info = pipeline.run(source(access_token))
-    print(load_info)  # noqa
+pipeline = dlt.pipeline(
+    pipeline_name="rest_api_ingest",
+    destination="duckdb",
+    # `refresh="drop_sources"` ensures the data and the state is cleaned
+    # on each `pipeline.run()`; remove the argument once you have a
+    # working pipeline.
+    refresh="drop_sources",
+)
 
 
 if __name__ == "__main__":
-    get_data()
+    load_info = pipeline.run(rest_api_source())
+    print(load_info)  # noqa: T201

dlt/_workspace/cli/_init_command.py

@@ -340,7 +340,7 @@ def init_pipeline_at_destination(
     transformed_nodes = source_detection.find_call_arguments_to_replace(
         visitor,
         [
-            ("destination", destination_type or "duckdb"),
+            ("destination", destination_type or "duckdb", True),
         ],
         source_configuration.src_pipeline_script,
     )
@@ -372,9 +372,9 @@ def init_pipeline_at_destination(
             transformed_nodes = source_detection.find_call_arguments_to_replace(
                 visitor,
                 [
-                    ("destination", destination_type),
-                    ("pipeline_name", display_source_name + "_pipeline"),
-                    ("dataset_name", display_source_name + "_data"),
+                    ("destination", destination_type, True),
+                    ("pipeline_name", display_source_name + "_pipeline", True),
+                    ("dataset_name", display_source_name + "_data", False),
                 ],
                 source_configuration.src_pipeline_script,
             )

dlt/_workspace/cli/source_detection.py

@@ -14,7 +14,9 @@
 
 
 def find_call_arguments_to_replace(
-    visitor: PipelineScriptVisitor, replace_nodes: List[Tuple[str, str]], init_script_name: str
+    visitor: PipelineScriptVisitor,
+    replace_nodes: List[Tuple[str, str, bool]],
+    init_script_name: str,
 ) -> List[Tuple[ast.AST, ast.AST]]:
     # the input tuple (call argument name, replacement value)
     # the returned tuple (node, replacement value, node type)
@@ -23,7 +25,7 @@ def find_call_arguments_to_replace(
     known_calls: Dict[str, List[inspect.BoundArguments]] = visitor.known_calls
     for arg_name, calls in known_calls.items():
         for args in calls:
-            for t_arg_name, t_value in replace_nodes:
+            for t_arg_name, t_value, _ in replace_nodes:
                 dn_node: ast.AST = args.arguments.get(t_arg_name)
                 if dn_node is not None:
                     if not isinstance(dn_node, ast.Constant) or not isinstance(dn_node.value, str):
@@ -36,8 +38,8 @@ def find_call_arguments_to_replace(
                         replaced_args.add(t_arg_name)
 
     # there was at least one replacement
-    for t_arg_name, _ in replace_nodes:
-        if t_arg_name not in replaced_args:
+    for t_arg_name, _, required in replace_nodes:
+        if t_arg_name not in replaced_args and required:
             raise CliCommandInnerException(
                 "init",
                 f"The pipeline script {init_script_name} is not explicitly passing the"