implement ZYTE_API_DEFAULT_PARAMS in the settings

BurnzZ · BurnzZ · commit 97ea1e4b6ec7 · 2022-05-03T10:54:32.000+08:00
diff --git a/README.rst b/README.rst
@@ -79,6 +79,8 @@ You can see the full list of parameters in the `Zyte API Specification
 
 On the other hand, you could also control it on a per request basis by setting the
 ``zyte_api`` key in `Request.meta <https://docs.scrapy.org/en/latest/topics/request-response.html#scrapy.http.Request.meta>`_.
+When doing so, it will override any parameters that was set via the setting
+named ``ZYTE_API_DEFAULT_PARAMS``.
 
 .. code-block:: python
 
diff --git a/scrapy_zyte_api/handler.py b/scrapy_zyte_api/handler.py
@@ -31,6 +31,7 @@ def __init__(
         )
         self._stats = crawler.stats
         self._job_id = crawler.settings.get("JOB")
+        self._zyte_api_default_params = settings.getdict("ZYTE_API_DEFAULT_PARAMS")
         self._session = create_session()
 
     @classmethod
@@ -56,11 +57,14 @@ def download_request(self, request: Request, spider: Spider) -> Deferred:
     async def _download_request(
         self, request: Request, spider: Spider
     ) -> Union[ZyteAPITextResponse, ZyteAPIResponse]:
-        api_params: Dict[str, Any] = request.meta["zyte_api"]
-        if not isinstance(api_params, dict):
+        api_params: Dict[str, Any] = self._zyte_api_default_params or {}
+        try:
+            api_params.update(request.meta.get("zyte_api") or {})
+        except TypeError:
             logger.error(
-                "zyte_api parameters in the request meta should be "
-                f"provided as dictionary, got {type(api_params)} instead ({request.url})."
+                f"zyte_api parameters in the request meta should be "
+                f"provided as dictionary, got {type(request.meta.get('zyte_api'))} "
+                f"instead ({request.url})."
             )
             raise IgnoreRequest()
         # Define url by default
diff --git a/tests/test_api_requests.py b/tests/test_api_requests.py
@@ -1,6 +1,7 @@
 import os
 from asyncio import iscoroutine
 from typing import Any, Dict
+from unittest import mock
 
 import pytest
 from _pytest.logging import LogCaptureFixture  # NOQA
@@ -23,17 +24,16 @@
 
 
 class TestAPI:
-
     @staticmethod
-    async def produce_request_response(meta):
+    async def produce_request_response(meta, custom_settings=None):
         with MockServer() as server:
-            async with make_handler({}, server.urljoin("/")) as handler:
+            async with make_handler(custom_settings, server.urljoin("/")) as handler:
                 req = Request(
                     "http://example.com",
                     method="POST",
                     meta=meta,
                 )
-                coro = handler._download_request(req, Spider("test"))
+                coro = handler._download_request(req, None)
                 assert iscoroutine(coro)
                 assert not isinstance(coro, Deferred)
                 resp = await coro  # type: ignore
@@ -101,6 +101,53 @@ async def test_http_response_headers_request(self, meta: Dict[str, Dict[str, Any
         assert resp.body == b"<html></html>"
         assert resp.headers == {b"Test_Header": [b"test_value"]}
 
+    @pytest.mark.parametrize(
+        "meta,custom_settings,expected",
+        [
+            ({}, {}, {}),
+            ({"zyte_api": {}}, {}, {}),
+            (
+                {},
+                {"ZYTE_API_DEFAULT_PARAMS": {"browserHtml": True, "geolocation": "CA"}},
+                {"browserHtml": True, "geolocation": "CA"},
+            ),
+            (
+                {"zyte_api": {}},
+                {"ZYTE_API_DEFAULT_PARAMS": {"browserHtml": True, "geolocation": "CA"}},
+                {"browserHtml": True, "geolocation": "CA"},
+            ),
+            (
+                {"zyte_api": {"javascript": True, "geolocation": "US"}},
+                {"ZYTE_API_DEFAULT_PARAMS": {"browserHtml": True, "geolocation": "CA"}},
+                {"browserHtml": True, "geolocation": "US", "javascript": True},
+            ),
+        ],
+    )
+    @mock.patch("tests.AsyncClient")
+    @pytest.mark.asyncio
+    async def test_empty_zyte_api_request_meta(
+        self,
+        mock_client,
+        meta: Dict[str, Dict[str, Any]],
+        custom_settings: Dict[str, str],
+        expected: Dict[str, str],
+    ):
+        try:
+            # This would always error out since the mocked client doesn't
+            # return the expected API response.
+            await self.produce_request_response(meta, custom_settings=custom_settings)
+        except:
+            pass
+
+        request_call = [c for c in mock_client.mock_calls if "request_raw(" in str(c)]
+        if not request_call:
+            pytest.fail("The client's request_raw() method was not called.")
+
+        args_used = request_call[0].args[0]
+        args_used.pop("url")
+
+        assert args_used == expected
+
     @pytest.mark.parametrize(
         "meta, api_relevant",
         [
