Release notes for 0.29.0 (#255)

Author: Gallaecio

Diff for: .github/workflows/publish.yml

@@ -1,32 +1,23 @@
-# This workflows will upload a Python Package using Twine when a release is created
-# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
-
-name: publish
-
+name: Publish
 on:
   push:
     tags:
-      - "[0-9]+.[0-9]+.[0-9]+"
-
+    - '[0-9]+.[0-9]+.[0-9]+'
 jobs:
-  deploy:
+  publish:
     runs-on: ubuntu-latest
-
+    environment:
+      name: pypi
+      url: https://pypi.org/p/${{ github.event.repository.name }}
+    permissions:
+      id-token: write
     steps:
     - uses: actions/checkout@v4
-    - name: Set up Python
-      uses: actions/setup-python@v5
+    - uses: actions/setup-python@v5
       with:
-        python-version: '3.13'
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install setuptools wheel twine
-    - name: Build and publish
-      if: startsWith(github.ref, 'refs/tags')
-      env:
-        TWINE_USERNAME: __token__
-        TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
-      run: |
-        python setup.py sdist bdist_wheel
-        twine upload dist/*
+        python-version: 3.13
+    - run: |
+        python -m pip install --upgrade build
+        python -m build
+    - name: Publish to PyPI
+      uses: pypa/gh-action-pypi-publish@release/v1

Diff for: .github/workflows/test.yml

@@ -70,7 +70,7 @@ jobs:
       fail-fast: false
       matrix:
         python-version: ["3.12"]  # Keep in sync with .readthedocs.yml
-        tox-job: ["mypy", "linters", "twine-check", "docs"]
+        tox-job: ["mypy", "pre-commit", "twine-check", "docs"]
 
     steps:
     - uses: actions/checkout@v4

Diff for: .pre-commit-config.yaml

@@ -1,19 +1,10 @@
 repos:
--   repo: https://github.com/PyCQA/isort
-    rev: 5.13.2
-    hooks:
-    - id: isort
--   repo: https://github.com/psf/black
-    rev: 24.10.0
-    hooks:
-    - id: black
--   repo: https://github.com/pycqa/flake8
-    rev: 7.1.1
-    hooks:
-    - id: flake8
-      additional_dependencies:
-        - flake8-docstrings
-        - flake8-print
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  rev: v0.11.0
+  hooks:
+    - id: ruff
+      args: [ --fix ]
+    - id: ruff-format
 - repo: https://github.com/adamchainz/blacken-docs
   rev: 1.19.0
   hooks:

Diff for: CHANGES.rst

@@ -1,6 +1,24 @@
 Changes
 =======
 
+0.29.0 (unreleased)
+-------------------
+
+-   Improve the removal and mapping of proxy headers accidentally included in
+    requests:
+
+    -   Remove or map :ref:`Zyte API proxy mode headers <zapi-proxy-headers>`
+        (``Zyte-…``), not only :ref:`Smart Proxy Manager headers
+        <spm-request-headers>` (``X-Crawlera-…``).
+
+    -   Remove or map headers defined through
+        :http:`request:customHttpRequestHeaders`, not only those defined in
+        :attr:`Request.headers <scrapy.http.Request.headers>`.
+
+-   Support :meth:`~scrapy.Spider.start_requests` yielding items, which is
+    possible since Scrapy 2.12.
+
+
 0.28.0 (2025-02-18)
 -------------------
 

Diff for: pyproject.toml

@@ -1,5 +1,47 @@
-[tool.black]
-target-version = ["py39", "py310", "py311", "py312", "py313"]
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "scrapy-zyte-api"
+version = "0.28.0"
+description = "Client library to process URLs through Zyte API"
+authors = [{name = "Zyte Group Ltd", email = "[email protected]"}]
+license = "BSD-3-Clause"
+readme = "README.rst"
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "Natural Language :: English",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+]
+requires-python = ">=3.9"
+# Sync with [pinned] @ tox.ini
+dependencies = [
+    "packaging>=20.0",
+    "scrapy>=2.0.1",
+    "zyte-api>=0.5.1",
+]
+
+[project.optional-dependencies]
+# Sync with [testenv:pinned-provider] @ tox.ini
+provider = [
+    "andi>=0.6.0",
+    "scrapy-poet>=0.22.3",
+    "web-poet>=0.17.0",
+    "zyte-common-items>=0.27.0",
+]
+
+[project.urls]
+source = "https://github.com/scrapy-plugins/scrapy-zyte-api"
+docs = "https://scrapy-zyte-api.readthedocs.io/en/latest/"
 
 [tool.bumpversion]
 current_version = "0.28.0"
@@ -16,12 +58,17 @@ regex = true
 [[tool.bumpversion.files]]
 filename = "docs/conf.py"
 
+[[tool.bumpversion.files]]
+filename = "pyproject.toml"
+parse = "version\\s*=\\s*['\"](?P<major>\\d+)\\.(?P<minor>\\d+)\\.(?P<patch>\\d+)"
+
 [[tool.bumpversion.files]]
 filename = "scrapy_zyte_api/__version__.py"
 
-[tool.isort]
-profile = "black"
-multi_line_output = 3
+[tool.hatch.build.targets.sdist]
+include = [
+  "scrapy_zyte_api/*.py",
+]
 
 [tool.mypy]
 check_untyped_defs = true

Diff for: requirements-dev.txt

@@ -46,3 +46,27 @@
 #: Instance of :class:`web_poet.rules.RulesRegistry` that holds :ref:`session
 #: configs <session-configs>`.
 session_config_registry = _session_config_registry
+
+__all__ = [
+    "ExtractFrom",
+    "Actions",
+    "Geolocation",
+    "Screenshot",
+    "ScrapyZyteAPIDownloadHandler",
+    "ScrapyZyteAPIRequestFingerprinter",
+    "ScrapyZyteAPIDownloaderMiddleware",
+    "ScrapyZyteAPIRefererSpiderMiddleware",
+    "ScrapyZyteAPISpiderMiddleware",
+    "ScrapyZyteAPISessionDownloaderMiddleware",
+    "Addon",
+    "actions",
+    "custom_attrs",
+    "get_request_session_id",
+    "is_session_init_request",
+    "session_config",
+    "session_config_registry",
+    "LocationSessionConfig",
+    "SessionConfig",
+    "SESSION_DEFAULT_RETRY_POLICY",
+    "SESSION_AGGRESSIVE_RETRY_POLICY",
+]

Diff for: scrapy_zyte_api/__init__.py

@@ -11,8 +11,8 @@ class ExtractFrom(str, Enum):
     See :ref:`annotations`.
     """
 
-    httpResponseBody: str = "httpResponseBody"
-    browserHtml: str = "browserHtml"
+    httpResponseBody = "httpResponseBody"
+    browserHtml = "browserHtml"
 
 
 class _Selector(TypedDict, total=False):

Diff for: scrapy_zyte_api/_annotations.py

@@ -209,7 +209,6 @@ async def process_spider_output_async(self, response, result, spider):
 
 
 class ScrapyZyteAPIRefererSpiderMiddleware:
-
     @classmethod
     def from_crawler(cls, crawler):
         return cls(crawler)

Diff for: scrapy_zyte_api/_middlewares.py

@@ -744,7 +744,7 @@ def _set_http_response_headers_from_request(
         api_params.setdefault("httpResponseHeaders", True)
     elif (
         api_params.get("httpResponseHeaders") is False
-        and not default_params.get("httpResponseHeaders") is False
+        and default_params.get("httpResponseHeaders") is not False
     ):
         logger.warning(
             "You do not need to set httpResponseHeaders to False if "

Diff for: scrapy_zyte_api/_params.py

@@ -121,8 +121,7 @@ def get_retry_request(  # type: ignore[misc]
             return new_request
         stats.inc_value(f"{stats_base_key}/max_reached")
         logger.error(
-            "Gave up retrying %(request)s (failed %(retry_times)d times): "
-            "%(reason)s",
+            "Gave up retrying %(request)s (failed %(retry_times)d times): %(reason)s",
             {"request": request, "retry_times": retry_times, "reason": reason},
             extra={"spider": spider},
         )
@@ -453,7 +452,6 @@ def check(self, response: Response, request: Request) -> bool:
 except ImportError:
 
     class SessionConfigRulesRegistry:
-
         def session_config_cls(self, request: Request) -> Type[SessionConfig]:
             return SessionConfig
 
@@ -522,8 +520,7 @@ def check(self, response: Response, request: Request) -> bool:
             and session configs are registered in their own rule registry.
             """
             raise RuntimeError(
-                "To use the @session_config decorator you first must install "
-                "web-poet."
+                "To use the @session_config decorator you first must install web-poet."
             )
 
 else:
@@ -532,14 +529,15 @@ def check(self, response: Response, request: Request) -> bool:
     from web_poet.rules import Strings
 
     class SessionConfigRulesRegistry(RulesRegistry):  # type: ignore[no-redef]
-
         def __init__(self):
             rules = [ApplyRule(for_patterns=Patterns(include=[""]), use=SessionConfig)]  # type: ignore[arg-type]
             super().__init__(rules=rules)
 
         def session_config_cls(self, request: Request) -> Type[SessionConfig]:
             cls = SessionConfig
-            overrides: Dict[Type[SessionConfig], Type[SessionConfig]] = self.overrides_for(request.url)  # type: ignore[assignment]
+            overrides: Dict[Type[SessionConfig], Type[SessionConfig]] = (
+                self.overrides_for(request.url)  # type: ignore[assignment]
+            )
             while cls in overrides:
                 cls = overrides[cls]
             return cls
@@ -563,7 +561,6 @@ def session_config(
 
 
 class FatalErrorHandler:
-
     def __init__(self, crawler):
         self.crawler = crawler
 
@@ -593,7 +590,6 @@ def __exit__(self, exc_type, exc, tb):
 
 
 class _SessionManager:
-
     def __init__(self, crawler: Crawler):
         self._crawler = crawler
 
@@ -992,7 +988,6 @@ def handle_expiration(self, request: Request):
 
 
 class ScrapyZyteAPISessionDownloaderMiddleware:
-
     @classmethod
     def from_crawler(cls, crawler: Crawler):
         return cls(crawler)

Diff for: scrapy_zyte_api/_session.py

@@ -88,12 +88,12 @@ def update_settings(self, settings: BaseSettings) -> None:
                 "addon",
             )
 
-        settings["DOWNLOAD_HANDLERS"][
-            "http"
-        ] = "scrapy_zyte_api.handler.ScrapyZyteAPIHTTPDownloadHandler"
-        settings["DOWNLOAD_HANDLERS"][
-            "https"
-        ] = "scrapy_zyte_api.handler.ScrapyZyteAPIHTTPSDownloadHandler"
+        settings["DOWNLOAD_HANDLERS"]["http"] = (
+            "scrapy_zyte_api.handler.ScrapyZyteAPIHTTPDownloadHandler"
+        )
+        settings["DOWNLOAD_HANDLERS"]["https"] = (
+            "scrapy_zyte_api.handler.ScrapyZyteAPIHTTPSDownloadHandler"
+        )
         _setdefault(
             settings, "DOWNLOADER_MIDDLEWARES", ScrapyZyteAPIDownloaderMiddleware, 633
         )