diff --git a/README.md b/README.md index 6cf81d8..e79e132 100644 --- a/README.md +++ b/README.md @@ -173,6 +173,26 @@ PLAYWRIGHT_LAUNCH_OPTIONS = { } ``` +### `PLAYWRIGHT_BROWSER_PROVIDER` +Type `str` or `type`, default `"scrapy_playwright.provider.PlaywrightBrowserProvider"` + +A class that owns the browser lifecycle (startup, launching/connecting browsers, optional +persistent contexts, teardown). The value might be either an import path string or the provider +class itself. The provider is instantiated with the handler configuration object as argument. + +The default provider wraps vanilla Playwright and supports everything documented +in this README (local launch, `PLAYWRIGHT_CDP_URL`, `PLAYWRIGHT_CONNECT_URL`, persistent +contexts, etc). This is an extension point for integrating third-party drivers that +expose Playwright-compatible `Browser`/`BrowserContext`/`Page` objects (e.g. +[patchright](https://pypi.org/project/patchright/), +[camoufox](https://pypi.org/project/camoufox/)) without changing the handler. +See [`docs/pluggable-browser-providers.md`](docs/pluggable-browser-providers.md) for the interface +and ready-made example providers. + +```python +PLAYWRIGHT_BROWSER_PROVIDER = "myproject.providers.CustomBrowserProvider" +``` + ### `PLAYWRIGHT_CDP_URL` Type `Optional[str]`, default `None` diff --git a/docs/changelog.md b/docs/changelog.md index bf3a834..2e2be5c 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,6 +1,11 @@ # scrapy-playwright changelog +### [v0.0.48](https://github.com/scrapy-plugins/scrapy-playwright/releases/tag/v0.0.48) (unreleased) + +* Support for third-party browser providers (`PLAYWRIGHT_BROWSER_PROVIDER` setting) + + ### [v0.0.47](https://github.com/scrapy-plugins/scrapy-playwright/releases/tag/v0.0.47) (2026-06-13) * Python 3.14 support (#375) diff --git a/docs/pluggable-browser-providers.md b/docs/pluggable-browser-providers.md new file mode 100644 index 0000000..4d639a5 --- /dev/null +++ b/docs/pluggable-browser-providers.md @@ -0,0 +1,251 @@ +# Pluggable browser providers + +Third-party projects such as [patchright](https://pypi.org/project/patchright/), +[camoufox](https://pypi.org/project/camoufox/), and +[invisible_playwright](https://github.com/feder-cr/invisible_playwright) provide +drop-in replacements for Playwright's browser startup while keeping standard +`Browser`/`BrowserContext`/`Page` objects. Because those objects are unchanged, +pages, contexts and routing code keeps working — only how the browser is +started and stopped differs. + +scrapy-playwright exposes a single extension point for this: the +`PLAYWRIGHT_BROWSER_PROVIDER` setting. It takes the import path of a *browser +provider* — a class that owns the browser lifecycle. The built-in provider wraps +vanilla Playwright; you can integrate any other backend by pointing the setting +at your own provider class. + +## The `PLAYWRIGHT_BROWSER_PROVIDER` setting + +Type `str` or `type`, default `"scrapy_playwright.provider.PlaywrightBrowserProvider"`. + +```python +# settings.py +PLAYWRIGHT_BROWSER_PROVIDER = "myproject.providers.CustomBrowserProvider" +``` + +The value may be either an import path string or the provider class directly. The +class is instantiated with a configuration object (see +[Reading configuration](#reading-configuration)) and drives every browser that +scrapy-playwright uses. When the setting is not set, the built-in +`PlaywrightBrowserProvider` is used. + +## The `BrowserProvider` interface + +A provider is a class that accepts a configuration object in its constructor and +implements the following asynchronous lifecycle. You can subclass +`scrapy_playwright.provider.BrowserProvider` (a `typing.Protocol`) to document +the intent, but it is not required — any class with these methods works. + +```python +from playwright.async_api import Browser, BrowserContext +from scrapy.exceptions import NotSupported +from scrapy_playwright.handler import Config + + +class BrowserProvider: + def __init__(self, config: Config) -> None: + ... + + async def start(self) -> None: + """Perform any one-time initialization. + + Called once before the first browser is requested. Providers that create + their browser on demand can leave this empty. + """ + + async def launch_browser(self) -> Browser: + """Return a launched or connected Playwright-compatible ``Browser``. + + Called when a browser is needed, and again if the previous browser + disconnects and ``PLAYWRIGHT_RESTART_DISCONNECTED_BROWSER`` is enabled. + """ + + async def launch_persistent_context(self, context_kwargs: dict) -> BrowserContext: + """Return a persistent ``BrowserContext``. + + Called when a context requests a ``user_data_dir``. Raise + ``scrapy.exceptions.NotSupported`` if the backend has no equivalent. + """ + raise NotSupported("This provider does not support persistent contexts") + + async def close(self) -> None: + """Release any resources acquired in ``start`` / ``launch_browser``. + + Awaited once when the crawl finishes. + """ +``` + +Import any optional third-party library lazily (inside the methods that need +it), so the setting can point at a provider whose backend is only installed in +some environments without affecting others. + +## Reading configuration + +The object passed to the constructor exposes the resolved Playwright settings as +attributes, so a provider can honor them. The most commonly used are: + +| Attribute | Setting | +|---|---| +| `browser_type_name` | `PLAYWRIGHT_BROWSER_TYPE` | +| `launch_options` | `PLAYWRIGHT_LAUNCH_OPTIONS` | +| `cdp_url` / `cdp_kwargs` | `PLAYWRIGHT_CDP_URL` / `PLAYWRIGHT_CDP_KWARGS` | +| `connect_url` / `connect_kwargs` | `PLAYWRIGHT_CONNECT_URL` / `PLAYWRIGHT_CONNECT_KWARGS` | + +## Built-in provider + +`scrapy_playwright.provider.PlaywrightBrowserProvider` is the default provider. It wraps +vanilla Playwright and supports the full feature set documented in the README: launching a local +browser, connecting to a remote one through `PLAYWRIGHT_CDP_URL` or `PLAYWRIGHT_CONNECT_URL`, +and persistent contexts. Custom providers below follow the same interface. + +## About the examples + +The examples that follow are provided **only as guidelines** to illustrate how a +custom provider can be structured; they are not part of scrapy-playwright and are +not officially supported extensions. They may be incomplete or become outdated as +the third-party libraries evolve — treat them as a starting point and adapt them +as needed. + +scrapy-playwright is **not affiliated with, endorsed by, or otherwise connected +to** any of the third-party projects mentioned here (patchright, camoufox, +invisible_playwright, or any other). They are referenced purely as examples of +Playwright-compatible backends. Refer to each project's own documentation and +license for authoritative, up-to-date usage, and evaluate any third-party +dependency yourself before using it. + +## Example: patchright + +Patchright mirrors the Playwright async API (Chromium only), so a provider can +use it exactly like Playwright, including persistent contexts. + +```python +from contextlib import AsyncExitStack + +from scrapy_playwright.handler import Config + + +class PatchrightBrowserProvider: + def __init__(self, config: Config) -> None: + self.config = config + self.stack = AsyncExitStack() + self.browser_type: BrowserType + + async def start(self) -> None: + from patchright.async_api import async_playwright + + _patchright = await self.stack.enter_async_context(async_playwright()) + self.browser_type = _patchright.chromium + + async def launch_browser(self): + return await self.browser_type.launch(**self.config.launch_options) + + async def launch_persistent_context(self, context_kwargs: dict): + return await self.browser_type.launch_persistent_context(**context_kwargs) + + async def close(self) -> None: + await self.stack.aclose() +``` + +```python +# settings +PLAYWRIGHT_BROWSER_PROVIDER = "myproject.providers.PatchrightBrowserProvider" +PLAYWRIGHT_BROWSER_TYPE = "chromium" +``` + +## Example: camoufox + +`AsyncCamoufox` is an async context manager that yields a `Browser` directly, or +a persistent `BrowserContext` when passed `persistent_context=True` and a +`user_data_dir`. It is Firefox-based, so set `PLAYWRIGHT_BROWSER_TYPE = "firefox"`. + +```python +from contextlib import AsyncExitStack + +from scrapy_playwright.handler import Config, PERSISTENT_CONTEXT_PATH_KEY + + +class CamoufoxBrowserProvider: + def __init__(self, config: Config) -> None: + self.config = config + self.stack = AsyncExitStack() + + async def start(self) -> None: + pass + + async def launch_browser(self): + from camoufox.async_api import AsyncCamoufox + + return await self.stack.enter_async_context( + AsyncCamoufox(**self.config.launch_options) + ) + + async def launch_persistent_context(self, context_kwargs: dict): + from camoufox.async_api import AsyncCamoufox + + return await self.stack.enter_async_context( + AsyncCamoufox( + persistent_context=True, + user_data_dir=context_kwargs[PERSISTENT_CONTEXT_PATH_KEY], + **self.config.launch_options, + ) + ) + + async def close(self) -> None: + await self.stack.aclose() +``` + +```python +# settings +PLAYWRIGHT_BROWSER_PROVIDER = "myproject.providers.CamoufoxBrowserProvider" +PLAYWRIGHT_BROWSER_TYPE = "firefox" +``` + +## Example: invisible_playwright + +Same shape as camoufox — `InvisiblePlaywright(...)` is an async context manager +yielding a standard `playwright.async_api.Browser`, or a persistent +`BrowserContext` when passed a `profile_dir`. Firefox-based, so set +`PLAYWRIGHT_BROWSER_TYPE = "firefox"`. + +```python +from contextlib import AsyncExitStack + +from scrapy_playwright.handler import Config, PERSISTENT_CONTEXT_PATH_KEY + + +class InvisibleBrowserProvider: + def __init__(self, config: Config) -> None: + self.config = config + self.stack = AsyncExitStack() + + async def start(self) -> None: + pass + + async def launch_browser(self): + from invisible_playwright.async_api import InvisiblePlaywright + + # e.g. seed / proxy / timezone / pin via PLAYWRIGHT_LAUNCH_OPTIONS + return await self.stack.enter_async_context( + InvisiblePlaywright(**self.config.launch_options) + ) + + async def launch_persistent_context(self, context_kwargs: dict): + from invisible_playwright.async_api import InvisiblePlaywright + + # invisible_playwright names the profile path ``profile_dir`` + return await self.stack.enter_async_context( + InvisiblePlaywright( + profile_dir=context_kwargs[PERSISTENT_CONTEXT_PATH_KEY], + **self.config.launch_options, + ) + ) + + async def close(self) -> None: + await self.stack.aclose() +``` + +```python +# settings +PLAYWRIGHT_BROWSER_PROVIDER = "myproject.providers.InvisibleBrowserProvider" +PLAYWRIGHT_BROWSER_TYPE = "firefox" +``` diff --git a/scrapy_playwright/handler.py b/scrapy_playwright/handler.py index 50b8a89..4f135f5 100644 --- a/scrapy_playwright/handler.py +++ b/scrapy_playwright/handler.py @@ -7,17 +7,24 @@ from importlib.metadata import version as package_version from ipaddress import ip_address from time import time -from typing import Awaitable, Callable, Dict, Optional, Tuple, Type, TypeVar, Union +from typing import ( + TYPE_CHECKING, + Awaitable, + Callable, + Dict, + Optional, + Tuple, + Type, + TypeVar, + Union, +) from playwright._impl._errors import TargetClosedError from playwright.async_api import ( BrowserContext, - BrowserType, Download as PlaywrightDownload, Error as PlaywrightError, Page, - Playwright as AsyncPlaywright, - PlaywrightContextManager, Request as PlaywrightRequest, Response as PlaywrightResponse, Route, @@ -52,8 +59,11 @@ _set_redirect_meta, ) +if TYPE_CHECKING: + from scrapy_playwright.provider import BrowserProvider + -__all__ = ["ScrapyPlaywrightDownloadHandler"] +__all__ = ["ScrapyPlaywrightDownloadHandler", "Config"] _SCRAPY_ASYNC_API = scrapy_version_info >= (2, 14, 0) @@ -67,6 +77,7 @@ DEFAULT_BROWSER_TYPE = "chromium" DEFAULT_CONTEXT_NAME = "default" +DEFAULT_BROWSER_PROVIDER = "scrapy_playwright.provider.PlaywrightBrowserProvider" PERSISTENT_CONTEXT_PATH_KEY = "user_data_dir" @@ -144,8 +155,7 @@ def from_settings(cls, settings: Settings) -> "Config": class ScrapyPlaywrightDownloadHandler(HTTP11DownloadHandler): - playwright_context_manager: Optional[PlaywrightContextManager] = None - playwright: Optional[AsyncPlaywright] = None + browser_provider: "BrowserProvider" def __init__(self, crawler: Crawler) -> None: verify_installed_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") @@ -157,6 +167,9 @@ def __init__(self, crawler: Crawler) -> None: ) self.stats = crawler.stats self.config = Config.from_settings(crawler.settings) + self.browser_provider_cls: Type["BrowserProvider"] = load_object( + crawler.settings.get("PLAYWRIGHT_BROWSER_PROVIDER") or DEFAULT_BROWSER_PROVIDER + ) if self.config.use_threaded_loop: _ThreadedLoopAdapter.start(id(self)) @@ -214,9 +227,8 @@ async def _launch(self) -> None: __version__, package_version("playwright"), ) - self.playwright_context_manager = PlaywrightContextManager() - self.playwright = await self.playwright_context_manager.start() - self.browser_type: BrowserType = getattr(self.playwright, self.config.browser_type_name) + self.browser_provider = self.browser_provider_cls(self.config) + await self.browser_provider.start() if self.config.startup_context_kwargs: logger.info("Launching %i startup context(s)", len(self.config.startup_context_kwargs)) await asyncio.gather( @@ -230,33 +242,14 @@ async def _launch(self) -> None: self.stats.set_value("playwright/page_count", self._get_total_page_count()) async def _maybe_launch_browser(self) -> None: - async with self.browser_launch_lock: - if not hasattr(self, "browser"): - logger.info("Launching browser %s", self.browser_type.name) - self.browser = await self.browser_type.launch(**self.config.launch_options) - logger.info("Browser %s launched", self.browser_type.name) - self.stats.inc_value("playwright/browser_count") - self.browser.on("disconnected", self._browser_disconnected_callback) + """Obtain a browser from the provider if one is not available yet. - async def _maybe_connect_remote_devtools(self) -> None: - async with self.browser_launch_lock: - if not hasattr(self, "browser"): - logger.info("Connecting using CDP: %s", self.config.cdp_url) - self.browser = await self.browser_type.connect_over_cdp( - self.config.cdp_url, **self.config.cdp_kwargs - ) - logger.info("Connected using CDP: %s", self.config.cdp_url) - self.stats.inc_value("playwright/browser_count") - self.browser.on("disconnected", self._browser_disconnected_callback) - - async def _maybe_connect_remote(self) -> None: + The provider decides whether to launch locally or connect to a remote + browser; this method owns the lock, stats and disconnection listener. + """ async with self.browser_launch_lock: if not hasattr(self, "browser"): - logger.info("Connecting to remote Playwright") - self.browser = await self.browser_type.connect( - self.config.connect_url, **self.config.connect_kwargs - ) - logger.info("Connected to remote Playwright") + self.browser = await self.browser_provider.launch_browser() self.stats.inc_value("playwright/browser_count") self.browser.on("disconnected", self._browser_disconnected_callback) @@ -277,19 +270,12 @@ async def _create_browser_context( context_kwargs = context_kwargs or {} persistent = remote = False if context_kwargs.get(PERSISTENT_CONTEXT_PATH_KEY): - context = await self.browser_type.launch_persistent_context(**context_kwargs) + context = await self.browser_provider.launch_persistent_context(context_kwargs) persistent = True - elif self.config.cdp_url: - await self._maybe_connect_remote_devtools() - context = await self.browser.new_context(**context_kwargs) - remote = True - elif self.config.connect_url: - await self._maybe_connect_remote() - context = await self.browser.new_context(**context_kwargs) - remote = True else: await self._maybe_launch_browser() context = await self.browser.new_context(**context_kwargs) + remote = bool(self.config.cdp_url or self.config.connect_url) except Exception: if acquired: self.context_semaphore.release() @@ -416,10 +402,8 @@ async def _close(self) -> None: if hasattr(self, "browser"): logger.info("Closing browser") await self.browser.close() - if self.playwright_context_manager: - await self.playwright_context_manager.__aexit__() - if self.playwright: - await self.playwright.stop() + if getattr(self, "browser_provider", None): + await self.browser_provider.close() if _SCRAPY_ASYNC_API: diff --git a/scrapy_playwright/memusage.py b/scrapy_playwright/memusage.py index bf7aa52..15dc1c0 100644 --- a/scrapy_playwright/memusage.py +++ b/scrapy_playwright/memusage.py @@ -20,15 +20,18 @@ def __init__(self, *args, **kwargs) -> None: raise NotConfigured("The psutil module is not available") from exc def _get_main_process_ids(self) -> List[int]: + process_ids = [] try: - return [ - handler.playwright_context_manager._connection._transport._proc.pid - for handler in self.crawler.engine.downloader.handlers._handlers.values() - if isinstance(handler, ScrapyPlaywrightDownloadHandler) - and handler.playwright_context_manager - ] + for handler in self.crawler.engine.downloader.handlers._handlers.values(): + if not isinstance(handler, ScrapyPlaywrightDownloadHandler): + continue + provider = getattr(handler, "browser_provider", None) + context_manager = getattr(provider, "playwright_context_manager", None) + if context_manager is not None: + process_ids.append(context_manager._connection._transport._proc.pid) except Exception: return [] + return process_ids def _get_descendant_processes(self, process) -> list: children = process.children() diff --git a/scrapy_playwright/provider.py b/scrapy_playwright/provider.py new file mode 100644 index 0000000..e1dc587 --- /dev/null +++ b/scrapy_playwright/provider.py @@ -0,0 +1,115 @@ +"""Pluggable browser providers. + +A browser provider owns the browser lifecycle: startup, producing +launched/connected :class:`~playwright.async_api.Browser` objects, optionally +producing persistent contexts, and teardown. The download handler delegates to +the provider configured via the ``PLAYWRIGHT_BROWSER_PROVIDER`` setting (an import +path or class), defaulting to :class:`PlaywrightBrowserProvider`. + +Third-party drivers that expose Playwright-compatible ``Browser`` objects (e.g. +patchright, camoufox) can be integrated by implementing a provider, without any +driver-specific code living in the handler. See +``docs/pluggable-browser-providers.md``. +""" + +from typing import TYPE_CHECKING, Optional, Protocol, runtime_checkable + +from playwright.async_api import ( + Browser, + BrowserContext, + BrowserType, + Playwright as AsyncPlaywright, + PlaywrightContextManager, +) + +from scrapy_playwright.handler import logger + +if TYPE_CHECKING: + from scrapy_playwright.handler import Config + + +__all__ = ["BrowserProvider", "PlaywrightBrowserProvider"] + + +@runtime_checkable +class BrowserProvider(Protocol): + """Interface expected of ``PLAYWRIGHT_BROWSER_PROVIDER`` implementations. + + Instances receive a :class:`~scrapy_playwright.handler.Config` object as argument + to their __init__ method. All methods are coroutines and are awaited by the handler. + """ + + def __init__(self, config: "Config") -> None: ... + + async def start(self) -> None: + """One-time initialization, awaited once when the handler launches.""" + + async def launch_browser(self) -> Browser: + """Return a launched or connected Playwright-compatible ``Browser``. + + Called (behind a lock) the first time a browser is needed, and again after a + disconnection if ``PLAYWRIGHT_RESTART_DISCONNECTED_BROWSER`` is enabled. + """ + + async def launch_persistent_context(self, context_kwargs: dict) -> BrowserContext: + """Return a persistent context (a ``user_data_dir`` was requested). + + Providers whose backend has no equivalent should raise + :class:`scrapy.exceptions.NotSupported`. + """ + + async def close(self) -> None: + """Tear down any allocated resources. Awaited once when the handler shuts down.""" + + +class PlaywrightBrowserProvider: + """Default provider, wrapping vanilla Playwright. + + Encapsulates the browser lifecycle: it starts a + :class:`~playwright.async_api.PlaywrightContextManager`, resolves the + configured browser type, and launches locally or connects to a remote + browser (``PLAYWRIGHT_CDP_URL`` / ``PLAYWRIGHT_CONNECT_URL``) as needed. + """ + + def __init__(self, config: "Config") -> None: + self.config = config + self.playwright_context_manager: Optional[PlaywrightContextManager] = None + self.playwright: Optional[AsyncPlaywright] = None + self.browser_type: Optional[BrowserType] = None + + async def start(self) -> None: + self.playwright_context_manager = PlaywrightContextManager() + self.playwright = await self.playwright_context_manager.start() + self.browser_type = getattr(self.playwright, self.config.browser_type_name) + + async def launch_browser(self) -> Browser: + if self.browser_type is None: + raise RuntimeError("start() must be awaited before launch_browser()") + if self.config.cdp_url: + logger.info("Connecting using CDP: %s", self.config.cdp_url) + browser = await self.browser_type.connect_over_cdp( + self.config.cdp_url, **self.config.cdp_kwargs + ) + logger.info("Connected using CDP: %s", self.config.cdp_url) + elif self.config.connect_url: + logger.info("Connecting to remote Playwright") + browser = await self.browser_type.connect( + self.config.connect_url, **self.config.connect_kwargs + ) + logger.info("Connected to remote Playwright") + else: + logger.info("Launching browser %s", self.browser_type.name) + browser = await self.browser_type.launch(**self.config.launch_options) + logger.info("Browser %s launched", self.browser_type.name) + return browser + + async def launch_persistent_context(self, context_kwargs: dict) -> BrowserContext: + if self.browser_type is None: + raise RuntimeError("start() must be awaited before launch_persistent_context()") + return await self.browser_type.launch_persistent_context(**context_kwargs) + + async def close(self) -> None: + if self.playwright_context_manager: + await self.playwright_context_manager.__aexit__() + if self.playwright: + await self.playwright.stop() diff --git a/tests/tests_asyncio/test_extensions.py b/tests/tests_asyncio/test_extensions.py index 88cda11..734707d 100644 --- a/tests/tests_asyncio/test_extensions.py +++ b/tests/tests_asyncio/test_extensions.py @@ -28,12 +28,14 @@ def mock_crawler_with_handlers() -> dict: - handlers = {} + handlers = {"unused": MagicMock()} for schema, pid in SCHEMA_PID_MAP.items(): process = MagicMock() process.pid = pid handlers[schema] = MagicMock(spec=ScrapyPlaywrightDownloadHandler) - handlers[schema].playwright_context_manager._connection._transport._proc = process + handlers[schema].browser_provider = MagicMock() + provider = handlers[schema].browser_provider + provider.playwright_context_manager._connection._transport._proc = process crawler = MagicMock() crawler.engine.downloader.handlers._handlers = handlers return crawler diff --git a/tests/tests_asyncio/test_provider.py b/tests/tests_asyncio/test_provider.py new file mode 100644 index 0000000..6181089 --- /dev/null +++ b/tests/tests_asyncio/test_provider.py @@ -0,0 +1,143 @@ +import tempfile +from typing import Optional +from unittest import IsolatedAsyncioTestCase, TestCase +from uuid import uuid4 + +import pytest +from playwright.async_api import Browser, PlaywrightContextManager, async_playwright +from scrapy import Request, Spider +from scrapy.exceptions import NotSupported +from scrapy.settings import Settings + +from scrapy_playwright.handler import Config +from scrapy_playwright.provider import BrowserProvider, PlaywrightBrowserProvider + +from tests import ( + allow_windows, + assert_correct_response, + create_handler, + make_handler, + BaseTestCase, +) + + +class CustomBrowserProvider: + """A standalone provider (not subclassing the default) that yields a real + Playwright ``Browser``, mirroring how a third-party driver would integrate: + it enters an async context manager and returns the browser it produces. + """ + + def __init__(self, config: Config) -> None: + self.config = config + self._cm: Optional[PlaywrightContextManager] = None + self.started = False + self.launched = False + + async def start(self) -> None: + self.started = True + + async def launch_browser(self) -> Browser: + self.launched = True + self._cm = async_playwright() + playwright = await self._cm.__aenter__() + browser_type = getattr(playwright, self.config.browser_type_name) + return await browser_type.launch(**self.config.launch_options) + + async def launch_persistent_context(self, context_kwargs: dict): + raise NotSupported("CustomBrowserProvider does not support persistent contexts") + + async def close(self) -> None: + if self._cm is not None: + await self._cm.__aexit__(None, None, None) + + +class TestProviderSetting(TestCase): + def test_default_provider_class(self): + handler = create_handler({}) + assert handler.browser_provider_cls is PlaywrightBrowserProvider + + def test_default_provider_class_when_empty(self): + handler = create_handler({"PLAYWRIGHT_BROWSER_PROVIDER": ""}) + assert handler.browser_provider_cls is PlaywrightBrowserProvider + + def test_custom_provider_class_loaded(self): + handler = create_handler({"PLAYWRIGHT_BROWSER_PROVIDER": CustomBrowserProvider}) + assert handler.browser_provider_cls is CustomBrowserProvider + + def test_default_provider_implements_protocol(self): + assert isinstance( + PlaywrightBrowserProvider(Config.from_settings(Settings({}))), BrowserProvider + ) + + +class TestPlaywrightBrowserProvider(IsolatedAsyncioTestCase): + @allow_windows + async def test_launch_before_start_raises(self): + provider = PlaywrightBrowserProvider(Config.from_settings(Settings({}))) + with pytest.raises( + RuntimeError, match="start\\(\\) must be awaited before launch_browser" + ): + await provider.launch_browser() + with pytest.raises( + RuntimeError, match="start\\(\\) must be awaited before launch_persistent_context" + ): + await provider.launch_persistent_context({}) + + @allow_windows + async def test_lifecycle(self): + config = Config.from_settings( + Settings( + { + "PLAYWRIGHT_BROWSER_TYPE": "chromium", + "PLAYWRIGHT_LAUNCH_OPTIONS": {"headless": True}, + } + ) + ) + provider = PlaywrightBrowserProvider(config) + await provider.start() + assert provider.browser_type is not None + assert provider.browser_type.name == "chromium" + browser = await provider.launch_browser() + try: + assert isinstance(browser, Browser) + assert browser.is_connected() + finally: + await browser.close() + await provider.close() + + +class TestCustomProvider(IsolatedAsyncioTestCase, BaseTestCase): + @allow_windows + async def test_custom_provider_used_end_to_end(self): + settings = { + "PLAYWRIGHT_BROWSER_TYPE": "chromium", + "PLAYWRIGHT_BROWSER_PROVIDER": CustomBrowserProvider, + "PLAYWRIGHT_LAUNCH_OPTIONS": {"headless": True}, + } + async with make_handler(settings) as handler: + assert isinstance(handler.browser_provider, CustomBrowserProvider) + assert handler.browser_provider.started + req = Request(self.static_server.urljoin("/index.html"), meta={"playwright": True}) + resp = await handler._download_request(req, Spider("foo")) + assert_correct_response(resp, req) + assert handler.browser_provider.launched + assert isinstance(handler.browser, Browser) + + @allow_windows + async def test_persistent_context_not_supported(self): + temp_dir = f"{tempfile.gettempdir()}/{uuid4()}" + settings = { + "PLAYWRIGHT_BROWSER_TYPE": "chromium", + "PLAYWRIGHT_BROWSER_PROVIDER": CustomBrowserProvider, + } + async with make_handler(settings) as handler: + req = Request( + self.static_server.urljoin("/index.html"), + meta={ + "playwright": True, + "playwright_context": "persistent", + "playwright_context_kwargs": {"user_data_dir": temp_dir}, + }, + ) + with pytest.raises(NotSupported): + await handler._download_request(req, Spider("foo"))