add integration tests for web-poet

Author: BurnzZ

tests/po_lib/__init__.py

@@ -0,0 +1,27 @@
+"""
+This package is just for overrides testing purposes.
+"""
+import socket
+from typing import Dict, Any, Callable
+
+from url_matcher import Patterns
+from url_matcher.util import get_domain
+from web_poet import handle_urls, ItemWebPage
+
+from tests.mockserver import get_ephemeral_port
+
+
+# Need to define it here since it's always changing
+DOMAIN = get_domain(socket.gethostbyname(socket.gethostname()))
+PORT = get_ephemeral_port()
+
+
+class POOverriden(ItemWebPage):
+    def to_item(self):
+        return {"msg": "PO that will be replace"}
+
+
+@handle_urls(f"{DOMAIN}:{PORT}", POOverriden)
+class POIntegration(ItemWebPage):
+    def to_item(self):
+        return {"msg": "PO replacement"}

tests/test_middleware.py

@@ -24,6 +24,7 @@
 from scrapy_poet.page_input_providers import (
     PageObjectInputProvider
 )
+from web_poet import default_registry
 from web_poet.page_inputs import ResponseData
 from scrapy_poet import DummyResponse
 from tests.utils import (HtmlResource,
@@ -350,3 +351,26 @@ def get_middleware(settings):
         mock.call('/tmp/cache', compressed=True),
         mock.call().close()
     ]
+
+
+@inlineCallbacks
+def test_web_poet_integration(settings):
+    """This tests scrapy-poet's integration with web-poet most especially when
+    populating override settings via:
+
+        from web_poet import default_registry
+
+        SCRAPY_POET_OVERRIDES = default_registry.get_overrides()
+    """
+
+    # Only import them in this test scope since they need to be synced with
+    # the URL of the Page Object annotated with @handle_urls.
+    from tests.po_lib import DOMAIN, PORT, POOverriden
+
+    # Override rules are defined in `tests/po_lib/__init__.py`.
+    settings["SCRAPY_POET_OVERRIDES"] = default_registry.get_overrides()
+
+    item, url, _ = yield crawl_single_item(
+        spider_for(POOverriden), ProductHtml, settings, port=PORT
+    )
+    assert item == {"msg": "PO replacement"}