Update to use HttpReponse which replaces ResponseData

Reference PR: scrapinghub/web-poet#30

Author: BurnzZ

docs/intro/advanced-tutorial.rst

@@ -54,11 +54,10 @@ Suppose we have the following Page Object:
             }
 
             # Simulates clicking on a button that says "View All Images"
-            response: web_poet.ResponseData = await self.http_client.get(
+            response: web_poet.HttpResponse = await self.http_client.get(
                 f"https://api.example.com/v2/images?id={item['product_id']}"
             )
-            page = web_poet.WebPage(response)
-            item["images"] = page.css(".product-images img::attr(src)").getall()
+            item["images"] = response.css(".product-images img::attr(src)").getall()
             return item
 
 
@@ -122,11 +121,10 @@ This basically acts as a switch to update the behavior of the Page Object:
 
             # Simulates clicking on a button that says "View All Images"
             if self.meta.get("enable_extracting_all_images")
-                response: web_poet.ResponseData = await self.http_client.get(
+                response: web_poet.HttpResponse = await self.http_client.get(
                     f"https://api.example.com/v2/images?id={item['product_id']}"
                 )
-                page = web_poet.WebPage(response)
-                item["images"] = page.css(".product-images img::attr(src)").getall()
+                item["images"] = response.css(".product-images img::attr(src)").getall()
 
             return item
 

scrapy_poet/backend.py

@@ -3,9 +3,11 @@
 import attr
 import scrapy
 from scrapy.utils.defer import deferred_to_future
-from web_poet.page_inputs import ResponseData
+from web_poet.page_inputs import HttpResponse, HttpResponseHeaders
 from web_poet.requests import Request, RequestBackendError
 
+from scrapy_poet.utils import scrapy_response_to_http_response
+
 
 logger = logging.getLogger(__name__)
 
@@ -28,13 +30,8 @@ async def scrapy_backend(request: Request):
 
         try:
             deferred = backend(request)
-            response = await deferred_to_future(deferred)
-            return ResponseData(
-                url=response.url,
-                html=response.text,
-                status=response.status,
-                headers=response.headers,
-            )
+            response: scrapy.http.Response = await deferred_to_future(deferred)
+            return scrapy_response_to_http_response(response)
 
         except scrapy.exceptions.IgnoreRequest:
             logger.warning(f"Additional Request Ignored: {request}")

scrapy_poet/middleware.py

@@ -21,7 +21,7 @@
 
 
 DEFAULT_PROVIDERS = {
-    HttpResponseProvider: 500
+    HttpResponseProvider: 500,
     HttpClientProvider: 600,
     MetaProvider: 700,
 }

scrapy_poet/page_input_providers.py

@@ -19,6 +19,7 @@
 from scrapy.utils.reqser import request_to_dict
 from scrapy.utils.request import request_fingerprint
 
+from scrapy_poet.utils import scrapy_response_to_http_response
 from scrapy_poet.injection_errors import MalformedProvidedClassesError
 from scrapy_poet.backend import create_scrapy_backend
 from web_poet import HttpResponse, HttpResponseHeaders, Meta
@@ -164,14 +165,7 @@ class HttpResponseProvider(PageObjectInputProvider, CacheDataProviderMixin):
 
     def __call__(self, to_provide: Set[Callable], response: Response):
         """Builds a ``HttpResponse`` instance using a Scrapy ``Response``"""
-        return [
-            HttpResponse(
-                url=response.url,
-                body=response.body,
-                status=response.status,
-                headers=HttpResponseHeaders.from_bytes(response.headers),
-            )
-        ]
+        return [scrapy_response_to_http_response(response)]
 
     def fingerprint(self, to_provide: Set[Callable], request: Request) -> str:
         request_keys = {"url", "method", "body"}

scrapy_poet/utils.py

@@ -1,5 +1,7 @@
 import os
 
+from web_poet import HttpResponse, HttpResponseHeaders
+from scrapy.http import Response
 from scrapy.utils.project import project_data_dir, inside_project
 from tldextract import tldextract
 
@@ -28,3 +30,15 @@ def get_scrapy_data_path(createdir: bool = True, default_dir: str = ".scrapy") -
     if createdir:
         os.makedirs(path, exist_ok=True)
     return path
+
+
+def scrapy_response_to_http_response(response: Response):
+    """Convenience method to convert a ``scrapy.http.Response`` into a
+    ``web_poet.HttpResponse``.
+    """
+    return HttpResponse(
+        url=response.url,
+        body=response.body,
+        status=response.status,
+        headers=HttpResponseHeaders.from_bytes(response.headers),
+    )

setup.py

@@ -14,7 +14,7 @@
         'andi >= 0.4.1',
         'attrs',
         'parsel',
-        'web-poet @ git+https://[email protected]/scrapinghub/web-poet@additional-requests#egg=web-poet',
+        'web-poet @ git+https://[email protected]/scrapinghub/web-poet@tmp-dep-AR-1#egg=web-poet',
         'tldextract',
         'sqlitedict',
     ],

tests/test_backend.py

@@ -38,21 +38,39 @@ async def test_incompatible_scrapy_request(scrapy_backend):
         await scrapy_backend(req)
 
 
+@pytest.fixture
+def fake_http_response():
+    return web_poet.HttpResponse(
+        "https://example.com",
+        b"some content",
+        200,
+        {"Content-Type": "text/html; charset=utf-8"},
+    )
+
+
 @pytest.mark.asyncio
-async def test_scrapy_poet_backend():
+async def test_scrapy_poet_backend(fake_http_response):
     req = web_poet.Request("https://example.com")
 
     with mock.patch(
         "scrapy_poet.backend.deferred_to_future", new_callable=AsyncMock
     ) as mock_dtf:
 
+        mock_dtf.return_value = fake_http_response
+
         mock_downloader = mock.MagicMock(return_value=AsyncMock)
         scrapy_backend = create_scrapy_backend(mock_downloader)
 
         response = await scrapy_backend(req)
 
         mock_downloader.assert_called_once()
-        assert isinstance(response, web_poet.ResponseData)
+        assert isinstance(response, web_poet.HttpResponse)
+
+        assert response.url == "https://example.com"
+        assert response.text == "some content"
+        assert response.status == 200
+        assert response.headers.get("Content-Type") == "text/html; charset=utf-8"
+        assert len(response.headers) == 1
 
 
 @pytest.mark.asyncio

tox.ini

@@ -11,7 +11,7 @@ deps =
     pytest-asyncio
     scrapy >= 2.6.0
     pytest-twisted
-    web-poet @ git+https://[email protected]/scrapinghub/web-poet@meta#egg=web-poet
+    web-poet @ git+https://[email protected]/scrapinghub/web-poet@tmp-dep-AR-1#egg=web-poet
 
 commands =
     py.test \