Skip to content

Commit 5d5932d

Browse files
committed
Update readme examples to async start
1 parent f391b7c commit 5d5932d

1 file changed

Lines changed: 22 additions & 14 deletions

File tree

README.md

Lines changed: 22 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,7 @@ import scrapy
101101
class AwesomeSpider(scrapy.Spider):
102102
name = "awesome"
103103

104-
def start_requests(self):
104+
async def start(self): # start_requests in Scrapy < 2.13
105105
# GET request
106106
yield scrapy.Request("https://httpbin.org/get", meta={"playwright": True})
107107
# POST request
@@ -116,7 +116,15 @@ class AwesomeSpider(scrapy.Spider):
116116
return {"url": response.url}
117117
```
118118

119-
### Notes about the User-Agent header
119+
### About the start/start_requests methods
120+
121+
Scrapy 2.13 introduced the `async start() -> AsyncIterator` Spider method and deprecated the
122+
traditional `start_requests` method. Support for the latter was removed in Scrapy 2.16.
123+
For simplicity, the rest of the examples in this Readme will use `start`, replace
124+
`async def start` with `def start_requests` if you are using an older Scrapy version.
125+
126+
127+
### About the User-Agent header
120128

121129
By default, outgoing requests include the `User-Agent` set by Scrapy (either with the
122130
`USER_AGENT` or `DEFAULT_REQUEST_HEADERS` settings or via the `Request.headers` attribute).
@@ -482,7 +490,7 @@ async def init_page(page, request):
482490
await page.add_init_script(path="./custom_script.js")
483491

484492
class AwesomeSpider(scrapy.Spider):
485-
def start_requests(self):
493+
async def start(self):
486494
yield scrapy.Request(
487495
url="https://httpbin.org/headers",
488496
meta={
@@ -515,7 +523,7 @@ requests using the same page. For instance:
515523
```python
516524
from playwright.async_api import Page
517525

518-
def start_requests(self):
526+
async def start(self):
519527
yield scrapy.Request(
520528
url="https://httpbin.org/get",
521529
meta={"playwright": True, "playwright_include_page": True},
@@ -601,7 +609,7 @@ import scrapy
601609
class AwesomeSpiderWithPage(scrapy.Spider):
602610
name = "page_spider"
603611

604-
def start_requests(self):
612+
async def start(self):
605613
yield scrapy.Request(
606614
url="https://example.org",
607615
callback=self.parse_first,
@@ -803,7 +811,7 @@ class ProxySpider(Spider):
803811
}
804812
}
805813

806-
def start_requests(self):
814+
async def start(self):
807815
yield Request("http://httpbin.org/get", meta={"playwright": True})
808816

809817
def parse(self, response, **kwargs):
@@ -866,7 +874,7 @@ will be stored in the `PageMethod.result` attribute.
866874

867875
For instance:
868876
```python
869-
def start_requests(self):
877+
async def start(self):
870878
yield Request(
871879
url="https://example.org",
872880
meta={
@@ -884,7 +892,7 @@ def parse(self, response, **kwargs):
884892

885893
produces the same effect as:
886894
```python
887-
def start_requests(self):
895+
async def start(self):
888896
yield Request(
889897
url="https://example.org",
890898
meta={"playwright": True, "playwright_include_page": True},
@@ -914,7 +922,7 @@ async def scroll_page(page: Page) -> str:
914922
class MySpyder(scrapy.Spider):
915923
name = "scroll"
916924

917-
def start_requests(self):
925+
async def start(self):
918926
yield Request(
919927
url="https://quotes.toscrape.com/scroll",
920928
meta={
@@ -956,7 +964,7 @@ async def handle_dialog(dialog: Dialog) -> None:
956964
class EventSpider(scrapy.Spider):
957965
name = "event"
958966

959-
def start_requests(self):
967+
async def start(self):
960968
yield scrapy.Request(
961969
url="https://example.org",
962970
meta={
@@ -1028,7 +1036,7 @@ module is not available.
10281036
class ClickAndSavePdfSpider(scrapy.Spider):
10291037
name = "pdf"
10301038

1031-
def start_requests(self):
1039+
async def start(self):
10321040
yield scrapy.Request(
10331041
url="https://example.org",
10341042
meta=dict(
@@ -1053,7 +1061,7 @@ class ClickAndSavePdfSpider(scrapy.Spider):
10531061
class ScrollSpider(scrapy.Spider):
10541062
name = "scroll"
10551063

1056-
def start_requests(self):
1064+
async def start(self):
10571065
yield scrapy.Request(
10581066
url="http://quotes.toscrape.com/scroll",
10591067
meta=dict(
@@ -1104,7 +1112,7 @@ import scrapy
11041112
class ExampleSpider(scrapy.Spider):
11051113
name = "example"
11061114

1107-
def start_requests(self):
1115+
async def start(self):
11081116
yield scrapy.Request(
11091117
url="https://example.org",
11101118
meta=dict(
@@ -1186,7 +1194,7 @@ class ExampleSpider(scrapy.Spider):
11861194
},
11871195
}
11881196

1189-
def start_requests(self):
1197+
async def start(self):
11901198
yield scrapy.Request(
11911199
url="https://example.org",
11921200
meta={"playwright": True},

0 commit comments

Comments
 (0)