@@ -101,7 +101,7 @@ import scrapy
101101class AwesomeSpider (scrapy .Spider ):
102102 name = " awesome"
103103
104- def start_requests (self ):
104+ async def start (self ): # start_requests in Scrapy < 2.13
105105 # GET request
106106 yield scrapy.Request(" https://httpbin.org/get" , meta = {" playwright" : True })
107107 # POST request
@@ -116,7 +116,15 @@ class AwesomeSpider(scrapy.Spider):
116116 return {" url" : response.url}
117117```
118118
119- ### Notes about the User-Agent header
119+ ### About the start/start_requests methods
120+
121+ Scrapy 2.13 introduced the ` async start() -> AsyncIterator ` Spider method and deprecated the
122+ traditional ` start_requests ` method. Support for the latter was removed in Scrapy 2.16.
123+ For simplicity, the rest of the examples in this Readme will use ` start ` , replace
124+ ` async def start ` with ` def start_requests ` if you are using an older Scrapy version.
125+
126+
127+ ### About the User-Agent header
120128
121129By default, outgoing requests include the ` User-Agent ` set by Scrapy (either with the
122130` USER_AGENT ` or ` DEFAULT_REQUEST_HEADERS ` settings or via the ` Request.headers ` attribute).
@@ -482,7 +490,7 @@ async def init_page(page, request):
482490 await page.add_init_script(path = " ./custom_script.js" )
483491
484492class AwesomeSpider (scrapy .Spider ):
485- def start_requests (self ):
493+ async def start (self ):
486494 yield scrapy.Request(
487495 url = " https://httpbin.org/headers" ,
488496 meta = {
@@ -515,7 +523,7 @@ requests using the same page. For instance:
515523``` python
516524from playwright.async_api import Page
517525
518- def start_requests (self ):
526+ async def start (self ):
519527 yield scrapy.Request(
520528 url = " https://httpbin.org/get" ,
521529 meta = {" playwright" : True , " playwright_include_page" : True },
@@ -601,7 +609,7 @@ import scrapy
601609class AwesomeSpiderWithPage (scrapy .Spider ):
602610 name = " page_spider"
603611
604- def start_requests (self ):
612+ async def start (self ):
605613 yield scrapy.Request(
606614 url = " https://example.org" ,
607615 callback = self .parse_first,
@@ -803,7 +811,7 @@ class ProxySpider(Spider):
803811 }
804812 }
805813
806- def start_requests (self ):
814+ async def start (self ):
807815 yield Request(" http://httpbin.org/get" , meta = {" playwright" : True })
808816
809817 def parse (self , response , ** kwargs ):
@@ -866,7 +874,7 @@ will be stored in the `PageMethod.result` attribute.
866874
867875For instance:
868876``` python
869- def start_requests (self ):
877+ async def start (self ):
870878 yield Request(
871879 url = " https://example.org" ,
872880 meta = {
@@ -884,7 +892,7 @@ def parse(self, response, **kwargs):
884892
885893produces the same effect as:
886894``` python
887- def start_requests (self ):
895+ async def start (self ):
888896 yield Request(
889897 url = " https://example.org" ,
890898 meta = {" playwright" : True , " playwright_include_page" : True },
@@ -914,7 +922,7 @@ async def scroll_page(page: Page) -> str:
914922class MySpyder (scrapy .Spider ):
915923 name = " scroll"
916924
917- def start_requests (self ):
925+ async def start (self ):
918926 yield Request(
919927 url = " https://quotes.toscrape.com/scroll" ,
920928 meta = {
@@ -956,7 +964,7 @@ async def handle_dialog(dialog: Dialog) -> None:
956964class EventSpider (scrapy .Spider ):
957965 name = " event"
958966
959- def start_requests (self ):
967+ async def start (self ):
960968 yield scrapy.Request(
961969 url = " https://example.org" ,
962970 meta = {
@@ -1028,7 +1036,7 @@ module is not available.
10281036class ClickAndSavePdfSpider (scrapy .Spider ):
10291037 name = " pdf"
10301038
1031- def start_requests (self ):
1039+ async def start (self ):
10321040 yield scrapy.Request(
10331041 url = " https://example.org" ,
10341042 meta = dict (
@@ -1053,7 +1061,7 @@ class ClickAndSavePdfSpider(scrapy.Spider):
10531061class ScrollSpider (scrapy .Spider ):
10541062 name = " scroll"
10551063
1056- def start_requests (self ):
1064+ async def start (self ):
10571065 yield scrapy.Request(
10581066 url = " http://quotes.toscrape.com/scroll" ,
10591067 meta = dict (
@@ -1104,7 +1112,7 @@ import scrapy
11041112class ExampleSpider (scrapy .Spider ):
11051113 name = " example"
11061114
1107- def start_requests (self ):
1115+ async def start (self ):
11081116 yield scrapy.Request(
11091117 url = " https://example.org" ,
11101118 meta = dict (
@@ -1186,7 +1194,7 @@ class ExampleSpider(scrapy.Spider):
11861194 },
11871195 }
11881196
1189- def start_requests (self ):
1197+ async def start (self ):
11901198 yield scrapy.Request(
11911199 url = " https://example.org" ,
11921200 meta = {" playwright" : True },
0 commit comments