Actually start scraping something

Author: MarkusShepherd

.gitignore

@@ -81,6 +81,7 @@ venv.bak/
 # Project specific
 .scrapy/
 feeds/
+feeds_v3/
 images/
 jobs/
 logs/

src/board_game_scraper/settings.py

@@ -12,20 +12,24 @@
 
 BASE_DIR = Path(__file__).resolve().parent.parent.parent
 
+GAME_ITEM_URI = f"{BASE_DIR}/feeds_v3/%(name)s/GameItem/%(time)s.jl"
+USER_ITEM_URI = f"{BASE_DIR}/feeds_v3/%(name)s/UserItem/%(time)s.jl"
+COLLECTION_ITEM_URI = f"{BASE_DIR}/feeds_v3/%(name)s/CollectionItem/%(time)s.jl"
+
 FEEDS = {
-    BASE_DIR / "feeds_v3" / "GameItem" / "%(time)s.jl": {
+    GAME_ITEM_URI: {
         "item_classes": ["board_game_scraper.items.GameItem"],
         "format": "jsonlines",
         "overwrite": False,
         "store_empty": False,
     },
-    BASE_DIR / "feeds_v3" / "UserItem" / "%(time)s.jl": {
+    USER_ITEM_URI: {
         "item_classes": ["board_game_scraper.items.UserItem"],
         "format": "jsonlines",
         "overwrite": False,
         "store_empty": False,
     },
-    BASE_DIR / "feeds_v3" / "CollectionItem" / "%(time)s.jl": {
+    COLLECTION_ITEM_URI: {
         "item_classes": ["board_game_scraper.items.CollectionItem"],
         "format": "jsonlines",
         "overwrite": False,

src/board_game_scraper/spiders/bgg.py

@@ -1,10 +1,18 @@
+from __future__ import annotations
+
 import re
-from collections.abc import Generator, Iterable
-from typing import Any
+from datetime import datetime, timezone
+from typing import TYPE_CHECKING, Any
 
-from scrapy.http import Response
 from scrapy.spiders import SitemapSpider
 
+from board_game_scraper.items import CollectionItem, GameItem
+
+if TYPE_CHECKING:
+    from collections.abc import Generator, Iterable
+
+    from scrapy.http import Response
+
 
 class BggSpider(SitemapSpider):
     name = "bgg"
@@ -44,5 +52,31 @@ def sitemap_filter(
             )
             yield entry
 
-    def parse(self, response: Response) -> None:
-        pass  # TODO: Parse XML response
+    def parse(
+        self,
+        response: Response,
+    ) -> Generator[GameItem | CollectionItem, None, None]:
+        for game in response.xpath("/items/item"):
+            bgg_item_type = game.xpath("@type").get()
+            if bgg_item_type != "boardgame":
+                self.logger.info("Skipping item type <%s>", bgg_item_type)
+                continue
+
+            yield GameItem(
+                name=game.xpath("name[@type='primary']/@value").get(),
+                bgg_id=game.xpath("@id").get(),
+                year=game.xpath("yearpublished/@value").get(),
+                description=game.xpath("description/text()").get(),
+                image_url=game.xpath("image/text()").getall(),  # TODO: <thumbnail>
+                scraped_at=datetime.now(timezone.utc),
+            )
+
+            for comment in game.xpath("comments/comment"):
+                yield CollectionItem(
+                    item_id=f"{comment.xpath("@username").get()}:{game.xpath("@id").get()}",
+                    bgg_id=game.xpath("@id").get(),
+                    bgg_user_name=comment.xpath("@username").get(),
+                    bgg_user_rating=comment.xpath("@rating").get(),
+                    comment=comment.xpath("@value").get(),
+                    scraped_at=datetime.now(timezone.utc),
+                )