integrate with scrapy-redis havanagrawal#2

paulbroek · Jan 5, 2023 · e9243cc · e9243cc
1 parent 8d4028a
commit e9243cc
Showing 1 changed file with 4 additions and 3 deletions.
diff --git a/GoodreadsScraper/settings.py b/GoodreadsScraper/settings.py
@@ -23,11 +23,8 @@
 # Obey robots.txt rules
 ROBOTSTXT_OBEY = True
 
-# POSTGRES_HOST = "77.249.149.174"
-# POSTGRES_HOST = "postgres-goodreads-master"
 POSTGRES_HOST = "127.0.0.1"
 # POSTGRES_PORT = 5432
-# POSTGRES_PORT = 5439
 
 # scrapy-redis settings
 USER_AGENT = 'scrapy-redis (+https://github.com/rolando/scrapy-redis)'
@@ -38,6 +35,10 @@
 # SCHEDULER_QUEUE_CLASS = "scrapy_redis.queue.SpiderPriorityQueue"
 # SCHEDULER_QUEUE_CLASS = "scrapy_redis.queue.SpiderQueue"
 # SCHEDULER_QUEUE_CLASS = "scrapy_redis.queue.SpiderStack"
+REDIS_PARAMS = {
+    "REDIS_HOST": "81.169.252.73",
+    "REDIS_PORT" : 6382
+}
 
 # update postgres when author item is scraped. Caution: very slow if query is
 # not optimized