feat: moved tiny_web_crawller to datacrawl

fixed tests to pass without network connection

Author: indrajithi

README.md

@@ -24,14 +24,14 @@ A simple and efficient web crawler for Python.
 Install using pip:
 
 ```sh
-pip install tiny-web-crawler
+pip install datacrawl
 ```
 
 ## Usage
 
 ```python
-from tiny_web_crawler import Spider
-from tiny_web_crawler import SpiderSettings
+from datacrawl import Spider
+from datacrawl import SpiderSettings
 
 settings = SpiderSettings(
     root_url = 'http://github.com',

pyproject.toml

@@ -1,15 +1,15 @@
 [tool.poetry]
-name = "tiny-web-crawler"
-version = "0.5.0"
+name = "datacrawl"
+version = "0.6.0"
 description = "A simple and efficient web crawler in Python."
 authors = ["Indrajith Indraprastham <indr4jith@gmail.com>"]
 license = "GPL-3.0-or-later"
 readme = "README.md"
-homepage = "http://github.com/indrajithi/tiny-web-crawler"
-repository = "http://github.com/indrajithi/tiny-web-crawler"
-documentation = "http://github.com/indrajithi/tiny-web-crawler"
+homepage = "http://github.com/DataCrawl-AI/datacrawl"
+repository = "http://github.com/DataCrawl-AI/datacrawl"
+documentation = "http://github.com/DataCrawl-AI/datacrawl"
 packages = [
-    { include = "tiny_web_crawler", from = "src" }
+    { include = "datacrawl", from = "src" }
 ]
 
 [tool.poetry.scripts]

src/datacrawl/__init__.py

@@ -0,0 +1,4 @@
+from datacrawl.core.spider import Spider
+from datacrawl.core.spider_settings import SpiderSettings
+
+__all__ = ["Spider", "SpiderSettings"]

src/tiny_web_crawler/core/__init__.py src/datacrawl/core/__init__.pysrc/tiny_web_crawler/core/__init__.py renamed to src/datacrawl/core/__init__.py

@@ -12,16 +12,16 @@
 
 import requests
 
-from tiny_web_crawler.core.spider_settings import SpiderSettings
-from tiny_web_crawler.logger import get_logger, set_logging_level
-from tiny_web_crawler.networking.fetcher import fetch_url
-from tiny_web_crawler.networking.formatter import format_url
-from tiny_web_crawler.networking.robots_txt import (
+from datacrawl.core.spider_settings import SpiderSettings
+from datacrawl.logger import get_logger, set_logging_level
+from datacrawl.networking.fetcher import fetch_url
+from datacrawl.networking.formatter import format_url
+from datacrawl.networking.robots_txt import (
     get_robots_txt_url,
     is_robots_txt_allowed,
     setup_robots_txt_parser,
 )
-from tiny_web_crawler.networking.validator import is_valid_url
+from datacrawl.networking.validator import is_valid_url
 
 DEFAULT_SCHEME: str = "http://"
 logger = get_logger()

src/tiny_web_crawler/core/spider.py src/datacrawl/core/spider.pysrc/tiny_web_crawler/core/spider.py renamed to src/datacrawl/core/spider.py

@@ -6,7 +6,7 @@
 import requests
 from bs4 import BeautifulSoup
 
-from tiny_web_crawler.logger import get_logger
+from datacrawl.logger import get_logger
 
 TRANSIENT_ERRORS = [408, 502, 503, 504]