The script

Author: agoncharov-reef

.gitignore

@@ -158,3 +158,5 @@ cython_debug/
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
+
+.envrc

README.md

@@ -1,2 +1,28 @@
 # pyreadiness-parser
 Parses https://pyreadiness.org
+
+```sh
+python parser
+```
+
+Sample output:
+```
+  rating  ready    name                url                                          languages
+--------  -------  ------------------  -------------------------------------------  --------------------------------------------------------------
+       0  +        boto3               https://pypi.org/project/boto3               Python:100.00%
+       1  +        awscli              https://pypi.org/project/awscli              Python:99.92% Shell:0.04% Batchfile:0.04%
+       2  +        botocore            https://pypi.org/project/botocore            Python:99.32% Gherkin:0.68%
+       3  +        urllib3             https://pypi.org/project/urllib3             Python:99.97% Shell:0.03%
+       4  +        requests            https://pypi.org/project/requests            Python:99.77% Makefile:0.23%
+       5  +        charset-normalizer  https://pypi.org/project/charset-normalizer  Python:99.71% Shell:0.29%
+       6  +        typing-extensions   https://pypi.org/project/typing-extensions   Python:100.00%
+       7  -        setuptools          https://pypi.org/project/setuptools          Python:98.97% C:1.02% HTML:0.01% CMake:0.01%
+       8  +        certifi             https://pypi.org/project/certifi             Python:97.78% Makefile:2.22%
+       9  -        python-dateutil     https://pypi.org/project/python-dateutil     Python:99.62% Shell:0.32% Batchfile:0.06%
+      10  +        google-api-core     https://pypi.org/project/google-api-core     Python:95.76% Shell:3.98% Dockerfile:0.27%
+      11  +        s3transfer          https://pypi.org/project/s3transfer          Python:100.00%
+      12  +        idna                https://pypi.org/project/idna                Python:100.00%
+      13  -        six                 https://pypi.org/project/six                 Python:100.00%
+      14  +        pyyaml              https://pypi.org/project/pyyaml              Python:81.44% Cython:18.05% Makefile:0.26% Shell:0.17% C:0.08%
+
+```

parser/__main__.py

@@ -0,0 +1,90 @@
+import logging
+from dataclasses import dataclass, asdict
+from itertools import count
+from typing import Iterator
+
+import requests
+from bs4 import BeautifulSoup
+from more_itertools import one
+from ratelimit import limits
+from ratelimit.exception import RateLimitException
+from tabulate import tabulate
+from tenacity import retry, retry_if_exception_type, wait_fixed
+from tqdm import tqdm
+
+session = requests.Session()
+log = logging.getLogger()
+
+
+@dataclass
+class PypiProject:
+    name: str
+    rating: int
+    is_ready: bool
+
+    @property
+    def url(self) -> str:
+        return f'https://pypi.org/project/{self.name}'
+
+    def __str__(self) -> str:
+        return self.name
+
+    def get_info(self) -> dict:
+        response = session.get(f'https://pypi.org/pypi/{self.name}/json')
+        response.raise_for_status()
+        return response.json()
+
+    @retry(
+        wait=wait_fixed(1),
+        retry=retry_if_exception_type(RateLimitException),
+    )
+    @limits(calls=1, period=4)
+    def get_language_ratio(self) -> dict[str, float]:
+
+        links = self.get_info()['info']['project_urls']
+        try:
+            github_url = one({url for url in links.values() if url.startswith('https://github.com') and url.count('/') == 4})
+        except ValueError:
+            log.warning('Cannot detect github url for %s: %s', self.name, links)
+            return {}
+
+        owner, repo = github_url.removeprefix('https://github.com/').split('/')
+        response = session.get(f'https://api.github.com/repos/{owner}/{repo}/languages')
+        response.raise_for_status()
+        languages = response.json()
+
+        total = sum(languages.values())
+        return {language: value/total for language, value in languages.items()}
+
+
+def parse_pyreadiness(version: str = '3.11') -> Iterator[PypiProject]:
+    response = session.get(f'https://pyreadiness.org/{version}')
+    response.raise_for_status()
+
+    counter = count()
+    soup = BeautifulSoup(response.text, 'html.parser')
+    lists = soup.find_all('div', class_='list')
+    for list_ in lists:
+        projects = list_.find_all('a')
+        for project in projects:
+            name, sign = project.text.strip().split(' ')
+            yield PypiProject(
+                name=name,
+                rating=next(counter),
+                is_ready=sign == '✓',
+            )
+
+
+if __name__ == '__main__':
+    print(tabulate(
+        [
+            (
+                project.rating,
+                '+' if project.is_ready else '-',
+                project.name,
+                project.url,
+                ' '.join(f'{language}:{value*100:.2f}%' for language, value in project.get_language_ratio().items()),
+            ) for project in tqdm(parse_pyreadiness())
+        ],
+        headers=('rating', 'ready', 'name', 'url', 'languages'),
+    ))

requirements.txt

@@ -0,0 +1,7 @@
+requests==2.31.0
+beautifulsoup4==4.12.2
+more-itertools==9.1.0
+tabulate==0.9.0
+tqdm==4.65.0
+ratelimit==2.2.1
+tenacity==8.2.2