Initial commit

Author: EpicWink

.gitignore

@@ -0,0 +1,4 @@
+venv/
+__pycache__/
+.idea/
+*.egg-info/

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 Laurie Opperman
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,24 @@
+# PyPI mirror
+Local PyPI mirror cache
+
+## Installation
+```bash
+pip install pypi-mirror
+```
+
+## Usage
+```bash
+FLASK_APP=pypi_mirror flask run
+```
+
+```bash
+pip install --index-url http://127.0.0.1:5000/index/ simplejson
+```
+
+### Environment variables
+* `PIP_INDEX_URL`: root index URL, default: https://pypi.org/simple/
+* `INDEX_TTL`: root index time-to-live (aka cache time-out) in seconds, default: 30
+   minutes
+* `PIP_EXTRA_INDEX_URL`: extra index URLs (white-space separated)
+* `EXTRA_INDEX_TTL`: corresponding extra index times-to-live in seconds (white-space
+   separated), default: 5 minutes

setup.cfg

@@ -0,0 +1,32 @@
+[metadata]
+name = pypi-mirror
+version = 0.1.0a0.dev0
+url = https://github.com/EpicWink/pypi-mirror
+author = Laurie O
+author_email = [email protected]
+license = MIT
+description = Local PyPI mirror cache
+long_description = file: README.md
+long_description_content_type = text/markdown
+keywords = pypi, index, mirror, cache
+classifiers =
+    Environment :: Console
+    Intended Audience :: Developers
+    Programming Language :: Python :: 3 :: Only
+    Natural Language :: English
+    Operating System :: POSIX :: Linux
+    Operating System :: Microsoft :: Windows
+
+[options]
+install_requires =
+    flask
+    requests
+packages = find:
+package_dir =
+    =src
+
+[options.packages.find]
+package_dir = src
+
+[options.package_data]
+pypi_mirror = templates/*.html

setup.py

@@ -0,0 +1,5 @@
+"""Install script for ``pypi-mirror``."""
+
+import setuptools
+
+setuptools.setup()

src/pypi_mirror/__init__.py

@@ -0,0 +1,4 @@
+"""Local PyPI mirror cache."""
+
+from . import routes
+from .config import app

src/pypi_mirror/config.py

@@ -0,0 +1,16 @@
+"""Local PyPI mirror cache."""
+
+import os
+
+import flask
+import jinja2
+
+INDEX_URL = os.environ.get("PIP_INDEX_URL", "https://pypi.org/simple/")
+EXTRA_INDEX_URL = os.environ.get("PIP_EXTRA_INDEX_URL", "")
+INDEX_TTL = os.environ.get("INDEX_TTL", 1800)
+EXTRA_INDEX_TTL = os.environ.get(
+    "EXTRA_INDEX_TTL", " ".join("180" for s in EXTRA_INDEX_URL.split() if s)
+)
+
+app = flask.Flask("pypi_mirror")
+app.jinja_loader = jinja2.PackageLoader("pypi_mirror")

src/pypi_mirror/data.py

@@ -0,0 +1,188 @@
+"""Local PyPI mirror cache."""
+
+import os
+import re
+import time
+import shutil
+import tempfile
+import threading
+import collections
+import typing as t
+from urllib import parse as urllib_parse
+
+import bs4
+import requests
+
+from . import config
+
+_sha_fragment_re = re.compile("[#&]sha256=([^&]*)")
+File = collections.namedtuple("File", ("name", "url", "sha"))
+
+
+class NotFound(ValueError):
+    pass
+
+
+class _IndexCache:
+    def __init__(self, index_url: str, ttl: int):
+        self.index_url = index_url
+        self.ttl = ttl
+        self._package_dir = tempfile.mkdtemp()
+        self._index_t = None
+        self._packages_t = {}
+        self._files_t = {}
+        self._index = {}
+        self._packages = {}
+        self._files = {}
+
+    def __del__(self):
+        if os.path.isdir(self._package_dir):
+            shutil.rmtree(self._package_dir)
+
+    def _list_packages(self):
+        if self._index_t is not None and (time.monotonic() - self._index_t) < self.ttl:
+            return
+
+        response = requests.get(self.index_url)
+        self._index_t = time.monotonic()
+
+        soup = bs4.BeautifulSoup(response.text)
+        for link in soup.find_all("a"):
+            self._index[link.string] = link["href"]
+
+    def list_packages(self) -> t.Iterable[str]:
+        self._list_packages()
+        return tuple(self._index)
+
+    def _list_files(self, package_name: str):
+        packages_t = self._packages_t.get(package_name)
+        if packages_t is not None and (time.monotonic() - packages_t) < self.ttl:
+            return
+
+        self._list_packages()
+        if package_name not in self._index:
+            raise NotFound(package_name)
+
+        package_url = self._index[package_name]
+        url = urllib_parse.urljoin(self.index_url, package_url)
+        response = requests.get(url)
+        self._packages_t[package_name] = time.monotonic()
+
+        soup = bs4.BeautifulSoup(response.text)
+        self._packages.setdefault(package_name, {})
+        for link in soup.find_all("a"):
+            name = link.string
+            url = link["href"]
+            match = _sha_fragment_re.search(url)
+            sha = match.group(1) if match else None
+            self._packages[package_name][name] = File(name, url, sha)
+
+    def list_files(self, package_name: str) -> t.Iterable[File]:
+        self._list_files(package_name)
+        return tuple(self._packages[package_name].values())
+
+    @staticmethod
+    def _download_file(
+        url: str,
+        path: str,
+        get_callback: t.Callable[[], t.Any],
+        done_callback: t.Callable[[], t.Any],
+    ):
+        response = requests.get(url, stream=True)
+        get_callback()
+        with open(path, "wb") as f:
+            for chunk in response.iter_content(None):
+                f.write(chunk)
+        done_callback()
+
+    def _get_file(self, package_name: str, file_name: str):
+        files_t = self._files_t.get(package_name, {}).get(file_name)
+        if files_t is not None and (time.monotonic() - files_t) < self.ttl:
+            return
+
+        self._list_files(package_name)
+        if file_name not in self._packages[package_name]:
+            raise NotFound(file_name)
+
+        path = os.path.join(self._package_dir, package_name + "_" + file_name)
+        url = self._packages[package_name][file_name].url
+
+        def get_callback():
+            self._files_t.setdefault(package_name, {})[file_name] = time.monotonic()
+
+        def done_callback():
+            package_files[file_name] = path
+
+        package_files = self._files.setdefault(package_name, {})
+        if isinstance(package_files.get(file_name), threading.Thread):
+            package_files[file_name].join(0.9)
+            time.sleep(0.01)  # give control to original master
+            return
+
+        thread = threading.Thread(
+            target=self._download_file, args=(url, path, get_callback, done_callback)
+        )
+        package_files[file_name] = thread
+        thread.start()
+        thread.join(0.9)
+        if thread.is_alive():
+            self._files[package_name][file_name] = url
+
+    def get_file(self, package_name: str, file_name: str) -> str:
+        self._get_file(package_name, file_name)
+        return self._files[package_name][file_name]
+
+
+class Cache:
+    _index_cache_cls = _IndexCache
+
+    def __init__(
+        self, root_cache: _IndexCache, extra_caches: t.List[_IndexCache] = None
+    ):
+        self.root_cache = root_cache
+        self.extra_caches = extra_caches or []
+        self._packages = {}
+        self._list_dt = None
+        self._package_list_dt = {}
+
+    @classmethod
+    def from_config(cls):
+        root_cache = cls._index_cache_cls(config.INDEX_URL, int(config.INDEX_TTL))
+        extra_index_urls = [s for s in config.EXTRA_INDEX_URL.split() if s]
+        extra_ttls = [int(s) for s in config.EXTRA_INDEX_TTL.split() if s]
+        assert len(extra_index_urls) == len(extra_ttls)
+        extra_caches = [
+            cls._index_cache_cls(url, ttl)
+            for url, ttl in zip(extra_index_urls, extra_ttls)
+        ]
+        return cls(root_cache, extra_caches=extra_caches)
+
+    def list_packages(self) -> t.Iterable[str]:
+        packages = set(self.root_cache.list_packages())
+        for cache in self.extra_caches:
+            packages.update(cache.list_packages())
+        return sorted(packages)
+
+    def list_files(self, package_name: str) -> t.Iterable[File]:
+        try:
+            return self.root_cache.list_files(package_name)
+        except NotFound as e:
+            exc = e
+        for cache in self.extra_caches:
+            try:
+                return cache.list_files(package_name)
+            except NotFound:
+                pass
+        raise exc
+
+    def get_file(self, package_name: str, file_name: str) -> str:
+        try:
+            return self.root_cache.get_file(package_name, file_name)
+        except NotFound as e:
+            exc = e
+        for cache in self.extra_caches:
+            try:
+                return cache.get_file(package_name, file_name)
+            except NotFound:
+                pass
+        raise exc

src/pypi_mirror/routes.py

@@ -0,0 +1,38 @@
+"""Local PyPI mirror cache."""
+
+from urllib import parse as urllib_parse
+
+import flask
+import collections
+
+from . import config
+from . import data
+
+cache = data.Cache.from_config()
+cache.list_packages()
+Item = collections.namedtuple("Item", ("name", "url"))
+
+
+@config.app.route("/index/")
+def list_packages():
+    package_names = cache.list_packages()
+    packages = [Item(n, f"/index/{n}/") for n in package_names]
+    return flask.render_template("packages.html", packages=packages)
+
+
+@config.app.route("/index/<package_name>/")
+def list_files(package_name: str):
+    files = cache.list_files(package_name)
+    files = [
+        Item(f.name, f"/index/{package_name}/{f.name}#sha256={f.sha}") for f in files
+    ]
+    return flask.render_template("files.html", package_name=package_name, files=files)
+
+
+@config.app.route("/index/<package_name>/<file_name>")
+def get_file(package_name: str, file_name: str):
+    path = cache.get_file(package_name, file_name)
+    scheme = urllib_parse.urlparse(path).scheme
+    if scheme and scheme != "file":
+        return flask.redirect(path)
+    return flask.send_file(path)

src/pypi_mirror/templates/files.html

@@ -0,0 +1,12 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <title>{{ package_name }}</title>
+</head>
+<body>
+    {%- for file in files %}
+    <a href="{{ file.url }}">{{ file.name }}</a><br>
+    {%- endfor %}
+</body>
+</html>

src/pypi_mirror/templates/packages.html

@@ -0,0 +1,12 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <title>Index</title>
+</head>
+<body>
+    {%- for package in packages %}
+    <a href="{{ package.url }}">{{ package.name }}</a><br>
+    {%- endfor %}
+</body>
+</html>