Skip to content

Commit

Permalink
Update packaging + CI, switch aioredis -> redis
Browse files Browse the repository at this point in the history
  • Loading branch information
sciyoshi committed Jun 4, 2024
1 parent ae23bd6 commit d2c710f
Show file tree
Hide file tree
Showing 11 changed files with 175 additions and 289 deletions.
69 changes: 0 additions & 69 deletions .github/workflows/build_deploy.yml

This file was deleted.

72 changes: 23 additions & 49 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
@@ -1,63 +1,37 @@
name: CI
on: [push, pull_request]

on:
push:
pull_request:

jobs:
black:
lint:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "3.9"
- run: pip install riot==0.19.0
- run: riot -v run -s black -- --check .
mypy:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "3.9"
- run: pip install riot==0.19.0
- run: riot -v run mypy
flake8:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "3.9"
- run: pip install riot==0.19.0
- run: riot -v run flake8
- run: pip install uv
- run: uv venv
- run: uv pip install --requirement pyproject.toml --all-extras
- run: .venv/bin/ruff format --check .
- run: .venv/bin/ruff check .
- run: .venv/bin/mypy asyncio_connection_pool

test:
runs-on: ubuntu-latest
strategy:
matrix:
os: [ubuntu-latest, macos-latest]
python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v4
- name: Setup Python
uses: actions/setup-python@v5
- uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
- name: install riot
run: pip install riot==0.19.0
- name: run tests
run: riot -v run --python=${{ matrix.python-version }} test -- --cov=asyncio_connection_pool --cov-branch --cov-config=.coveragerc
- name: install coverage
run: pip install coverage
- name: upload coverage
uses: coverallsapp/github-action@v2
with:
parallel: true
flag-name: run ${{ join(matrix.*, ' - ') }}
finish-coveralls:
needs: test
if: ${{ always() }}
runs-on: ubuntu-latest
steps:
- name: Coveralls Finished
uses: coverallsapp/github-action@v2
with:
parallel-finished: true
allow-empty: true
- run: pip install uv
- run: uv venv
- run: uv pip install --requirement pyproject.toml --all-extras
- run: .venv/bin/pytest --cov=asyncio_connection_pool --cov-branch --cov-config=.coveragerc
- run: .venv/bin/codecov
if: matrix.python-version == '3.11'
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
25 changes: 25 additions & 0 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
name: Release

on:
release:
types:
- created

jobs:
build:
runs-on: ubuntu-latest
permissions:
# IMPORTANT: this permission is mandatory for trusted publishing
id-token: write
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "3.12"
- run: pip install uv
- run: uv venv
- run: uv pip install --requirement pyproject.toml
- run: uv pip install setuptools setuptools-scm wheel build
- run: .venv/bin/python -m build --no-isolation
- name: Publish package distributions to PyPI
uses: pypa/gh-action-pypi-publish@release/v1
20 changes: 9 additions & 11 deletions asyncio_connection_pool/__init__.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from abc import ABC, abstractmethod
import asyncio
import inspect
from abc import ABC, abstractmethod
from contextlib import asynccontextmanager
from typing import AsyncIterator, Awaitable, Generic, Optional, TypeVar

Expand All @@ -10,16 +10,13 @@

class ConnectionStrategy(ABC, Generic[Conn]):
@abstractmethod
async def make_connection(self) -> Conn:
...
async def make_connection(self) -> Conn: ...

@abstractmethod
def connection_is_closed(self, conn: Conn) -> bool:
...
def connection_is_closed(self, conn: Conn) -> bool: ...

@abstractmethod
async def close_connection(self, conn: Conn) -> None:
...
async def close_connection(self, conn: Conn) -> None: ...


async def _close_connection_compat(
Expand Down Expand Up @@ -60,18 +57,19 @@ def __init__(
*,
strategy: ConnectionStrategy[Conn],
max_size: int,
burst_limit: Optional[int] = None
burst_limit: Optional[int] = None,
) -> None:
self._loop = asyncio.get_event_loop()
self.strategy = strategy
self.max_size = max_size
self.burst_limit = burst_limit
if burst_limit is not None and burst_limit < max_size:
raise ValueError("burst_limit must be greater than or equal to max_size")
msg = "burst_limit must be greater than or equal to max_size"
raise ValueError(msg)
self.in_use = 0
self.currently_allocating = 0
self.currently_deallocating = 0
self.available: "asyncio.Queue[Conn]" = asyncio.Queue(maxsize=self.max_size)
self.available: asyncio.Queue[Conn] = asyncio.Queue(maxsize=self.max_size)

@property
def _total(self) -> int:
Expand Down Expand Up @@ -110,7 +108,7 @@ def _get_conn(self) -> Awaitable[Conn]:
# Incidentally, awaiting a done Future doesn't involve yielding to
# the event loop; it's more like getting the next value from a
# generator.
fut: "asyncio.Future[Conn]" = self._loop.create_future()
fut: asyncio.Future[Conn] = self._loop.create_future()
fut.set_result(self.available.get_nowait())
self.in_use += 1
return fut
Expand Down
12 changes: 7 additions & 5 deletions asyncio_connection_pool/contrib/aioredis.py
Original file line number Diff line number Diff line change
@@ -1,16 +1,18 @@
import aioredis
from functools import partial

from redis import asyncio as aioredis

from asyncio_connection_pool import ConnectionStrategy

__all__ = ("RedisConnectionStrategy",)


class RedisConnectionStrategy(ConnectionStrategy[aioredis.Redis]): # type: ignore
class RedisConnectionStrategy(ConnectionStrategy[aioredis.Redis]):
def __init__(self, *args, **kwargs):
self._create_redis = partial(aioredis.create_redis, *args, **kwargs)
self._create_redis = partial(aioredis.Redis, *args, **kwargs)

async def make_connection(self):
return await self._create_redis()
async def make_connection(self) -> aioredis.Redis:
return self._create_redis()

def connection_is_closed(self, conn):
return conn.closed
Expand Down
52 changes: 24 additions & 28 deletions asyncio_connection_pool/contrib/datadog.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
from contextlib import asynccontextmanager, AsyncExitStack
from datadog import statsd
from contextlib import AsyncExitStack, asynccontextmanager
from typing import AsyncIterator, Awaitable, TypeVar

from datadog.dogstatsd.base import statsd
from ddtrace import tracer
from typing import AsyncIterator, TypeVar

from asyncio_connection_pool import ConnectionPool as _ConnectionPool

__all__ = ("ConnectionPool",)
Expand All @@ -18,13 +20,13 @@ def __init__(self, service_name, *args, extra_tags=None, **kwargs):
self._extra_tags = extra_tags or []
self._loop.call_soon(self._periodically_send_metrics)

def _periodically_send_metrics(self):
def _periodically_send_metrics(self) -> None:
try:
self._record_pressure()
finally:
self._loop.call_later(60, self._periodically_send_metrics)

def _record_pressure(self):
def _record_pressure(self) -> None:
statsd.gauge(
f"{self._service_name}.pool.total_connections",
self._total,
Expand Down Expand Up @@ -74,7 +76,7 @@ def _record_pressure(self):
tags=self._extra_tags,
)

def _record_connection_acquiring(self, value=0):
def _record_connection_acquiring(self, value: int = 0) -> None:
self._connections_acquiring += value

statsd.gauge(
Expand All @@ -83,41 +85,35 @@ def _record_connection_acquiring(self, value=0):
tags=self._extra_tags,
)

def _connection_maker(self):
async def _connection_maker(self) -> Conn:
statsd.increment(
f"{self._service_name}.pool.getting_connection",
tags=self._extra_tags + ["method:new"],
tags=[*self._extra_tags, "method:new"],
)

async def connection_maker(self):
with tracer.trace(
f"{self._service_name}.pool._create_new_connection",
service=self._service_name,
):
return await super()._connection_maker()

return connection_maker(self)
with tracer.trace(
f"{self._service_name}.pool._create_new_connection",
service=self._service_name,
):
return await super()._connection_maker()

def _connection_waiter(self):
async def _connection_waiter(self) -> Conn:
statsd.increment(
f"{self._service_name}.pool.getting_connection",
tags=self._extra_tags + ["method:wait"],
tags=[*self._extra_tags, "method:wait"],
)

async def connection_waiter(self):
with tracer.trace(
f"{self._service_name}.pool._wait_for_connection",
service=self._service_name,
):
return await super()._connection_waiter()

return connection_waiter(self)
with tracer.trace(
f"{self._service_name}.pool._wait_for_connection",
service=self._service_name,
):
return await super()._connection_waiter()

def _get_conn(self):
def _get_conn(self) -> Awaitable[Conn]:
if not self.available.empty():
statsd.increment(
f"{self._service_name}.pool.getting_connection",
tags=self._extra_tags + ["method:available"],
tags=[*self._extra_tags, "method:available"],
)
return super()._get_conn()

Expand Down
18 changes: 0 additions & 18 deletions mypy.ini

This file was deleted.

Loading

0 comments on commit d2c710f

Please sign in to comment.