support python3.9 and pydantic 1.7 (#214)

* support python3.9 and pydantic 1.7

* linting

* uprev mypy

Author: samuelcolvin

.github/workflows/ci.yml

@@ -15,7 +15,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu]
-        python-version: ['3.6', '3.7', '3.8']
+        python-version: ['3.6', '3.7', '3.8', '3.9']
 
     env:
       PYTHON: ${{ matrix.python-version }}

arq/connections.py

@@ -1,15 +1,17 @@
 import asyncio
 import functools
 import logging
+import ssl
 from dataclasses import dataclass
 from datetime import datetime, timedelta
 from operator import attrgetter
-from ssl import SSLContext
-from typing import Any, Callable, List, Optional, Tuple, Union
+from typing import Any, Callable, Generator, List, Optional, Tuple, Union
+from urllib.parse import urlparse
 from uuid import uuid4
 
 import aioredis
 from aioredis import MultiExecError, Redis
+from pydantic.validators import make_arbitrary_type_validator
 
 from .constants import default_queue_name, job_key_prefix, result_key_prefix
 from .jobs import Deserializer, Job, JobDef, JobResult, Serializer, deserialize_job, serialize_job
@@ -18,6 +20,16 @@
 logger = logging.getLogger('arq.connections')
 
 
+class SSLContext(ssl.SSLContext):
+    """
+    Required to avoid problems with
+    """
+
+    @classmethod
+    def __get_validators__(cls) -> Generator[Callable[..., Any], None, None]:
+        yield make_arbitrary_type_validator(ssl.SSLContext)
+
+
 @dataclass
 class RedisSettings:
     """
@@ -38,8 +50,20 @@ class RedisSettings:
     sentinel: bool = False
     sentinel_master: str = 'mymaster'
 
+    @classmethod
+    def from_dsn(cls, dsn: str) -> 'RedisSettings':
+        conf = urlparse(dsn)
+        assert conf.scheme in {'redis', 'rediss'}, 'invalid DSN scheme'
+        return RedisSettings(
+            host=conf.hostname or 'localhost',
+            port=conf.port or 6379,
+            ssl=conf.scheme == 'rediss',
+            password=conf.password,
+            database=int((conf.path or '0').strip('/')),
+        )
+
     def __repr__(self) -> str:
-        return '<RedisSettings {}>'.format(' '.join(f'{k}={v}' for k, v in self.__dict__.items()))
+        return 'RedisSettings({})'.format(', '.join(f'{k}={v!r}' for k, v in self.__dict__.items()))
 
 
 # extra time after the job is expected to start when the job key should expire, 1 day in ms

docs/requirements.txt

@@ -1,4 +1,4 @@
 docutils==0.14
-Pygments==2.3.1
+Pygments==2.7.2
 Sphinx==2.0.1
 sphinxcontrib-websupport==1.1.0

requirements.txt

@@ -1,2 +1,4 @@
 -r docs/requirements.txt
 -r tests/requirements.txt
+
+pydantic==1.7

setup.py

@@ -34,6 +34,7 @@
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
         'Topic :: Internet',
         'Topic :: Software Development :: Libraries :: Python Modules',
         'Topic :: System :: Clustering',

tests/requirements.txt

@@ -4,7 +4,7 @@ flake8==3.7.9
 flake8-quotes==3
 isort==4.3.21
 msgpack==0.6.1
-mypy==0.770
+mypy==0.790
 pycodestyle==2.5.0
 pyflakes==2.1.1
 pytest==5.3.5

tests/test_utils.py

@@ -3,6 +3,7 @@
 from datetime import timedelta
 
 import pytest
+from pydantic import BaseModel, validator
 
 import arq.typing
 import arq.utils
@@ -13,8 +14,8 @@ def test_settings_changed():
     settings = RedisSettings(port=123)
     assert settings.port == 123
     assert (
-        '<RedisSettings host=localhost port=123 database=0 password=None ssl=None conn_timeout=1 conn_retries=5 '
-        'conn_retry_delay=1 sentinel=False sentinel_master=mymaster>'
+        "RedisSettings(host='localhost', port=123, database=0, password=None, ssl=None, conn_timeout=1, "
+        "conn_retries=5, conn_retry_delay=1, sentinel=False, sentinel_master='mymaster')"
     ) == str(settings)
 
 
@@ -94,3 +95,33 @@ def test_to_seconds(input, output):
 
 def test_typing():
     assert 'OptionType' in arq.typing.__all__
+
+
+def test_redis_settings_validation():
+    class Settings(BaseModel):
+        redis_settings: RedisSettings
+
+        @validator('redis_settings', always=True, pre=True)
+        def parse_redis_settings(cls, v):
+            if isinstance(v, str):
+                return RedisSettings.from_dsn(v)
+            else:
+                return v
+
+    s1 = Settings(redis_settings='redis://foobar:123/4')
+    assert s1.redis_settings.host == 'foobar'
+    assert s1.redis_settings.host == 'foobar'
+    assert s1.redis_settings.port == 123
+    assert s1.redis_settings.database == 4
+    assert s1.redis_settings.ssl is False
+
+    s2 = Settings(redis_settings={'host': 'testing.com'})
+    assert s2.redis_settings.host == 'testing.com'
+    assert s2.redis_settings.port == 6379
+
+    with pytest.raises(ValueError, match='instance of SSLContext expected'):
+        Settings(redis_settings={'ssl': 123})
+
+    s3 = Settings(redis_settings={'ssl': True})
+    assert s3.redis_settings.host == 'localhost'
+    assert s3.redis_settings.ssl is True