Async django

Dockerfile

@@ -1,11 +1,12 @@
 FROM kozzztik/tulius:base_3.0.3
 
+CMD [ "python", "app.py" ]
+
 ADD tulius /opt/tulius/tulius
 ADD djfw /opt/tulius/djfw
 ADD manage.py /opt/tulius/manage.py
 ADD requirements.txt /opt/tulius/requirements.txt
-ADD wsgi.py /opt/tulius/wsgi.py
-ADD async_app.py /opt/tulius/async_app.py
+ADD app.py /opt/tulius/app.py
 ADD settings.py /opt/tulius/settings.py
 ADD .pylintrc /opt/tulius/.pylintrc
 ADD tests /opt/tulius/tests
@@ -21,4 +22,5 @@ RUN apt-get install git -y
 ADD .git /opt/tulius/.git
 
 ENV TULIUS_BRANCH local
+ENV HTTP_PORT 7000
 RUN python manage.py compilemessages

README.md

@@ -90,8 +90,8 @@ Repo for http://tulius.com project.
 
 11. Install nginx. Configure it using templates:
     ```bash
-    cp /home/travis/master/scripts/tulius/nginx/sentry.conf /etc/nginx/conf.d/sentry.conf
-    cp /home/travis/master/scripts/tulius/nginx/tulius.conf /etc/nginx/conf.d/tulius.conf
+    cp /home/travis/master/scripts/tulius/nginx_production.conf /etc/nginx/conf.d/tulius_prod.conf
+    cp /home/travis/master/scripts/tulius/nginx_dev.conf /etc/nginx/conf.d/tulius_dev.conf
     ```
 
 12. Install letsEncrypt and configure SSL.
@@ -117,7 +117,7 @@ Update repo if needed (use separate branch and PR)
 
 ## Running on local environment
 
-To use Tulius on local dev environment you need to run 3 instances. For both of them
+To use Tulius on local dev environment you need to run 2 instances. For both of them
 it is needed to set environment variable:
 
 ```bash
@@ -129,8 +129,7 @@ file from template and set needed options there.
 
 Instances, that needed to run:
 1. `manage.py runserver` - Django instance for normal HTTP requests
-2. `async_app.py` - for web sockets support
-3. `celery -A tulius worker -l info` - for deferred tasks
+2. `celery -A tulius worker -l info` - for deferred tasks (optional)
 
 On Windows, as Celery not supports it yet, install gevent:
 
@@ -140,13 +139,20 @@ and start celery with:
 
 ```celery -A tulius worker -l info -P gevent```
 
+or, instead of starting Celery, you can switch it off, by adding:
+```CELERY_TASK_ALWAYS_EAGER = True```
+to settings_production.py. However, some heavy requests, like reindexing may
+became too slow to render pages, as deferred tasks will be resolved in request 
+context. But for most things it will be enough.
+
 ## Running tests
 
 ``` 
 python -m pylint tests tulius djfw
 python -m pytest tests tulius djfw
 ```
 
+```
 ## Remove elastic disk limit on dev environment
 ```
 curl -XPUT "http://localhost:9200/_cluster/settings" \

async_app.py → app.py

@@ -1,14 +1,11 @@
 import os
-import django
-
-from tulius.websockets import app
 
+from django_asyncio.app import run_app
 
 if os.path.exists('settings_production.py'):
     settings_file = 'settings_production'
 else:
     settings_file = 'settings'
-
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", settings_file)
-django.setup()
-app.main()
+
+run_app()

djfw/flatpages/middleware.py

@@ -1,9 +1,18 @@
+import asyncio
+import functools
+
 from django import http
 from django.conf import settings
 
 from djfw.flatpages import views
 
 
+def flatpage_middleware(get_response):
+    if asyncio.iscoroutinefunction(get_response):
+        return AsyncFlatpageFallbackMiddleware(get_response)
+    return FlatpageFallbackMiddleware(get_response)
+
+
 class FlatpageFallbackMiddleware:
     def __init__(self, get_response):
         self.get_response = get_response
@@ -24,3 +33,32 @@ def __call__(self, request):
             if settings.DEBUG:
                 raise
             return response
+
+
+class AsyncFlatpageFallbackMiddleware:
+    _is_coroutine = asyncio.coroutines._is_coroutine
+
+    def __init__(self, get_response):
+        self.get_response = get_response
+
+    async def __call__(self, request):
+        response = await self.get_response(request)
+
+        if response.status_code != 404:
+            # No need to check for a flatpage for non-404 responses.
+            return response
+        try:
+            return await asyncio.get_event_loop().run_in_executor(
+                None, functools.partial(
+                    views.flatpage, request, request.path_info))
+        # Return the original response if any errors happened. Because this
+        # is a middleware, we can't assume the errors will be caught elsewhere.
+        except http.Http404:
+            return response
+        except:
+            if settings.DEBUG:
+                raise
+            return response
+
+
+flatpage_middleware.async_capable = True

djfw/pagination/middleware.py

@@ -1,3 +1,6 @@
+import asyncio
+
+
 def get_page(self):
     """
     A function which will be monkeypatched onto the request to get the current
@@ -15,6 +18,12 @@ def get_page(self):
         return 1
 
 
+def pagination_middleware(get_response):
+    if asyncio.iscoroutinefunction(get_response):
+        return AsyncPaginationMiddleware(get_response)
+    return PaginationMiddleware(get_response)
+
+
 class PaginationMiddleware:
     """
     Inserts a variable representing the current page onto the request object if
@@ -26,3 +35,17 @@ def __init__(self, get_response):
     def __call__(self, request):
         request.page = get_page(request)
         return self.get_response(request)
+
+
+class AsyncPaginationMiddleware:
+    _is_coroutine = asyncio.coroutines._is_coroutine
+
+    def __init__(self, get_response):
+        self.get_response = get_response
+
+    async def __call__(self, request):
+        request.page = get_page(request)
+        return await self.get_response(request)
+
+
+pagination_middleware.async_capable = True

requirements.txt

@@ -22,6 +22,7 @@ aiohttp==3.8.1
 aioredis == 1.3.1
 celery == 5.2.3
 django-celery-results == 2.4.0
+git+https://github.com/kozzztik/django_asyncio
 
 # testing
 pytest == 5.4.1

scripts/BaseDockerfile

@@ -6,17 +6,10 @@ ENV TULIUS_BRANCH local
 RUN mkdir /opt/tulius
 WORKDIR /opt/tulius
 
-CMD [ "uwsgi", "--socket", "0.0.0.0:7000", \
-               "--protocol", "uwsgi", \
-               "--max-requests", "5000", \
-               "--threads", "4", \
-               "--processes", "2", \
-               "--master", \
-               "--disable-write-exception", \
-               "--wsgi", "wsgi:application" ]
+CMD [ "/opt/app.py" ]
 
 ## install requirements, so they can be cached by Docker
-RUN pip install uwsgi django==3.1.1 pytz==2020.1 pillow==6.2.0 \
+RUN pip install django==3.1.1 pytz==2020.1 pillow==6.2.0 \
     mysqlclient==2.0.1 pyyaml==5.3.1 django-hamlpy==1.1.1 \
     redis==3.5.3 django-redis-cache==2.1.1 aioredis==1.3.1 \
     celery==4.4.7 django-celery-results==1.2.1 \

scripts/on_update.sh

@@ -14,7 +14,7 @@ fi
 echo "Stop existing compose"
 cd scripts/tulius/$ENV
 # First stop web then wait till celery will finish all tasks
-docker-compose stop uwsgi
+docker-compose stop
 docker-compose exec celery python manage.py wait_celery || true
 docker-compose down --remove-orphans
 docker system prune --force

scripts/tulius/dev/docker-compose.yml

@@ -1,13 +1,14 @@
 version: '3.5'
 services:
-  uwsgi:
+  web1:
     image: kozzztik/tulius:dev
     networks:
       tuliusnet:
         ipv4_address: 10.5.0.20
     restart: unless-stopped
     environment:
       TULIUS_BRANCH: dev
+      HTTP_THREADS: 10
     volumes:
       - ../../../data/media:/opt/tulius/data/media
       - ../../../data/mail:/opt/tulius/data/mail
@@ -17,23 +18,24 @@ services:
       options:
         max-size: "100m"
         max-file: "10"
-
-  websockets:
+  web2:
     image: kozzztik/tulius:dev
     networks:
       tuliusnet:
         ipv4_address: 10.5.0.21
     restart: unless-stopped
-    command: ["python3", "/opt/tulius/async_app.py"]
     environment:
       TULIUS_BRANCH: dev
+      HTTP_THREADS: 10
     volumes:
+      - ../../../data/media:/opt/tulius/data/media
+      - ../../../data/mail:/opt/tulius/data/mail
       - ../../../settings_production.py:/opt/tulius/settings_production.py
     logging:
       driver: json-file
       options:
         max-size: "100m"
-        max-file: "2"
+        max-file: "10"
 
   celery:
     image: kozzztik/tulius:dev

scripts/tulius/docker-compose.yml

@@ -21,21 +21,6 @@ services:
       - ./mysql.cnf:/etc/my.cnf:ro
     container_name: tulius_mysql
 
-  nginx:
-    image: nginx
-    restart: always
-    networks:
-      tuliusnet:
-        ipv4_address: 10.5.0.4
-    ports:
-      - "127.0.0.1:8080:80"
-    volumes:
-      - ./nginx.conf:/etc/nginx/nginx.conf:ro
-      - /home/travis/master/data:/opt/master_data
-      - /home/travis/dev/data:/opt/dev_data
-    command: ['nginx', '-g', 'daemon off;']
-    container_name: tulius_nginx
-
   mail:
     build:
       context: ./postfix

scripts/tulius/master/docker-compose.yml

@@ -1,13 +1,14 @@
 version: '3.5'
 services:
-  uwsgi:
+  web1:
     image: kozzztik/tulius:master
     networks:
       tuliusnet:
         ipv4_address: 10.5.0.10
     restart: unless-stopped
     environment:
       TULIUS_BRANCH: master
+      HTTP_THREADS: 1
     volumes:
       - ../../../data/media:/opt/tulius/data/media
       - ../../../settings_production.py:/opt/tulius/settings_production.py
@@ -16,23 +17,23 @@ services:
       options:
         max-size: "100m"
         max-file: "10"
-
-  websockets:
+  web2:
     image: kozzztik/tulius:master
     networks:
       tuliusnet:
         ipv4_address: 10.5.0.11
     restart: unless-stopped
-    command: ["python3", "/opt/tulius/async_app.py"]
     environment:
       TULIUS_BRANCH: master
+      HTTP_THREADS: 1
     volumes:
+      - ../../../data/media:/opt/tulius/data/media
       - ../../../settings_production.py:/opt/tulius/settings_production.py
     logging:
       driver: json-file
       options:
         max-size: "100m"
-        max-file: "2"
+        max-file: "10"
 
   celery:
     image: kozzztik/tulius:master