Previous dev

.gitignore

@@ -15,7 +15,7 @@ dist/
 downloads/
 eggs/
 .eggs/
-lib/
+/lib/
 lib64/
 parts/
 sdist/
@@ -94,3 +94,6 @@ ENV/
 settings_production.py
 /sql-logfile.txt
 /logfile.txt
+/tulius/static/node_modules/
+/tulius/static/app/index_bundle.js
+/tulius/static/package-lock.json

Dockerfile

@@ -1,11 +1,16 @@
 FROM kozzztik/tulius:base_3.0.3
 
+RUN curl -sL https://deb.nodesource.com/setup_12.x | bash -
+RUN apt-get update && apt-get install nodejs git -y
+RUN npm install @vue/cli -g
+RUN pip install hypercorn==0.11.1
+CMD [ "hypercorn", "-b", "0.0.0.0:7000", "-w", "2", "asgi:application" ]
+
 ADD tulius /opt/tulius/tulius
 ADD djfw /opt/tulius/djfw
 ADD manage.py /opt/tulius/manage.py
 ADD requirements.txt /opt/tulius/requirements.txt
-ADD wsgi.py /opt/tulius/wsgi.py
-ADD async_app.py /opt/tulius/async_app.py
+ADD asgi.py /opt/tulius/asgi.py
 ADD settings.py /opt/tulius/settings.py
 ADD .pylintrc /opt/tulius/.pylintrc
 ADD tests /opt/tulius/tests
@@ -22,3 +27,4 @@ ADD .git /opt/tulius/.git
 
 ENV TULIUS_BRANCH local
 RUN python manage.py compilemessages
+RUN cd tulius/static && npm install && npm run build

README.md

@@ -88,10 +88,10 @@ Repo for http://tulius.com project.
     ```
     Edit settings files. Change DB passwords and sentry DSN.
 
-11. Install nginx. Configure it using templates:
+11. Configure Nginx using templates:
     ```bash
-    cp /home/travis/master/scripts/tulius/nginx/sentry.conf /etc/nginx/conf.d/sentry.conf
-    cp /home/travis/master/scripts/tulius/nginx/tulius.conf /etc/nginx/conf.d/tulius.conf
+    cp /home/travis/master/scripts/tulius/nginx_production.conf /etc/nginx/conf.d/tulius_prod.conf
+    cp /home/travis/master/scripts/tulius/nginx_dev.conf /etc/nginx/conf.d/tulius_dev.conf
     ```
 
 12. Install letsEncrypt and configure SSL.
@@ -115,10 +115,21 @@ Update repo if needed (use separate branch and PR)
     ``` 
 16. Check that everything works. Profit.
 
-## Running on local environment
+# Running on local environment
 
-To use Tulius on local dev environment you need to run 3 instances. For both of them
-it is needed to set environment variable:
+To run frontend part you need to install node.js & npm:
+- for Windows use installer https://nodejs.org/en/download/ 
+- for Linux (Debian based) ```apt-get install nodejs```
+
+Then switch to tulius/static folder and run 
+```
+npm install @vue/cli -g
+npm install -D
+```
+
+To use Tulius on local dev environment you need to at least 2 instances
+3 for correct work of heavy requests). 
+For all of them it is needed to set environment variable:
 
 ```bash
 TULIUS_BRANCH=local
@@ -128,10 +139,10 @@ If you need some special configuration options, you can create `settings_product
 file from template and set needed options there.
 
 Instances, that needed to run:
-1. `manage.py runserver` - Django instance for normal HTTP requests
-2. `async_app.py` - for web sockets support
-3. `celery -A tulius worker -l info` - for deferred tasks
-
+1. `manage.py runserver` - Django instance for backend HTTP requests
+2. `celery -A tulius worker -l info` - for deferred tasks (optional)
+3. `npm run serve` in tulius/static directory for frontend webpack dev server
+ 
 On Windows, as Celery not supports it yet, install gevent:
 
 ```pip install gevent```
@@ -140,6 +151,14 @@ and start celery with:
 
 ```celery -A tulius worker -l info -P gevent```
 
+or, instead of starting Celery, you can switch it off, by adding:
+
+```CELERY_TASK_ALWAYS_EAGER = True```
+
+to settings_production.py. However, some heavy requests, like reindexing may
+became too slow to render pages, as deferred tasks will be resolved in request 
+context. But for most things it will be enough.
+
 ## Running tests
 
 ``` 

asgi.py

@@ -0,0 +1,18 @@
+import os
+
+from django.conf import settings
+from django.contrib.staticfiles import handlers as static_handlers
+
+from tulius.websockets.asgi.asgi_handler import get_asgi_application
+
+if os.path.exists('settings_production.py'):
+    settings_file = 'settings_production'
+else:
+    settings_file = 'settings'
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", settings_file)
+
+
+application = get_asgi_application()
+
+if settings.DEBUG:
+    application = static_handlers.ASGIStaticFilesHandler(application)

djfw/flatpages/middleware.py

@@ -1,9 +1,18 @@
+import asyncio
+import functools
+
 from django import http
 from django.conf import settings
 
 from djfw.flatpages import views
 
 
+def flatpage_middleware(get_response):
+    if asyncio.iscoroutinefunction(get_response):
+        return AsyncFlatpageFallbackMiddleware(get_response)
+    return FlatpageFallbackMiddleware(get_response)
+
+
 class FlatpageFallbackMiddleware:
     def __init__(self, get_response):
         self.get_response = get_response
@@ -20,7 +29,32 @@ def __call__(self, request):
         # is a middleware, we can't assume the errors will be caught elsewhere.
         except http.Http404:
             return response
+
+
+class AsyncFlatpageFallbackMiddleware:
+    _is_coroutine = asyncio.coroutines._is_coroutine
+
+    def __init__(self, get_response):
+        self.get_response = get_response
+
+    async def __call__(self, request):
+        response = await self.get_response(request)
+
+        if response.status_code != 404:
+            # No need to check for a flatpage for non-404 responses.
+            return response
+        try:
+            return await asyncio.get_event_loop().run_in_executor(
+                None, functools.partial(
+                    views.flatpage, request, request.path_info))
+        # Return the original response if any errors happened. Because this
+        # is a middleware, we can't assume the errors will be caught elsewhere.
+        except http.Http404:
+            return response
         except:
             if settings.DEBUG:
                 raise
             return response
+
+
+flatpage_middleware.async_capable = True

djfw/pagination/middleware.py

@@ -1,3 +1,6 @@
+import asyncio
+
+
 def get_page(self):
     """
     A function which will be monkeypatched onto the request to get the current
@@ -15,6 +18,12 @@ def get_page(self):
         return 1
 
 
+def pagination_middleware(get_response):
+    if asyncio.iscoroutinefunction(get_response):
+        return AsyncPaginationMiddleware(get_response)
+    return PaginationMiddleware(get_response)
+
+
 class PaginationMiddleware:
     """
     Inserts a variable representing the current page onto the request object if
@@ -26,3 +35,17 @@ def __init__(self, get_response):
     def __call__(self, request):
         request.page = get_page(request)
         return self.get_response(request)
+
+
+class AsyncPaginationMiddleware:
+    _is_coroutine = asyncio.coroutines._is_coroutine
+
+    def __init__(self, get_response):
+        self.get_response = get_response
+
+    async def __call__(self, request):
+        request.page = get_page(request)
+        return await self.get_response(request)
+
+
+pagination_middleware.async_capable = True

scripts/BaseDockerfile

@@ -1,22 +1,17 @@
 FROM python:3.7
 EXPOSE 7000
-RUN apt-get update && apt-get install gettext -y
+RUN curl -sL https://deb.nodesource.com/setup_12.x | bash -
+RUN apt-get update && apt-get install gettext git nodejs -y
+RUN npm install @vue/cli -g
 ENV PYTHONUNBUFFERED 1
 ENV TULIUS_BRANCH local
 RUN mkdir /opt/tulius
 WORKDIR /opt/tulius
 
-CMD [ "uwsgi", "--socket", "0.0.0.0:7000", \
-               "--protocol", "uwsgi", \
-               "--max-requests", "5000", \
-               "--threads", "4", \
-               "--processes", "2", \
-               "--master", \
-               "--disable-write-exception", \
-               "--wsgi", "wsgi:application" ]
+CMD [ "hypercorn", "-b", "0.0.0.0:7000", "-w", "2", "asgi:application" ]
 
 ## install requirements, so they can be cached by Docker
-RUN pip install uwsgi django==3.1.1 pytz==2020.1 pillow==6.2.0 \
+RUN pip install hypercorn==0.11.1 django==3.1.1 pytz==2020.1 pillow==6.2.0 \
     mysqlclient==2.0.1 pyyaml==5.3.1 django-hamlpy==1.1.1 \
     redis==3.5.3 django-redis-cache==2.1.1 aioredis==1.3.1 \
     celery==4.4.7 django-celery-results==1.2.1 \

scripts/on_update.sh

@@ -14,7 +14,7 @@ fi
 echo "Stop existing compose"
 cd scripts/tulius/$ENV
 # First stop web then wait till celery will finish all tasks
-docker-compose stop uwsgi
+docker-compose stop web
 docker-compose exec celery python manage.py wait_celery || true
 docker-compose down --remove-orphans
 docker system prune --force

scripts/tulius/dev/docker-compose.yml

@@ -1,6 +1,6 @@
 version: '3.5'
 services:
-  uwsgi:
+  web:
     image: kozzztik/tulius:dev
     networks:
       tuliusnet:
@@ -18,23 +18,6 @@ services:
         max-size: "100m"
         max-file: "10"
 
-  websockets:
-    image: kozzztik/tulius:dev
-    networks:
-      tuliusnet:
-        ipv4_address: 10.5.0.21
-    restart: unless-stopped
-    command: ["python3", "/opt/tulius/async_app.py"]
-    environment:
-      TULIUS_BRANCH: dev
-    volumes:
-      - ../../../settings_production.py:/opt/tulius/settings_production.py
-    logging:
-      driver: json-file
-      options:
-        max-size: "100m"
-        max-file: "2"
-
   celery:
     image: kozzztik/tulius:dev
     networks:

scripts/tulius/docker-compose.yml

@@ -21,21 +21,6 @@ services:
       - ./mysql.cnf:/etc/my.cnf:ro
     container_name: tulius_mysql
 
-  nginx:
-    image: nginx
-    restart: always
-    networks:
-      tuliusnet:
-        ipv4_address: 10.5.0.4
-    ports:
-      - "127.0.0.1:8080:80"
-    volumes:
-      - ./nginx.conf:/etc/nginx/nginx.conf:ro
-      - /home/travis/master/data:/opt/master_data
-      - /home/travis/dev/data:/opt/dev_data
-    command: ['nginx', '-g', 'daemon off;']
-    container_name: tulius_nginx
-
   mail:
     build:
       context: ./postfix

scripts/tulius/master/docker-compose.yml

@@ -1,6 +1,6 @@
 version: '3.5'
 services:
-  uwsgi:
+  web:
     image: kozzztik/tulius:master
     networks:
       tuliusnet:
@@ -17,23 +17,6 @@ services:
         max-size: "100m"
         max-file: "10"
 
-  websockets:
-    image: kozzztik/tulius:master
-    networks:
-      tuliusnet:
-        ipv4_address: 10.5.0.11
-    restart: unless-stopped
-    command: ["python3", "/opt/tulius/async_app.py"]
-    environment:
-      TULIUS_BRANCH: master
-    volumes:
-      - ../../../settings_production.py:/opt/tulius/settings_production.py
-    logging:
-      driver: json-file
-      options:
-        max-size: "100m"
-        max-file: "2"
-
   celery:
     image: kozzztik/tulius:master
     networks: