diff --git a/Dockerfile b/Dockerfile index 4a82a913..dc3878a1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,12 @@ FROM kozzztik/tulius:base_3.0.3 +CMD [ "python", "app.py" ] + ADD tulius /opt/tulius/tulius ADD djfw /opt/tulius/djfw ADD manage.py /opt/tulius/manage.py ADD requirements.txt /opt/tulius/requirements.txt -ADD wsgi.py /opt/tulius/wsgi.py -ADD async_app.py /opt/tulius/async_app.py +ADD app.py /opt/tulius/app.py ADD settings.py /opt/tulius/settings.py ADD .pylintrc /opt/tulius/.pylintrc ADD tests /opt/tulius/tests @@ -21,4 +22,5 @@ RUN apt-get install git -y ADD .git /opt/tulius/.git ENV TULIUS_BRANCH local +ENV HTTP_PORT 7000 RUN python manage.py compilemessages diff --git a/README.md b/README.md index 7f217e64..42192ac9 100644 --- a/README.md +++ b/README.md @@ -90,8 +90,8 @@ Repo for http://tulius.com project. 11. Install nginx. Configure it using templates: ```bash - cp /home/travis/master/scripts/tulius/nginx/sentry.conf /etc/nginx/conf.d/sentry.conf - cp /home/travis/master/scripts/tulius/nginx/tulius.conf /etc/nginx/conf.d/tulius.conf + cp /home/travis/master/scripts/tulius/nginx_production.conf /etc/nginx/conf.d/tulius_prod.conf + cp /home/travis/master/scripts/tulius/nginx_dev.conf /etc/nginx/conf.d/tulius_dev.conf ``` 12. Install letsEncrypt and configure SSL. @@ -117,7 +117,7 @@ Update repo if needed (use separate branch and PR) ## Running on local environment -To use Tulius on local dev environment you need to run 3 instances. For both of them +To use Tulius on local dev environment you need to run 2 instances. For both of them it is needed to set environment variable: ```bash @@ -129,8 +129,7 @@ file from template and set needed options there. Instances, that needed to run: 1. `manage.py runserver` - Django instance for normal HTTP requests -2. `async_app.py` - for web sockets support -3. `celery -A tulius worker -l info` - for deferred tasks +2. `celery -A tulius worker -l info` - for deferred tasks (optional) On Windows, as Celery not supports it yet, install gevent: @@ -140,6 +139,12 @@ and start celery with: ```celery -A tulius worker -l info -P gevent``` +or, instead of starting Celery, you can switch it off, by adding: +```CELERY_TASK_ALWAYS_EAGER = True``` +to settings_production.py. However, some heavy requests, like reindexing may +became too slow to render pages, as deferred tasks will be resolved in request +context. But for most things it will be enough. + ## Running tests ``` @@ -147,6 +152,7 @@ python -m pylint tests tulius djfw python -m pytest tests tulius djfw ``` +``` ## Remove elastic disk limit on dev environment ``` curl -XPUT "http://localhost:9200/_cluster/settings" \ diff --git a/async_app.py b/app.py similarity index 72% rename from async_app.py rename to app.py index 3bc767a1..ecaede2e 100644 --- a/async_app.py +++ b/app.py @@ -1,14 +1,11 @@ import os -import django - -from tulius.websockets import app +from django_asyncio.app import run_app if os.path.exists('settings_production.py'): settings_file = 'settings_production' else: settings_file = 'settings' - os.environ.setdefault("DJANGO_SETTINGS_MODULE", settings_file) -django.setup() -app.main() + +run_app() diff --git a/djfw/flatpages/middleware.py b/djfw/flatpages/middleware.py index c93b83e2..3fecb541 100755 --- a/djfw/flatpages/middleware.py +++ b/djfw/flatpages/middleware.py @@ -1,9 +1,18 @@ +import asyncio +import functools + from django import http from django.conf import settings from djfw.flatpages import views +def flatpage_middleware(get_response): + if asyncio.iscoroutinefunction(get_response): + return AsyncFlatpageFallbackMiddleware(get_response) + return FlatpageFallbackMiddleware(get_response) + + class FlatpageFallbackMiddleware: def __init__(self, get_response): self.get_response = get_response @@ -24,3 +33,32 @@ def __call__(self, request): if settings.DEBUG: raise return response + + +class AsyncFlatpageFallbackMiddleware: + _is_coroutine = asyncio.coroutines._is_coroutine + + def __init__(self, get_response): + self.get_response = get_response + + async def __call__(self, request): + response = await self.get_response(request) + + if response.status_code != 404: + # No need to check for a flatpage for non-404 responses. + return response + try: + return await asyncio.get_event_loop().run_in_executor( + None, functools.partial( + views.flatpage, request, request.path_info)) + # Return the original response if any errors happened. Because this + # is a middleware, we can't assume the errors will be caught elsewhere. + except http.Http404: + return response + except: + if settings.DEBUG: + raise + return response + + +flatpage_middleware.async_capable = True diff --git a/djfw/pagination/middleware.py b/djfw/pagination/middleware.py index cf435728..5c7b005e 100755 --- a/djfw/pagination/middleware.py +++ b/djfw/pagination/middleware.py @@ -1,3 +1,6 @@ +import asyncio + + def get_page(self): """ A function which will be monkeypatched onto the request to get the current @@ -15,6 +18,12 @@ def get_page(self): return 1 +def pagination_middleware(get_response): + if asyncio.iscoroutinefunction(get_response): + return AsyncPaginationMiddleware(get_response) + return PaginationMiddleware(get_response) + + class PaginationMiddleware: """ Inserts a variable representing the current page onto the request object if @@ -26,3 +35,17 @@ def __init__(self, get_response): def __call__(self, request): request.page = get_page(request) return self.get_response(request) + + +class AsyncPaginationMiddleware: + _is_coroutine = asyncio.coroutines._is_coroutine + + def __init__(self, get_response): + self.get_response = get_response + + async def __call__(self, request): + request.page = get_page(request) + return await self.get_response(request) + + +pagination_middleware.async_capable = True diff --git a/requirements.txt b/requirements.txt index 8bdd62ff..c94936ed 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,6 +22,7 @@ aiohttp==3.8.1 aioredis == 1.3.1 celery == 5.2.3 django-celery-results == 2.4.0 +git+https://github.com/kozzztik/django_asyncio # testing pytest == 5.4.1 diff --git a/scripts/BaseDockerfile b/scripts/BaseDockerfile index efcfb528..4fe5f76d 100644 --- a/scripts/BaseDockerfile +++ b/scripts/BaseDockerfile @@ -6,17 +6,10 @@ ENV TULIUS_BRANCH local RUN mkdir /opt/tulius WORKDIR /opt/tulius -CMD [ "uwsgi", "--socket", "0.0.0.0:7000", \ - "--protocol", "uwsgi", \ - "--max-requests", "5000", \ - "--threads", "4", \ - "--processes", "2", \ - "--master", \ - "--disable-write-exception", \ - "--wsgi", "wsgi:application" ] +CMD [ "/opt/app.py" ] ## install requirements, so they can be cached by Docker -RUN pip install uwsgi django==3.1.1 pytz==2020.1 pillow==6.2.0 \ +RUN pip install django==3.1.1 pytz==2020.1 pillow==6.2.0 \ mysqlclient==2.0.1 pyyaml==5.3.1 django-hamlpy==1.1.1 \ redis==3.5.3 django-redis-cache==2.1.1 aioredis==1.3.1 \ celery==4.4.7 django-celery-results==1.2.1 \ diff --git a/scripts/on_update.sh b/scripts/on_update.sh index 0b2f35cf..4332a88d 100644 --- a/scripts/on_update.sh +++ b/scripts/on_update.sh @@ -14,7 +14,7 @@ fi echo "Stop existing compose" cd scripts/tulius/$ENV # First stop web then wait till celery will finish all tasks -docker-compose stop uwsgi +docker-compose stop docker-compose exec celery python manage.py wait_celery || true docker-compose down --remove-orphans docker system prune --force diff --git a/scripts/tulius/dev/docker-compose.yml b/scripts/tulius/dev/docker-compose.yml index 1dda0d60..0032d5e7 100644 --- a/scripts/tulius/dev/docker-compose.yml +++ b/scripts/tulius/dev/docker-compose.yml @@ -1,6 +1,6 @@ version: '3.5' services: - uwsgi: + web1: image: kozzztik/tulius:dev networks: tuliusnet: @@ -8,6 +8,7 @@ services: restart: unless-stopped environment: TULIUS_BRANCH: dev + HTTP_THREADS: 10 volumes: - ../../../data/media:/opt/tulius/data/media - ../../../data/mail:/opt/tulius/data/mail @@ -17,23 +18,24 @@ services: options: max-size: "100m" max-file: "10" - - websockets: + web2: image: kozzztik/tulius:dev networks: tuliusnet: ipv4_address: 10.5.0.21 restart: unless-stopped - command: ["python3", "/opt/tulius/async_app.py"] environment: TULIUS_BRANCH: dev + HTTP_THREADS: 10 volumes: + - ../../../data/media:/opt/tulius/data/media + - ../../../data/mail:/opt/tulius/data/mail - ../../../settings_production.py:/opt/tulius/settings_production.py logging: driver: json-file options: max-size: "100m" - max-file: "2" + max-file: "10" celery: image: kozzztik/tulius:dev diff --git a/scripts/tulius/docker-compose.yml b/scripts/tulius/docker-compose.yml index 41fe49b4..627a5085 100644 --- a/scripts/tulius/docker-compose.yml +++ b/scripts/tulius/docker-compose.yml @@ -21,21 +21,6 @@ services: - ./mysql.cnf:/etc/my.cnf:ro container_name: tulius_mysql - nginx: - image: nginx - restart: always - networks: - tuliusnet: - ipv4_address: 10.5.0.4 - ports: - - "127.0.0.1:8080:80" - volumes: - - ./nginx.conf:/etc/nginx/nginx.conf:ro - - /home/travis/master/data:/opt/master_data - - /home/travis/dev/data:/opt/dev_data - command: ['nginx', '-g', 'daemon off;'] - container_name: tulius_nginx - mail: build: context: ./postfix diff --git a/scripts/tulius/master/docker-compose.yml b/scripts/tulius/master/docker-compose.yml index b0a39635..7fec1532 100644 --- a/scripts/tulius/master/docker-compose.yml +++ b/scripts/tulius/master/docker-compose.yml @@ -1,6 +1,6 @@ version: '3.5' services: - uwsgi: + web1: image: kozzztik/tulius:master networks: tuliusnet: @@ -8,6 +8,7 @@ services: restart: unless-stopped environment: TULIUS_BRANCH: master + HTTP_THREADS: 1 volumes: - ../../../data/media:/opt/tulius/data/media - ../../../settings_production.py:/opt/tulius/settings_production.py @@ -16,23 +17,23 @@ services: options: max-size: "100m" max-file: "10" - - websockets: + web2: image: kozzztik/tulius:master networks: tuliusnet: ipv4_address: 10.5.0.11 restart: unless-stopped - command: ["python3", "/opt/tulius/async_app.py"] environment: TULIUS_BRANCH: master + HTTP_THREADS: 1 volumes: + - ../../../data/media:/opt/tulius/data/media - ../../../settings_production.py:/opt/tulius/settings_production.py logging: driver: json-file options: max-size: "100m" - max-file: "2" + max-file: "10" celery: image: kozzztik/tulius:master diff --git a/scripts/tulius/nginx.conf b/scripts/tulius/nginx.conf deleted file mode 100644 index 46c7a06d..00000000 --- a/scripts/tulius/nginx.conf +++ /dev/null @@ -1,156 +0,0 @@ -user www-data; -worker_processes 4; -pid /run/nginx.pid; - -events { - worker_connections 768; - # multi_accept on; -} - -http { - sendfile on; - tcp_nopush on; - tcp_nodelay on; - keepalive_timeout 65; - types_hash_max_size 2048; - - include /etc/nginx/mime.types; - default_type application/octet-stream; - - ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # Dropping SSLv3, ref: POODLE - ssl_prefer_server_ciphers on; - - access_log /var/log/nginx/access.log; - error_log /var/log/nginx/error.log; - gzip on; - gzip_disable "msie6"; - - - server { - listen 80; - server_name tulius.com; - server_name tulius.co-de.org; - server_name master.tulius.co-de.org; - charset utf-8; - - # max upload size - client_max_body_size 75M; # adjust to taste - - # Django media - location /media { - alias /opt/master_data/media; # your Django project's media files - amend as required - } - location /static { - alias /opt/master_data/static; # your Django project's static files - amend as required - } - location /robots.txt { - alias /opt/master_data/static/robots.txt; - } - location /favicon.ico { - alias /opt/master_data/static/favicon.ico; - } - location /ws/ { - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - proxy_pass http://10.5.0.11:7000; - } - location /ws_new/ { - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - proxy_pass http://10.5.0.11:7000; - } - # Finally, send all non-media requests to the Django server. - location / { - uwsgi_pass uwsgi://10.5.0.10:7000; - include uwsgi_params; # the uwsgi_params file you installed - } - } - - server { - listen 80; - # the domain name it will serve for - server_name test.tulius.com; - server_name test.tulius.co-de.org; - server_name dev.tulius.co-de.org; - server_name dev.tulius.kozzz.ru; - charset utf-8; - - # max upload size - client_max_body_size 75M; # adjust to taste - - # Django media - location /media { - alias /opt/dev_data/media; # your Django project's media files - amend as required - } - location /static { - alias /opt/dev_data/static; # your Django project's static files - amend as required - } - location /robots.txt { - alias /opt/dev_data/static/no-robots.txt; - } - location /favicon.ico { - alias /opt/dev_data/static/favicon.ico; - } - location /ws/ { - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - proxy_pass http://10.5.0.21:7000; - } - location /ws_new/ { - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - proxy_pass http://10.5.0.21:7000; - } - # Finally, send all non-media requests to the Django server. - location / { - uwsgi_pass uwsgi://10.5.0.20:7000; - include uwsgi_params; # the uwsgi_params file you installed - } - } - - server { - listen 80; - # the domain name it will serve for - server_name sentry.co-de.org; - charset utf-8; - location / { - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_pass http://10.5.0.6:9000; - proxy_redirect off; - } - } - server { - listen 80; - server_name kibana.tulius.com; - server_name kibana.tulius.co-de.org; - charset utf-8; - location / { - proxy_pass http://10.5.0.13:5601; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_redirect off; - } - } - server { - listen 80; - server_name kibana.test.tulius.co-de.org; - charset utf-8; - location / { - proxy_pass http://10.5.0.23:5601; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_redirect off; - } - } -} diff --git a/scripts/tulius/nginx/sentry.conf b/scripts/tulius/nginx/sentry.conf deleted file mode 100644 index e8028fec..00000000 --- a/scripts/tulius/nginx/sentry.conf +++ /dev/null @@ -1,24 +0,0 @@ -server { - listen 80; - server_name sentry.co-de.org; - - location / { - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_pass http://192.168.0.2:80; - proxy_redirect off; - } - - listen 443 ssl; # managed by Certbot - ssl_certificate /etc/letsencrypt/live/svn.milana.co-de.org/fullchain.pem; # managed by Certbot - ssl_certificate_key /etc/letsencrypt/live/svn.milana.co-de.org/privkey.pem; # managed by Certbot - ssl_session_cache shared:le_nginx_SSL:1m; # managed by Certbot - ssl_session_timeout 1440m; # managed by Certbot - - ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # managed by Certbot - ssl_prefer_server_ciphers on; # managed by Certbot - - ssl_ciphers "ECDHE-ECDSA-AES128-GCM-SHA256 ECDHE-ECDSA-AES256-GCM-SHA384 ECDHE-ECDSA-AES128-SHA ECDHE-ECDSA-AES256-SHA ECDHE-ECDSA-AES128-SHA256 ECDHE-ECDSA-AES256-SHA384 ECDHE-RSA-AES128-GCM-SHA256 ECDHE-RSA-AES256-GCM-SHA384 ECDHE-RSA-AES128-SHA ECDHE-RSA-AES128-SHA256 ECDHE-RSA-AES256-SHA384 DHE-RSA-AES128-GCM-SHA256 DHE-RSA-AES256-GCM-SHA384 DHE-RSA-AES128-SHA DHE-RSA-AES256-SHA DHE-RSA-AES128-SHA256 DHE-RSA-AES256-SHA256 EDH-RSA-DES-CBC3-SHA"; # managed by Certbot -} diff --git a/scripts/tulius/nginx/tulius.conf b/scripts/tulius/nginx/tulius.conf deleted file mode 100644 index ceb93f43..00000000 --- a/scripts/tulius/nginx/tulius.conf +++ /dev/null @@ -1,106 +0,0 @@ -server { - listen 80; - server_name www.tulius.com; - return 302 http://tulius.com$request_uri; - - listen 443 ssl; # managed by Certbot - ssl_certificate /etc/letsencrypt/live/svn.milana.co-de.org/fullchain.pem; # managed by Certbot - ssl_certificate_key /etc/letsencrypt/live/svn.milana.co-de.org/privkey.pem; # managed by Certbot - ssl_session_cache shared:le_nginx_SSL:1m; # managed by Certbot - ssl_session_timeout 1440m; # managed by Certbot - - ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # managed by Certbot - ssl_prefer_server_ciphers on; # managed by Certbot - - ssl_ciphers "ECDHE-ECDSA-AES128-GCM-SHA256 ECDHE-ECDSA-AES256-GCM-SHA384 ECDHE-ECDSA-AES128-SHA ECDHE-ECDSA-AES256-SHA ECDHE-ECDSA-AES128-SHA256 ECDHE-ECDSA-AES256-SHA384 ECDHE-RSA-AES128-GCM-SHA256 ECDHE-RSA-AES256-GCM-SHA384 ECDHE-RSA-AES128-SHA ECDHE-RSA-AES128-SHA256 ECDHE-RSA-AES256-SHA384 DHE-RSA-AES128-GCM-SHA256 DHE-RSA-AES256-GCM-SHA384 DHE-RSA-AES128-SHA DHE-RSA-AES256-SHA DHE-RSA-AES128-SHA256 DHE-RSA-AES256-SHA256 EDH-RSA-DES-CBC3-SHA"; # managed by Certbot -} - -server { - listen 80; - server_name test.tulius.com; - server_name test.tulius.co-de.org; - server_name dev.tulius.co-de.org; - server_name dev.tulius.kozzz.ru; -# if ($remote_addr != 178.236.141.69) { -# return 503; -# } - location / { - proxy_set_header Host $host; - proxy_pass http://127.0.0.1:8080; - } - location /ws/ { - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header Host $host; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - proxy_pass http://127.0.0.1:8080; - } - location /ws_new/ { - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header Host $host; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - proxy_pass http://127.0.0.1:8080; - } - listen 443 ssl; # managed by Certbot - ssl_certificate /etc/letsencrypt/live/svn.milana.co-de.org/fullchain.pem; # managed by Certbot - ssl_certificate_key /etc/letsencrypt/live/svn.milana.co-de.org/privkey.pem; # managed by Certbot - - ssl_session_cache shared:le_nginx_SSL:1m; # managed by Certbot - ssl_session_timeout 1440m; # managed by Certbot - - ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # managed by Certbot - ssl_prefer_server_ciphers on; # managed by Certbot - - ssl_ciphers "ECDHE-ECDSA-AES128-GCM-SHA256 ECDHE-ECDSA-AES256-GCM-SHA384 ECDHE-ECDSA-AES128-SHA ECDHE-ECDSA-AES256-SHA ECDHE-ECDSA-AES128-SHA256 ECDHE-ECDSA-AES256-SHA384 ECDHE-RSA-AES128-GCM-SHA256 ECDHE-RSA-AES256-GCM-SHA384 ECDHE-RSA-AES128-SHA ECDHE-RSA-AES128-SHA256 ECDHE-RSA-AES256-SHA384 DHE-RSA-AES128-GCM-SHA256 DHE-RSA-AES256-GCM-SHA384DHE-RSA-AES128-SHA DHE-RSA-AES256-SHA DHE-RSA-AES128-SHA256 DHE-RSA-AES256-SHA256 EDH-RSA-DES-CBC3-SHA"; # managed by Certbot -} - -server { - listen 80; - server_name kibana.test.tulius.co-de.org; - server_name kibana.tulius.co-de.org; - - location / { - proxy_set_header Host $host; - proxy_pass http://127.0.0.1:8080; - auth_basic "Restricted"; - auth_basic_user_file /etc/nginx/htpasswd; - } -} - -server { - listen 80; - listen 443 ssl; - server_name tulius.com; - server_name tulius.co-de.org; - server_name master.tulius.co-de.org; - ssl_certificate /etc/letsencrypt/live/svn.milana.co-de.org/fullchain.pem; # managed by Certbot - ssl_certificate_key /etc/letsencrypt/live/svn.milana.co-de.org/privkey.pem; # managed by Certbot - -# if ($remote_addr != 178.236.141.69) { -# return 503; -# } - - location / { - proxy_set_header Host $host; - proxy_pass http://127.0.0.1:8080; - } - location /ws/ { - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header Host $host; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - proxy_pass http://127.0.0.1:8080; - } - location /ws_new/ { - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header Host $host; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - proxy_pass http://127.0.0.1:8080; - } -} diff --git a/scripts/tulius/nginx_dev.conf b/scripts/tulius/nginx_dev.conf new file mode 100644 index 00000000..6b1b1b5a --- /dev/null +++ b/scripts/tulius/nginx_dev.conf @@ -0,0 +1,110 @@ +#server { +# listen 80; +# listen [::]:80; +# +# server_name test.tulius.com; +# server_name test.tulius.co-de.org; +# server_name dev.tulius.co-de.org; +# server_name dev.tulius.kozzz.ru; +# +# return 301 https://$server_name$request_uri; +#} +upstream aiohttp { + # fail_timeout=0 means we always retry an upstream even if it failed + # to return a good HTTP response + + server 10.5.0.20:7000 fail_timeout=0; + server 10.5.0.21:7000 fail_timeout=0; +} + +server { + listen 80; + # the domain name it will serve for + server_name test.tulius.com; + server_name test.tulius.co-de.org; + server_name dev.tulius.co-de.org; + server_name dev.tulius.kozzz.ru; + charset utf-8; + +# if ($remote_addr != 178.236.141.69) { +# return 503; +# } + + # max upload size + client_max_body_size 75M; # adjust to taste + + # Django media + location /media { + # your Django project's media files - amend as required + alias /home/travis/dev/data/media; + } + location /static { + # your Django project's static files - amend as required + alias /home/travis/dev/data/static; + } + location /robots.txt { + alias /home/travis/dev/data/static/no-robots.txt; + } + location /favicon.ico { + alias /home/travis/dev/data/static/favicon.ico; + } + # Finally, send all non-media requests to the Django server. + location / { + proxy_set_header Host $host; + proxy_set_header User-Agent $http_user_agent; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto-Version $scheme; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "keep-alive, Upgrade"; + proxy_http_version 1.1; + proxy_redirect off; + proxy_buffering off; + proxy_pass http://aiohttp; + } + listen 443 ssl; # managed by Certbot + ssl_certificate /etc/letsencrypt/live/svn.milana.co-de.org/fullchain.pem; # managed by Certbot + ssl_certificate_key /etc/letsencrypt/live/svn.milana.co-de.org/privkey.pem; # managed by Certbot + + ssl_session_cache shared:le_nginx_SSL:1m; # managed by Certbot + ssl_session_timeout 1440m; # managed by Certbot + + ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # managed by Certbot + ssl_prefer_server_ciphers on; # managed by Certbot + + ssl_ciphers "ECDHE-ECDSA-AES128-GCM-SHA256 ECDHE-ECDSA-AES256-GCM-SHA384 ECDHE-ECDSA-AES128-SHA ECDHE-ECDSA-AES256-SHA ECDHE-ECDSA-AES128-SHA256 ECDHE-ECDSA-AES256-SHA384 ECDHE-RSA-AES128-GCM-SHA256 ECDHE-RSA-AES256-GCM-SHA384 ECDHE-RSA-AES128-SHA ECDHE-RSA-AES128-SHA256 ECDHE-RSA-AES256-SHA384 DHE-RSA-AES128-GCM-SHA256 DHE-RSA-AES256-GCM-SHA384DHE-RSA-AES128-SHA DHE-RSA-AES256-SHA DHE-RSA-AES128-SHA256 DHE-RSA-AES256-SHA256 EDH-RSA-DES-CBC3-SHA"; # managed by Certbot + +} + +server { + listen 80; + server_name kibana.tulius.com; + server_name kibana.tulius.co-de.org; + charset utf-8; + location / { + proxy_pass http://10.5.0.13:5601; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_redirect off; + auth_basic "Restricted"; + auth_basic_user_file /etc/nginx/htpasswd; + } +} + +server { + listen 80; + server_name kibana.test.tulius.co-de.org; + charset utf-8; + location / { + proxy_pass http://10.5.0.23:5601; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_redirect off; + auth_basic "Restricted"; + auth_basic_user_file /etc/nginx/htpasswd; + } +} diff --git a/scripts/tulius/nginx_production.conf b/scripts/tulius/nginx_production.conf new file mode 100644 index 00000000..6144b217 --- /dev/null +++ b/scripts/tulius/nginx_production.conf @@ -0,0 +1,84 @@ +server { + listen 80; + server_name www.tulius.com; + return 302 http://tulius.com$request_uri; + + listen 443 ssl; # managed by Certbot + ssl_certificate /etc/letsencrypt/live/svn.milana.co-de.org/fullchain.pem; # managed by Certbot + ssl_certificate_key /etc/letsencrypt/live/svn.milana.co-de.org/privkey.pem; # managed by Certbot + ssl_session_cache shared:le_nginx_SSL:1m; # managed by Certbot + ssl_session_timeout 1440m; # managed by Certbot + + ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # managed by Certbot + ssl_prefer_server_ciphers on; # managed by Certbot + + ssl_ciphers "ECDHE-ECDSA-AES128-GCM-SHA256 ECDHE-ECDSA-AES256-GCM-SHA384 ECDHE-ECDSA-AES128-SHA ECDHE-ECDSA-AES256-SHA ECDHE-ECDSA-AES128-SHA256 ECDHE-ECDSA-AES256-SHA384 ECDHE-RSA-AES128-GCM-SHA256 ECDHE-RSA-AES256-GCM-SHA384 ECDHE-RSA-AES128-SHA E +CDHE-RSA-AES128-SHA256 ECDHE-RSA-AES256-SHA384 DHE-RSA-AES128-GCM-SHA256 DHE-RSA-AES256-GCM-SHA384 DHE-RSA-AES128-SHA DHE-RSA-AES256-SHA DHE-RSA-AES128-SHA256 DHE-RSA-AES256-SHA256 EDH-RSA-DES-CBC3-SHA"; # managed by Certbot +} + +upstream aiohttp { + # fail_timeout=0 means we always retry an upstream even if it failed + # to return a good HTTP response + + server 10.5.0.10:7000 fail_timeout=0; + server 10.5.0.11:7000 fail_timeout=0; +} + +server { + listen 80; + listen 443 ssl; + server_name tulius.com; + server_name tulius.co-de.org; + server_name master.tulius.co-de.org; + ssl_certificate /etc/letsencrypt/live/tulius.com/fullchain.pem; # managed by Certbot + ssl_certificate_key /etc/letsencrypt/live/tulius.com/privkey.pem; # managed by Certbot + client_max_body_size 75M; # adjust to taste + +# if ($remote_addr != 178.236.141.69) { +# return 503; +# } + + # Django media + location /media { + # your Django project's media files - amend as required + alias /home/travis/master/data/media; + } + location /static { + # your Django project's static files - amend as required + alias /home/travis/master/data/static; + } + location /robots.txt { + alias /home/travis/master/data/static/robots.txt; + } + location /favicon.ico { + alias /home/travis/master/data/static/favicon.ico; + } + # Finally, send all non-media requests to the Django server. + location / { + proxy_set_header Host $host; + proxy_set_header User-Agent $http_user_agent; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto-Version $scheme; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "keep-alive, Upgrade"; + proxy_http_version 1.1; + proxy_redirect off; + proxy_buffering off; + proxy_pass http://aiohttp; + } +} + +server { + listen 9201; + server_name tulius.com; + + if ($remote_addr != 178.236.141.69) { + return 403; + } + location / { + proxy_set_header Host $host; + proxy_pass http://10.5.0.30:9200; + } + +} \ No newline at end of file diff --git a/settings.py b/settings.py index dbd8654f..f8f1ef4e 100644 --- a/settings.py +++ b/settings.py @@ -89,19 +89,20 @@ 'tulius.vk', 'tulius.counters', 'tulius.websockets', + 'django_asyncio.DjangoAsyncio', ) MIDDLEWARE = ( # 'raven.contrib.django.raven_compat.middleware.Sentry404CatchMiddleware', - 'tulius.core.profiler.ProfilerMiddleware', + 'tulius.core.profiler.profiler_middleware', 'django.middleware.common.CommonMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', 'django.middleware.locale.LocaleMiddleware', 'django.middleware.csrf.CsrfViewMiddleware', 'django.contrib.auth.middleware.AuthenticationMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', - 'djfw.pagination.middleware.PaginationMiddleware', - 'djfw.flatpages.middleware.FlatpageFallbackMiddleware', + 'djfw.pagination.middleware.pagination_middleware', + 'djfw.flatpages.middleware.flatpage_middleware', ) STATICFILES_FINDERS = ( @@ -127,7 +128,6 @@ 'django.template.context_processors.media', 'django.template.context_processors.request', 'django.template.context_processors.static', - 'tulius.websockets.context_processors.default', 'djfw.flatpages.context_processors.flatpages', 'djfw.datablocks.context_processors.datablocks', ], @@ -182,7 +182,7 @@ LOGGING = { 'version': 1, - 'disable_existing_loggers': True, + 'disable_existing_loggers': False, 'filters': { 'require_debug_false': { '()': 'django.utils.log.RequireDebugFalse' @@ -243,11 +243,6 @@ 'level': 'DEBUG', 'propagate': True, }, - 'async_app': { - 'handlers': ['console'], - 'level': 'DEBUG' if env == 'dev' else 'ERROR', - 'propagate': True, - }, 'profiler': { 'handlers': ['null' if TEST_RUN else 'log_stash'], 'level': 'DEBUG', @@ -298,14 +293,6 @@ 'password': '', } -ASYNC_SERVER = { - 'host': '127.0.0.1' if env == 'dev' else '0.0.0.0', - 'port': 7000 -} - -WEBSOCKET_URL = '/ws/' -WEBSOCKET_URL_NEW = '/ws_new/' - # Actual credentials are hold in settings_production.py file. DATABASES = { 'default': { @@ -362,3 +349,8 @@ CELERY_WORKER_CONCURRENCY = 3 CELERY_EVENT_QUEUE_PREFIX = f'{env}_' CELERY_TASK_ALWAYS_EAGER = TEST_RUN + +HTTP_HOST = '0.0.0.0' +HTTP_PORT = os.environ.get('HTTP_PORT', 7000) +HTTP_KEEP_ALIVE = 75.0 +HTTP_THREADS = int(os.environ.get('HTTP_THREADS', 10)) diff --git a/tulius/core/profiler.py b/tulius/core/profiler.py index 1ab51f3f..9ff9bee8 100644 --- a/tulius/core/profiler.py +++ b/tulius/core/profiler.py @@ -1,84 +1,133 @@ +import asyncio +import socket + import threading import time import logging -from django import http from ua_parser import user_agent_parser +from django import http + +host_name = socket.gethostname() + + +def get_user_data(request): + result = {} + try: + user_data = user_agent_parser.Parse( + request.META['HTTP_USER_AGENT'] + if 'HTTP_USER_AGENT' in request.META else '') + user_agent = user_data['user_agent'] + os = user_data['os'] + device_family = user_data['device']['family'] + result = { + 'browser': user_agent['family'], + 'browser_version': user_agent['major'], + 'os': os['family'], + 'os_version': os['major'], + 'device': device_family, + 'mobile': device_family not in [None, 'Spider', 'Other'], + } + if user_agent['minor']: + result['browser_version'] += '.' + user_agent['minor'] + if not result['os_version']: + os_list = result['os'].split() + if len(os_list) == 2: + result['os'] = os_list[0] + result['os_version'] = os_list[1] + if os['minor']: + result['os_version'] += '.' + os['minor'] + except Exception: + logger = logging.getLogger('django.request') + logger.error( + 'Cant parse user agent %s', + request.META.get('HTTP_USER_AGENT')) + return result -class ProfilerMiddleware: + +def log_record(request, exec_time, response): + # get only cached user. Don't trigger auth just for profiling needs. + # Also, auth middleware may be not installed. + user = getattr(request, '_cached_user', None) + if isinstance(response, http.StreamingHttpResponse): + content_length = None + else: + content_length = len(response.content) + aiohttp = getattr(request, 'aiohttp_context', None) + data = {} + if aiohttp: + data['session_miss_count'] = aiohttp.get('session_miss', 0) + if 'session' in aiohttp: + data['context_session_key'] = aiohttp['session'].session_key + session = getattr(request, 'session', None) + if session: + data['session_key'] = session.session_key + if 'context_session_key' in data and 'session_key' in data: + data['session_miss'] = \ + data['session_key'] != data['context_session_key'] + logging.getLogger('profiler').info(request.path, extra={ + 'host_name': host_name, + 'aiohttp': bool(aiohttp), + 'requests_count': aiohttp["requests"] if aiohttp else 1, + 'method': request.method, + 'status_code': response.status_code, + 'content_length': content_length, + **({ + 'app_name': request.resolver_match.app_name, + 'url_name': request.resolver_match.url_name, + 'view_name': request.resolver_match.view_name, + 'url_args': request.resolver_match.args, + 'url_kwargs': [{ + 'name': name, + 'value': value + } for name, value in request.resolver_match.kwargs.items()] + } if request.resolver_match else {}), + 'user': { + 'id': user.id, + 'title': user.username + } if user and user.is_authenticated else None, + 'exec_time': exec_time / 1000000, + 'thread_id': threading.current_thread().ident, + 'ip': request.META['REMOTE_ADDR'], + **get_user_data(request), + **request.profiling_data, + **data, + }) + + +def profiler_middleware(get_response): + if asyncio.iscoroutinefunction(get_response): + return AsyncProfilerMiddleware(get_response) + return SyncProfilerMiddleware(get_response) + + +class SyncProfilerMiddleware: def __init__(self, get_response): self.get_response = get_response - @staticmethod - def get_user_data(request): - result = {} - try: - user_data = user_agent_parser.Parse( - request.META['HTTP_USER_AGENT'] - if 'HTTP_USER_AGENT' in request.META else '') - user_agent = user_data['user_agent'] - os = user_data['os'] - device_family = user_data['device']['family'] - result = { - 'browser': user_agent['family'], - 'browser_version': user_agent['major'], - 'os': os['family'], - 'os_version': os['major'], - 'device': device_family, - 'mobile': device_family not in [None, 'Spider', 'Other'], - - } - if user_agent['minor']: - result['browser_version'] += '.' + user_agent['minor'] - if not result['os_version']: - os_list = result['os'].split() - if len(os_list) == 2: - result['os'] = os_list[0] - result['os_version'] = os_list[1] - if os['minor']: - result['os_version'] += '.' + os['minor'] - except Exception: - logger = logging.getLogger('django.request') - logger.error( - 'Cant parse user agent %s', - request.META.get('HTTP_USER_AGENT')) - return result - - def log_record(self, request, exec_time, response): - if isinstance(response, http.StreamingHttpResponse): - content_length = None - else: - content_length = len(response.content) - logging.getLogger('profiler').info(request.path, extra={ - 'method': request.method, - 'status_code': response.status_code, - 'content_length': content_length, - **({ - 'app_name': request.resolver_match.app_name, - 'url_name': request.resolver_match.url_name, - 'view_name': request.resolver_match.view_name, - 'url_args': request.resolver_match.args, - 'url_kwargs': [{ - 'name': name, - 'value': value - } for name, value in request.resolver_match.kwargs.items()] - } if request.resolver_match else {}), - 'user': { - 'id': request.user.id, - 'title': request.user.username - } if request.user.is_authenticated else None, - 'exec_time': exec_time / 1000000, - 'thread_id': threading.current_thread().ident, - 'ip': request.META['REMOTE_ADDR'], - **self.get_user_data(request), - **request.profiling_data, - }) - def __call__(self, request): start_time = time.perf_counter_ns() request.profiling_data = {} response = self.get_response(request) exec_time = time.perf_counter_ns() - start_time - self.log_record(request, exec_time, response) + log_record(request, exec_time, response) return response + + +class AsyncProfilerMiddleware: + _is_coroutine = asyncio.coroutines._is_coroutine + + def __init__(self, get_response): + self.get_response = get_response + + async def __call__(self, request): + start_time = time.perf_counter_ns() + request.profiling_data = {} + response = await self.get_response(request) + exec_time = time.perf_counter_ns() - start_time + log_record(request, exec_time, response) + return response + + +profiler_middleware.async_capable = True diff --git a/tulius/forum/elastic_search/mapping.py b/tulius/forum/elastic_search/mapping.py index f435d119..ff3e46b6 100644 --- a/tulius/forum/elastic_search/mapping.py +++ b/tulius/forum/elastic_search/mapping.py @@ -1,5 +1,5 @@ -from django.db import models from elasticsearch7 import exceptions +from django.db import models from tulius.forum.elastic_search import models as elastic_models diff --git a/tulius/forum/elastic_search/models.py b/tulius/forum/elastic_search/models.py index 2f5e3ba0..fb89a4da 100644 --- a/tulius/forum/elastic_search/models.py +++ b/tulius/forum/elastic_search/models.py @@ -4,6 +4,7 @@ import datetime from logging import handlers +import elasticsearch7 from django.apps import apps from django.conf import settings from django.db import models @@ -11,7 +12,6 @@ from django.db.models.fields import related from django.db.models.fields import reverse_related from django.core.serializers.json import DjangoJSONEncoder -import elasticsearch7 logger = logging.getLogger('elastic_search_indexing') diff --git a/tulius/forum/online_status/__init__.py b/tulius/forum/online_status/__init__.py index e6d6a366..d7ff34da 100755 --- a/tulius/forum/online_status/__init__.py +++ b/tulius/forum/online_status/__init__.py @@ -1,11 +1,12 @@ import datetime +from redis import client from django import dispatch from django.conf import settings from django.contrib import auth from django.core.cache import cache from django.utils import html -from redis import client +from django.db import transaction from tulius.forum import core from tulius.forum.threads import models as thread_models @@ -79,6 +80,7 @@ def get_online_users(self, do_update=True): users = {u.pk: u for u in users} return [users.get(int(pk)) for pk in ids] + @transaction.non_atomic_requests def get(self, request, *args, **kwargs): pk = kwargs['pk'] if 'pk' in kwargs else None if pk: diff --git a/tulius/forum/read_marks/tasks.py b/tulius/forum/read_marks/tasks.py index 2245fcc9..2b39db0b 100644 --- a/tulius/forum/read_marks/tasks.py +++ b/tulius/forum/read_marks/tasks.py @@ -1,5 +1,5 @@ -from django.apps import apps from celery import shared_task +from django.apps import apps def update_read_marks_on_rights_async(thread, only_users=None): diff --git a/tulius/players/avatar_upload.py b/tulius/players/avatar_upload.py index 69f8a6c9..ef37c793 100755 --- a/tulius/players/avatar_upload.py +++ b/tulius/players/avatar_upload.py @@ -2,10 +2,10 @@ from mimetypes import guess_type import os +from PIL import Image from django.contrib.auth.decorators import login_required from django.core.files.base import ContentFile from django.conf import settings -from PIL import Image from tulius.stories.models import AVATAR_SIZES, AVATAR_SAVE_SIZE from djfw.uploader import handle_upload diff --git a/tulius/static/app/app.js b/tulius/static/app/app.js index 5e1f99ba..82427b9e 100644 --- a/tulius/static/app/app.js +++ b/tulius/static/app/app.js @@ -12,8 +12,15 @@ const router = new VueRouter({ routes: routes, }) +function production_url() { + var schema = window.location.protocol == 'https:' ? 'wss://' : 'ws://'; + return schema + window.location.host + '/api/ws/'; +} + +var websockets_url = production_url(); + axios.get('/api/app_settings/').then(response => { - Vue.use(VueNativeSock.default, response.data.websockets_url, { + Vue.use(VueNativeSock.default, websockets_url, { reconnection: true, reconnectionDelay: 3000, format: 'json' diff --git a/tulius/stories/avatar_uploads.py b/tulius/stories/avatar_uploads.py index 239f128f..afd74090 100755 --- a/tulius/stories/avatar_uploads.py +++ b/tulius/stories/avatar_uploads.py @@ -2,12 +2,12 @@ import io from mimetypes import guess_type +from PIL import Image from django.contrib.auth.decorators import login_required from django.http import Http404, HttpResponseBadRequest from django.shortcuts import get_object_or_404 from django.core.files.base import ContentFile from django.conf import settings -from PIL import Image from djfw.uploader import handle_upload from tulius.stories.models import AvatarAlternative diff --git a/tulius/stories/materials_views.py b/tulius/stories/materials_views.py index 764c50a3..c44444da 100755 --- a/tulius/stories/materials_views.py +++ b/tulius/stories/materials_views.py @@ -2,11 +2,11 @@ import io from mimetypes import guess_type +from PIL import Image from django.http import Http404, HttpResponseBadRequest from django.conf import settings from django.core.files.base import ContentFile from django.shortcuts import get_object_or_404 -from PIL import Image from djfw.uploader import handle_upload, handle_download_file_path from djfw.common import generate_random_id diff --git a/tulius/templates/base.haml b/tulius/templates/base.haml index 2738920c..9fab8a43 100755 --- a/tulius/templates/base.haml +++ b/tulius/templates/base.haml @@ -18,7 +18,8 @@ %script{type: 'text/javascript', src: '{{ STATIC_URL }}common/js/autocomplete-select.js'} %script{type: 'text/javascript', src: '{{ STATIC_URL }}websockets/ws4redis.js'}