From 0eb2a1393a1339d4a34d183fb8ecfdd8b4a88f23 Mon Sep 17 00:00:00 2001 From: Eugene Polischuk Date: Thu, 1 Apr 2021 22:26:09 +0300 Subject: [PATCH] add varnish container with configurations --- compose/docker-compose.dev.yml | 4 + compose/docker-compose.yml | 24 ++- images/nginx/1.18/conf/default.conf | 37 ++++- images/varnish/4.0/Dockerfile | 7 + images/varnish/4.0/default.vcl | 241 ++++++++++++++++++++++++++++ images/varnish/4.0/varnish.vcl | 240 +++++++++++++++++++++++++++ 6 files changed, 544 insertions(+), 9 deletions(-) create mode 100644 images/varnish/4.0/Dockerfile create mode 100644 images/varnish/4.0/default.vcl create mode 100644 images/varnish/4.0/varnish.vcl diff --git a/compose/docker-compose.dev.yml b/compose/docker-compose.dev.yml index 577df4773..54b631338 100644 --- a/compose/docker-compose.dev.yml +++ b/compose/docker-compose.dev.yml @@ -32,6 +32,10 @@ services: #extra_hosts: # - "host.docker.internal:IP" + varnish: + extra_hosts: + - "host.docker.gateway:172.28.0.1" #host machine ip + mailhog: image: mailhog/mailhog ports: diff --git a/compose/docker-compose.yml b/compose/docker-compose.yml index 8ffc0e77c..8e6426a61 100644 --- a/compose/docker-compose.yml +++ b/compose/docker-compose.yml @@ -7,9 +7,9 @@ version: "3" services: app: - image: markoshust/magento-nginx:1.18-4 + build: images/nginx ports: - - "80:8000" + - "8080:8000" - "443:8443" links: - db @@ -20,6 +20,26 @@ services: - sockdata:/sock - ssldata:/etc/nginx/certs + varnish: + build: images/varnish + ports: + - "80:6081" + - "6085:6085" + depends_on: + - app + environment: + BACKENDS: "app" + BACKENDS_PORT: "8080" + DNS_ENABLED: "true" + BACKENDS_PROBE_INTERVAL: "3s" + BACKENDS_PROBE_TIMEOUT: "1s" + BACKENDS_PROBE_WINDOW: "3" + BACKENDS_PROBE_THRESHOLD: "2" + DASHBOARD_USER: "admin" + DASHBOARD_PASSWORD: "admin" + DASHBOARD_SERVERS: "varnish" + DASHBOARD_DNS_ENABLED: "true" + phpfpm: image: markoshust/magento-php:7.4-fpm-5 links: diff --git a/images/nginx/1.18/conf/default.conf b/images/nginx/1.18/conf/default.conf index 4ef3f1dfe..05a7253d2 100644 --- a/images/nginx/1.18/conf/default.conf +++ b/images/nginx/1.18/conf/default.conf @@ -2,22 +2,45 @@ upstream fastcgi_backend { server unix:/sock/docker.sock; } -server { - listen 8000; - return 301 https://$host$request_uri; +map $http_host $MAGE_RUN_CODE { + default base; + magento2.test base; } server { - listen [::]:8443 ssl http2 ipv6only=on; - listen 8443 ssl http2; + listen 8000; - ssl_certificate /etc/nginx/certs/nginx.crt; - ssl_certificate_key /etc/nginx/certs/nginx.key; + server_name magento2.test; set $MAGE_ROOT /var/www/html; + set $MAGE_RUN_TYPE website; fastcgi_buffer_size 64k; fastcgi_buffers 8 128k; include /var/www/html/nginx[.]conf; } + +server { + listen [::]:8443 ssl http2 ipv6only=on; + listen 8443 ssl http2; + + server_name magento2.test; + + ssl_certificate /etc/nginx/certs/nginx.crt; + ssl_certificate_key /etc/nginx/certs/nginx.key; + + location / { + proxy_pass http://varnish:6081; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto https; + proxy_set_header X-Forwarded-Port 8443; + proxy_set_header Host $host; + proxy_buffer_size 128k; + proxy_buffers 4 256k; + proxy_busy_buffers_size 256k; + fastcgi_buffer_size 64k; + fastcgi_buffers 8 128k; + } +} diff --git a/images/varnish/4.0/Dockerfile b/images/varnish/4.0/Dockerfile new file mode 100644 index 000000000..f6eb3be77 --- /dev/null +++ b/images/varnish/4.0/Dockerfile @@ -0,0 +1,7 @@ +#FROM ghcr.io/emgag/varnish:6.6.0 + +#COPY default.vcl /etc/varnish/default.vcl + +FROM eeacms/varnish + +COPY varnish.vcl /etc/varnish/conf.d/ diff --git a/images/varnish/4.0/default.vcl b/images/varnish/4.0/default.vcl new file mode 100644 index 000000000..fbe5bfb89 --- /dev/null +++ b/images/varnish/4.0/default.vcl @@ -0,0 +1,241 @@ +vcl 4.0; +import std; + +# The minimal Varnish version is 4.0 +# For SSL offloading, pass the following header in your proxy server or load balancer: 'X-Forwarded-Proto: https' + +backend default { + .host = "app"; + .port = "8000"; + .first_byte_timeout = 600s; + .probe = { + .url = "/health_check.php"; + .timeout = 2s; + .interval = 5s; + .window = 10; + .threshold = 5; + } +} + +acl purge { + "app"; + "varnish"; + "phpfpm"; + "host.docker.gateway"; +} + +sub vcl_recv { + if (req.method == "PURGE") { + if (client.ip !~ purge) { + return (synth(405, "Method not allowed")); + } + # To use the X-Pool header for purging varnish during automated deployments, make sure the X-Pool header + # has been added to the response in your backend server config. This is used, for example, by the + # capistrano-magento2 gem for purging old content from varnish during it's deploy routine. + if (!req.http.X-Magento-Tags-Pattern && !req.http.X-Pool) { + return (synth(400, "X-Magento-Tags-Pattern or X-Pool header required")); + } + if (req.http.X-Magento-Tags-Pattern) { + ban("obj.http.X-Magento-Tags ~ " + req.http.X-Magento-Tags-Pattern); + } + if (req.http.X-Pool) { + ban("obj.http.X-Pool ~ " + req.http.X-Pool); + } + return (synth(200, "Purged")); + } + + if (req.method != "GET" && + req.method != "HEAD" && + req.method != "PUT" && + req.method != "POST" && + req.method != "TRACE" && + req.method != "OPTIONS" && + req.method != "DELETE") { + /* Non-RFC2616 or CONNECT which is weird. */ + return (pipe); + } + + # We only deal with GET and HEAD by default + if (req.method != "GET" && req.method != "HEAD") { + return (pass); + } + + # Bypass shopping cart and checkout + if (req.url ~ "/checkout") { + return (pass); + } + + # Bypass health check requests + if (req.url ~ "/pub/health_check.php") { + return (pass); + } + + # Set initial grace period usage status + set req.http.grace = "none"; + + # normalize url in case of leading HTTP scheme and domain + set req.url = regsub(req.url, "^http[s]?://", ""); + + # collect all cookies + std.collect(req.http.Cookie); + + # Compression filter. See https://www.varnish-cache.org/trac/wiki/FAQ/Compression + if (req.http.Accept-Encoding) { + if (req.url ~ "\.(jpg|jpeg|png|gif|gz|tgz|bz2|tbz|mp3|ogg|swf|flv)$") { + # No point in compressing these + unset req.http.Accept-Encoding; + } elsif (req.http.Accept-Encoding ~ "gzip") { + set req.http.Accept-Encoding = "gzip"; + } elsif (req.http.Accept-Encoding ~ "deflate" && req.http.user-agent !~ "MSIE") { + set req.http.Accept-Encoding = "deflate"; + } else { + # unknown algorithm + unset req.http.Accept-Encoding; + } + } + + # Remove all marketing get parameters to minimize the cache objects + if (req.url ~ "(\?|&)(gclid|cx|ie|cof|siteurl|zanpid|origin|fbclid|mc_[a-z]+|utm_[a-z]+|_bta_[a-z]+)=") { + set req.url = regsuball(req.url, "(gclid|cx|ie|cof|siteurl|zanpid|origin|fbclid|mc_[a-z]+|utm_[a-z]+|_bta_[a-z]+)=[-_A-z0-9+()%.]+&?", ""); + set req.url = regsub(req.url, "[?|&]+$", ""); + } + + # Static files caching + if (req.url ~ "^/(pub/)?(media|static)/") { + # Static files should not be cached by default + return (pass); + + # But if you use a few locales and don't use CDN you can enable caching static files by commenting previous line (#return (pass);) and uncommenting next 3 lines + #unset req.http.Https; + #unset req.http.X-Forwarded-Proto; + #unset req.http.Cookie; + } + + # Authenticated GraphQL requests should not be cached by default + if (req.url ~ "/graphql" && req.http.Authorization ~ "^Bearer") { + return (pass); + } + + return (hash); +} + +sub vcl_hash { + if (req.http.cookie ~ "X-Magento-Vary=") { + hash_data(regsub(req.http.cookie, "^.*?X-Magento-Vary=([^;]+);*.*$", "\1")); + } + + if (req.url ~ "/graphql") { + call process_graphql_headers; + } + + # To make sure http users don't see ssl warning + if (req.http.X-Forwarded-Proto) { + hash_data(req.http.X-Forwarded-Proto); + } + +} + +sub process_graphql_headers { + if (req.http.Store) { + hash_data(req.http.Store); + } + if (req.http.Content-Currency) { + hash_data(req.http.Content-Currency); + } +} + +sub vcl_backend_response { + + set beresp.grace = 3d; + + if (beresp.http.content-type ~ "text") { + set beresp.do_esi = true; + } + + if (bereq.url ~ "\.js$" || beresp.http.content-type ~ "text") { + set beresp.do_gzip = true; + } + + if (beresp.http.X-Magento-Debug) { + set beresp.http.X-Magento-Cache-Control = beresp.http.Cache-Control; + } + + # cache only successfully responses and 404s + if (beresp.status != 200 && beresp.status != 404) { + set beresp.ttl = 0s; + set beresp.uncacheable = true; + return (deliver); + } elsif (beresp.http.Cache-Control ~ "private") { + set beresp.uncacheable = true; + set beresp.ttl = 86400s; + return (deliver); + } + + # validate if we need to cache it and prevent from setting cookie + if (beresp.ttl > 0s && (bereq.method == "GET" || bereq.method == "HEAD")) { + unset beresp.http.set-cookie; + } + + # If page is not cacheable then bypass varnish for 2 minutes as Hit-For-Pass + if (beresp.ttl <= 0s || + beresp.http.Surrogate-control ~ "no-store" || + (!beresp.http.Surrogate-Control && + beresp.http.Cache-Control ~ "no-cache|no-store") || + beresp.http.Vary == "*") { + # Mark as Hit-For-Pass for the next 2 minutes + set beresp.ttl = 120s; + set beresp.uncacheable = true; + } + + return (deliver); +} + +sub vcl_deliver { + if (resp.http.X-Magento-Debug) { + if (resp.http.x-varnish ~ " ") { + set resp.http.X-Magento-Cache-Debug = "HIT"; + set resp.http.Grace = req.http.grace; + } else { + set resp.http.X-Magento-Cache-Debug = "MISS"; + } + } else { + unset resp.http.Age; + } + + # Not letting browser to cache non-static files. + if (resp.http.Cache-Control !~ "private" && req.url !~ "^/(pub/)?(media|static)/") { + set resp.http.Pragma = "no-cache"; + set resp.http.Expires = "-1"; + set resp.http.Cache-Control = "no-store, no-cache, must-revalidate, max-age=0"; + } + + unset resp.http.X-Magento-Debug; + unset resp.http.X-Magento-Tags; + unset resp.http.X-Powered-By; + unset resp.http.Server; + unset resp.http.X-Varnish; + unset resp.http.Via; + unset resp.http.Link; +} + +sub vcl_hit { + if (obj.ttl >= 0s) { + # Hit within TTL period + return (deliver); + } + if (std.healthy(req.backend_hint)) { + if (obj.ttl + 300s > 0s) { + # Hit after TTL expiration, but within grace period + set req.http.grace = "normal (healthy server)"; + return (deliver); + } else { + # Hit after TTL and grace expiration + return (redirect); + } + } else { + # server is not healthy, retrieve from cache + set req.http.grace = "unlimited (unhealthy server)"; + return (deliver); + } +} + diff --git a/images/varnish/4.0/varnish.vcl b/images/varnish/4.0/varnish.vcl new file mode 100644 index 000000000..297125166 --- /dev/null +++ b/images/varnish/4.0/varnish.vcl @@ -0,0 +1,240 @@ +vcl 4.0; + +# The minimal Varnish version is 4.0 +# For SSL offloading, pass the following header in your proxy server or load balancer: 'X-Forwarded-Proto: https' + +backend default { + .host = "app"; + .port = "8000"; + .first_byte_timeout = 600s; + .probe = { + .url = "/health_check.php"; + .timeout = 2s; + .interval = 5s; + .window = 10; + .threshold = 5; + } +} + +acl purge { + "app"; + "varnish"; + "phpfpm"; + "host.docker.gateway"; +} + +sub vcl_recv { + if (req.method == "PURGE") { + if (client.ip !~ purge) { + return (synth(405, "Method not allowed")); + } + # To use the X-Pool header for purging varnish during automated deployments, make sure the X-Pool header + # has been added to the response in your backend server config. This is used, for example, by the + # capistrano-magento2 gem for purging old content from varnish during it's deploy routine. + if (!req.http.X-Magento-Tags-Pattern && !req.http.X-Pool) { + return (synth(400, "X-Magento-Tags-Pattern or X-Pool header required")); + } + if (req.http.X-Magento-Tags-Pattern) { + ban("obj.http.X-Magento-Tags ~ " + req.http.X-Magento-Tags-Pattern); + } + if (req.http.X-Pool) { + ban("obj.http.X-Pool ~ " + req.http.X-Pool); + } + return (synth(200, "Purged")); + } + + if (req.method != "GET" && + req.method != "HEAD" && + req.method != "PUT" && + req.method != "POST" && + req.method != "TRACE" && + req.method != "OPTIONS" && + req.method != "DELETE") { + /* Non-RFC2616 or CONNECT which is weird. */ + return (pipe); + } + + # We only deal with GET and HEAD by default + if (req.method != "GET" && req.method != "HEAD") { + return (pass); + } + + # Bypass shopping cart and checkout + if (req.url ~ "/checkout") { + return (pass); + } + + # Bypass health check requests + if (req.url ~ "/pub/health_check.php") { + return (pass); + } + + # Set initial grace period usage status + set req.http.grace = "none"; + + # normalize url in case of leading HTTP scheme and domain + set req.url = regsub(req.url, "^http[s]?://", ""); + + # collect all cookies + std.collect(req.http.Cookie); + + # Compression filter. See https://www.varnish-cache.org/trac/wiki/FAQ/Compression + if (req.http.Accept-Encoding) { + if (req.url ~ "\.(jpg|jpeg|png|gif|gz|tgz|bz2|tbz|mp3|ogg|swf|flv)$") { + # No point in compressing these + unset req.http.Accept-Encoding; + } elsif (req.http.Accept-Encoding ~ "gzip") { + set req.http.Accept-Encoding = "gzip"; + } elsif (req.http.Accept-Encoding ~ "deflate" && req.http.user-agent !~ "MSIE") { + set req.http.Accept-Encoding = "deflate"; + } else { + # unknown algorithm + unset req.http.Accept-Encoding; + } + } + + # Remove all marketing get parameters to minimize the cache objects + if (req.url ~ "(\?|&)(gclid|cx|ie|cof|siteurl|zanpid|origin|fbclid|mc_[a-z]+|utm_[a-z]+|_bta_[a-z]+)=") { + set req.url = regsuball(req.url, "(gclid|cx|ie|cof|siteurl|zanpid|origin|fbclid|mc_[a-z]+|utm_[a-z]+|_bta_[a-z]+)=[-_A-z0-9+()%.]+&?", ""); + set req.url = regsub(req.url, "[?|&]+$", ""); + } + + # Static files caching + if (req.url ~ "^/(pub/)?(media|static)/") { + # Static files should not be cached by default + return (pass); + + # But if you use a few locales and don't use CDN you can enable caching static files by commenting previous line (#return (pass);) and uncommenting next 3 lines + #unset req.http.Https; + #unset req.http.X-Forwarded-Proto; + #unset req.http.Cookie; + } + + # Authenticated GraphQL requests should not be cached by default + if (req.url ~ "/graphql" && req.http.Authorization ~ "^Bearer") { + return (pass); + } + + return (hash); +} + +sub vcl_hash { + if (req.http.cookie ~ "X-Magento-Vary=") { + hash_data(regsub(req.http.cookie, "^.*?X-Magento-Vary=([^;]+);*.*$", "\1")); + } + + if (req.url ~ "/graphql") { + call process_graphql_headers; + } + + # To make sure http users don't see ssl warning + if (req.http.X-Forwarded-Proto) { + hash_data(req.http.X-Forwarded-Proto); + } + +} + +sub process_graphql_headers { + if (req.http.Store) { + hash_data(req.http.Store); + } + if (req.http.Content-Currency) { + hash_data(req.http.Content-Currency); + } +} + +sub vcl_backend_response { + + set beresp.grace = 3d; + + if (beresp.http.content-type ~ "text") { + set beresp.do_esi = true; + } + + if (bereq.url ~ "\.js$" || beresp.http.content-type ~ "text") { + set beresp.do_gzip = true; + } + + if (beresp.http.X-Magento-Debug) { + set beresp.http.X-Magento-Cache-Control = beresp.http.Cache-Control; + } + + # cache only successfully responses and 404s + if (beresp.status != 200 && beresp.status != 404) { + set beresp.ttl = 0s; + set beresp.uncacheable = true; + return (deliver); + } elsif (beresp.http.Cache-Control ~ "private") { + set beresp.uncacheable = true; + set beresp.ttl = 86400s; + return (deliver); + } + + # validate if we need to cache it and prevent from setting cookie + if (beresp.ttl > 0s && (bereq.method == "GET" || bereq.method == "HEAD")) { + unset beresp.http.set-cookie; + } + + # If page is not cacheable then bypass varnish for 2 minutes as Hit-For-Pass + if (beresp.ttl <= 0s || + beresp.http.Surrogate-control ~ "no-store" || + (!beresp.http.Surrogate-Control && + beresp.http.Cache-Control ~ "no-cache|no-store") || + beresp.http.Vary == "*") { + # Mark as Hit-For-Pass for the next 2 minutes + set beresp.ttl = 120s; + set beresp.uncacheable = true; + } + + return (deliver); +} + +sub vcl_deliver { + if (resp.http.X-Magento-Debug) { + if (resp.http.x-varnish ~ " ") { + set resp.http.X-Magento-Cache-Debug = "HIT"; + set resp.http.Grace = req.http.grace; + } else { + set resp.http.X-Magento-Cache-Debug = "MISS"; + } + } else { + unset resp.http.Age; + } + + # Not letting browser to cache non-static files. + if (resp.http.Cache-Control !~ "private" && req.url !~ "^/(pub/)?(media|static)/") { + set resp.http.Pragma = "no-cache"; + set resp.http.Expires = "-1"; + set resp.http.Cache-Control = "no-store, no-cache, must-revalidate, max-age=0"; + } + + unset resp.http.X-Magento-Debug; + unset resp.http.X-Magento-Tags; + unset resp.http.X-Powered-By; + unset resp.http.Server; + unset resp.http.X-Varnish; + unset resp.http.Via; + unset resp.http.Link; +} + +sub vcl_hit { + if (obj.ttl >= 0s) { + # Hit within TTL period + return (deliver); + } + if (std.healthy(req.backend_hint)) { + if (obj.ttl + 300s > 0s) { + # Hit after TTL expiration, but within grace period + set req.http.grace = "normal (healthy server)"; + return (deliver); + } else { + # Hit after TTL and grace expiration + return (fetch); + } + } else { + # server is not healthy, retrieve from cache + set req.http.grace = "unlimited (unhealthy server)"; + return (deliver); + } +} +