From 35015a46fe9480e2ceba2417c44fdbc87866a215 Mon Sep 17 00:00:00 2001 From: Yoav Date: Sun, 18 Aug 2019 09:48:27 +0200 Subject: [PATCH 1/7] refs #54 nginx-proxy caching * proof-of-concept (WIP) using internal nginx proxy caching instead of relying on thumbor's result storage and try_files --- nginx-proxy/Dockerfile | 1 - nginx-proxy/imgpath.js | 5 -- nginx-proxy/nginx.tmpl | 64 ++----------------- .../docker-compose/simple/docker-compose.yml | 13 +--- 4 files changed, 8 insertions(+), 75 deletions(-) delete mode 100644 nginx-proxy/imgpath.js diff --git a/nginx-proxy/Dockerfile b/nginx-proxy/Dockerfile index 4bcd1086..a402b06e 100644 --- a/nginx-proxy/Dockerfile +++ b/nginx-proxy/Dockerfile @@ -3,5 +3,4 @@ FROM jwilder/nginx-proxy LABEL maintainer="MinimalCompact" COPY ./nginx.conf /etc/nginx/ -COPY ./imgpath.js /etc/nginx/ COPY ./nginx.tmpl /app/ diff --git a/nginx-proxy/imgpath.js b/nginx-proxy/imgpath.js deleted file mode 100644 index 47950c53..00000000 --- a/nginx-proxy/imgpath.js +++ /dev/null @@ -1,5 +0,0 @@ -function file_path(r) { - var uri = decodeURI(r.uri); - var digest = require('crypto').createHash('sha1').update(uri).digest('hex'); - return digest.slice(NaN, 2) + "/" + digest.slice(2, 4) + "/" + digest.slice(4); -} diff --git a/nginx-proxy/nginx.tmpl b/nginx-proxy/nginx.tmpl index be0de7a4..8cecb7cf 100644 --- a/nginx-proxy/nginx.tmpl +++ b/nginx-proxy/nginx.tmpl @@ -1,35 +1,5 @@ {{ $CurrentContainer := where $ "ID" .Docker.CurrentContainerID | first }} -{{ define "thumbor-include" }} - merge_slashes off; - js_include imgpath.js; - js_set $file_path file_path; -{{ end }} - -{{ define "thumbor-map" }} - map $http_accept $webp_prefix { - default "/default"; - "~*webp*" "/auto_webp"; - } - map $request_uri $format { - default ""; - "~*format\(jpeg\)" "image/jpeg"; - "~*format\(jpg\)" "image/jpeg"; - "~*format\(png\)" "image/png"; - "~*format\(gif\)" "image/gif"; - "~*format\(webp\)" "image/webp"; - "~*\.jpeg" "image/jpeg"; - "~*\.jpg" "image/jpeg"; - "~*\.png" "image/png"; - "~*\.gif" "image/gif"; - "~*\.webp" "image/webp"; - } - map $http_accept $auto_webp_content_type { - default $format; - "~*webp*" "image/webp"; - } -{{ end }} - {{ define "cors" }} add_header 'Access-Control-Allow-Origin' '{{ or .CorsOrigin "*" }}'; add_header 'Access-Control-Allow-Credentials' 'true'; @@ -40,42 +10,23 @@ {{ define "thumbor" }} {{ $proto := .Proto }} {{ $upstream_name := .UpstreamName }} - {{ $autoWebp := (first (groupByKeys .Containers "Env.AUTO_WEBP")) }} {{ $allowCors := (first (groupByKeys .Containers "Env.THUMBOR_ALLOW_CORS")) }} {{ $corsOrigin := (first (groupByKeys .Containers "Env.CORS_ALLOW_ORIGIN")) }} - location ~* "^/(?..)(?..)(.+)?$" { - root /data/result_storage; - expires 1M; - + location ~* "^/(.+)?$" { {{ if or $allowCors $corsOrigin }} {{ template "cors" (dict "CorsOrigin" $corsOrigin) }} {{ end }} - {{ if $autoWebp }} - try_files $webp_prefix/$file_path =404; - add_header Vary Accept; - #if auto webp is on then content-type is based on accept header - set $format $auto_webp_content_type; - {{ else }} - try_files /default/$file_path =404; - {{ end }} - # if $format is empty (default) then this is ignored - add_header Content-Type $format; - error_page 404 = @fetch; + + proxy_cache thumbor; + proxy_buffering on; + proxy_pass {{ trim $proto }}://{{ trim $upstream_name }}; } location = /healthcheck { proxy_pass {{ trim .Proto }}://{{ trim $upstream_name }}; access_log off; } - - location @fetch { - internal; - {{ if or $allowCors $corsOrigin }} - {{ template "cors" (dict "CorsOrigin" $corsOrigin) }} - {{ end }} - proxy_pass {{ trim $proto }}://{{ trim $upstream_name }}; - } {{ end }} {{ define "upstream" }} @@ -147,9 +98,6 @@ access_log off; resolver {{ $.Env.RESOLVERS }}; {{ end }} -{{ template "thumbor-include" }} -{{ template "thumbor-map" }} - {{ if (exists "/etc/nginx/proxy.conf") }} include /etc/nginx/proxy.conf; {{ else }} @@ -164,7 +112,7 @@ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $proxy_x_forwarded_proto; proxy_set_header X-Forwarded-Ssl $proxy_x_forwarded_ssl; proxy_set_header X-Forwarded-Port $proxy_x_forwarded_port; - +proxy_cache_path /var/cache/nginx levels=1:2 keys_zone=thumbor:100m inactive=300m use_temp_path=off; # Mitigate httpoxy attack (see README for details) proxy_set_header Proxy ""; {{ end }} diff --git a/recipes/docker-compose/simple/docker-compose.yml b/recipes/docker-compose/simple/docker-compose.yml index a4ada777..f2b5182b 100644 --- a/recipes/docker-compose/simple/docker-compose.yml +++ b/recipes/docker-compose/simple/docker-compose.yml @@ -15,15 +15,11 @@ services: - CORS_ALLOW_ORIGIN=* # returns a webp image if browser Accept headers match - AUTO_WEBP=True - # Basic thumbor setup to cache both original images and results (after manipulation) - # nginx-proxy would detect if there's a cached version and serve it for you automatically + # nginx-proxy does caching automatically, so no need to store the result storage cache # (this greatly speeds up and saves on CPU) - - RESULT_STORAGE=thumbor.result_storages.file_storage + - RESULT_STORAGE=thumbor.result_storages.no_storage - RESULT_STORAGE_STORES_UNSAFE=True - STORAGE=thumbor.storages.file_storage - volumes: - # mounting a /data folder to store cached images - - ./data:/data restart: always networks: - app @@ -38,17 +34,12 @@ services: # this is essential for nginx-proxy to detect docker containers, scaling etc # see https://github.com/jwilder/nginx-proxy - /var/run/docker.sock:/tmp/docker.sock:ro - # mapping the same data folder, to allow nginx-proxy to fetch images from cache - - ./data:/data ports: - "80:80" - "443:443" restart: always networks: - app -volumes: - data: - driver: local networks: app: driver: bridge From c0cbd812b8c0d32120cf56ff86aabf4403ec92d4 Mon Sep 17 00:00:00 2001 From: Yoav Date: Sun, 25 Aug 2019 10:09:42 +0200 Subject: [PATCH 2/7] adding new image: thumbor-nginx-proxy-cache * the new nginx-proxy-cache will replace nginx-proxy with built-in proxy caching by nginx instead of mapping to thumbor's data folder * refreshed nginx.tmpl from upstream (jwilder/nginx-proxy) * updated template to add thumbor + proxy caching with minimal control using environment variables (for further customization, users can supply a proxy.conf file) * updated tests: added proxy caching tests + removed some tests that are unnecessary or difficult to do with the built-in proxy caching (for example, overriding the cached data is somehow detected by nginx) * updated recipes to reflect the new setup (no need for result storage caching in thumbor, and no data volume) * TODO: revert changes to nginx-proxy and add deprecation warning --- build | 9 +- nginx-proxy-cache/Dockerfile | 5 + nginx-proxy-cache/nginx.tmpl | 402 ++++++++++++++++++ push | 6 +- .../letsencrypt/docker-compose.yml | 13 +- .../remotecv/docker-compose.yml | 19 +- .../docker-compose/simple/docker-compose.yml | 6 +- .../docker-compose/swarm/docker-compose.yml | 9 +- tests/nginx-proxy-autowebp.bats | 13 +- tests/nginx-proxy-cache.bats | 38 ++ tests/nginx-proxy.bats | 38 +- tests/setup/basic/docker-compose.yml | 14 +- 12 files changed, 503 insertions(+), 69 deletions(-) create mode 100644 nginx-proxy-cache/Dockerfile create mode 100644 nginx-proxy-cache/nginx.tmpl create mode 100644 tests/nginx-proxy-cache.bats diff --git a/build b/build index e37f68d1..21304232 100755 --- a/build +++ b/build @@ -26,13 +26,20 @@ docker tag minimalcompact/thumbor-simd-avx2 minimalcompact/thumbor:$THUMBOR_VERS echo "--> TAGGING minimalcompact/thumbor:latest-simd-avx2" docker tag minimalcompact/thumbor-simd-avx2 minimalcompact/thumbor:latest-simd-avx2 -echo "--> BUILDING minimalcompact/thumbor-nginx-proxy" +echo "--> BUILDING minimalcompact/thumbor-nginx-proxy (DEPRECATED)" docker build --pull -f nginx-proxy/Dockerfile -t minimalcompact/thumbor-nginx-proxy nginx-proxy/ echo "--> TAGGING minimalcompact/thumbor-nginx-proxy:$THUMBOR_VERSION" docker tag minimalcompact/thumbor-nginx-proxy minimalcompact/thumbor-nginx-proxy:$THUMBOR_VERSION echo "--> TAGGING minimalcompact/thumbor-nginx-proxy:latest" docker tag minimalcompact/thumbor-nginx-proxy minimalcompact/thumbor-nginx-proxy:latest +echo "--> BUILDING minimalcompact/thumbor-nginx-proxy-cache" +docker build --pull -f nginx-proxy-cache/Dockerfile -t minimalcompact/thumbor-nginx-proxy-cache nginx-proxy-cache/ +echo "--> TAGGING minimalcompact/thumbor-nginx-proxy-cache:$THUMBOR_VERSION" +docker tag minimalcompact/thumbor-nginx-proxy-cache minimalcompact/thumbor-nginx-proxy-cache:$THUMBOR_VERSION +echo "--> TAGGING minimalcompact/thumbor-nginx-proxy-cache:latest" +docker tag minimalcompact/thumbor-nginx-proxy-cache minimalcompact/thumbor-nginx-proxy-cache:latest + echo "--> BUILDING minimalcompact/remotecv" docker build --build-arg THUMBOR_TAG=latest -f remotecv/Dockerfile -t minimalcompact/remotecv remotecv/ echo "--> TAGGING minimalcompact/remotecv:$THUMBOR_VERSION" diff --git a/nginx-proxy-cache/Dockerfile b/nginx-proxy-cache/Dockerfile new file mode 100644 index 00000000..5ee8ccb5 --- /dev/null +++ b/nginx-proxy-cache/Dockerfile @@ -0,0 +1,5 @@ +FROM jwilder/nginx-proxy + +LABEL maintainer="MinimalCompact" + +COPY ./nginx.tmpl /app/ diff --git a/nginx-proxy-cache/nginx.tmpl b/nginx-proxy-cache/nginx.tmpl new file mode 100644 index 00000000..a828cb72 --- /dev/null +++ b/nginx-proxy-cache/nginx.tmpl @@ -0,0 +1,402 @@ +{{ $CurrentContainer := where $ "ID" .Docker.CurrentContainerID | first }} + +{{ define "cors" }} + add_header 'Access-Control-Allow-Origin' '{{ or .CorsOrigin "*" }}'; + add_header 'Access-Control-Allow-Credentials' 'true'; + add_header 'Access-Control-Allow-Methods' 'GET, POST, PUT, DELETE, OPTIONS'; + add_header 'Access-Control-Allow-Headers' 'Accept,Authorization,Cache-Control,Content-Type,DNT,If-Modified-Since,Keep-Alive,Origin,User-Agent,X-Mx-ReqToken,X-Requested-With'; +{{ end }} + +{{ define "thumbor" }} + {{ $proto := .Proto }} + {{ $upstream_name := .UpstreamName }} + {{ $allowCors := (first (groupByKeys .Containers "Env.THUMBOR_ALLOW_CORS")) }} + {{ $corsOrigin := (first (groupByKeys .Containers "Env.CORS_ALLOW_ORIGIN")) }} + + location ~* "^/(.+)?$" { + {{ if or $allowCors $corsOrigin }} + {{ template "cors" (dict "CorsOrigin" $corsOrigin) }} + {{ end }} + + proxy_cache thumbor; + proxy_buffering on; + proxy_pass {{ trim $proto }}://{{ trim $upstream_name }}; + } + + location = /healthcheck { + proxy_pass {{ trim .Proto }}://{{ trim $upstream_name }}; + access_log off; + } +{{ end }} + +{{ define "thumbor-caching" }} + {{ $proxyCacheSize := or .ProxyCacheSize "100m" }} + {{ $proxyCacheInactive := or .ProxyCacheInactive "300m" }} + proxy_cache_path /var/cache/nginx levels=1:2 keys_zone=thumbor:{{ $proxyCacheSize }} inactive={{ $proxyCacheInactive }} use_temp_path=off; +{{ end }} + +{{ define "upstream" }} + {{ if .Address }} + {{/* If we got the containers from swarm and this container's port is published to host, use host IP:PORT */}} + {{ if and .Container.Node.ID .Address.HostPort }} + # {{ .Container.Node.Name }}/{{ .Container.Name }} + server {{ .Container.Node.Address.IP }}:{{ .Address.HostPort }}; + {{/* If there is no swarm node or the port is not published on host, use container's IP:PORT */}} + {{ else if .Network }} + # {{ .Container.Name }} + server {{ .Network.IP }}:{{ .Address.Port }}; + {{ end }} + {{ else if .Network }} + # {{ .Container.Name }} + {{ if .Network.IP }} + server {{ .Network.IP }} down; + {{ else }} + server 127.0.0.1 down; + {{ end }} + {{ end }} + +{{ end }} + +# If we receive X-Forwarded-Proto, pass it through; otherwise, pass along the +# scheme used to connect to this server +map $http_x_forwarded_proto $proxy_x_forwarded_proto { + default $http_x_forwarded_proto; + '' $scheme; +} + +# If we receive X-Forwarded-Port, pass it through; otherwise, pass along the +# server port the client connected to +map $http_x_forwarded_port $proxy_x_forwarded_port { + default $http_x_forwarded_port; + '' $server_port; +} + +# If we receive Upgrade, set Connection to "upgrade"; otherwise, delete any +# Connection header that may have been passed to this server +map $http_upgrade $proxy_connection { + default upgrade; + '' close; +} + +# Apply fix for very long server names +server_names_hash_bucket_size 128; + +# Default dhparam +{{ if (exists "/etc/nginx/dhparam/dhparam.pem") }} +ssl_dhparam /etc/nginx/dhparam/dhparam.pem; +{{ end }} + +# Set appropriate X-Forwarded-Ssl header +map $scheme $proxy_x_forwarded_ssl { + default off; + https on; +} + +gzip_types text/plain text/css application/javascript application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript; + +log_format vhost '$host $remote_addr - $remote_user [$time_local] ' + '"$request" $status $body_bytes_sent ' + '"$http_referer" "$http_user_agent"'; + +access_log off; + +{{ if $.Env.RESOLVERS }} +resolver {{ $.Env.RESOLVERS }}; +{{ end }} + +{{ if (exists "/etc/nginx/proxy.conf") }} +include /etc/nginx/proxy.conf; +{{ else }} +# HTTP 1.1 support +proxy_http_version 1.1; +proxy_buffering off; +proxy_set_header Host $http_host; +proxy_set_header Upgrade $http_upgrade; +proxy_set_header Connection $proxy_connection; +proxy_set_header X-Real-IP $remote_addr; +proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; +proxy_set_header X-Forwarded-Proto $proxy_x_forwarded_proto; +proxy_set_header X-Forwarded-Ssl $proxy_x_forwarded_ssl; +proxy_set_header X-Forwarded-Port $proxy_x_forwarded_port; + +{{ template "thumbor-caching" (dict "ProxyCacheSize" ($.Env.PROXY_CACHE_SIZE) "ProxyCacheInactive" ($.Env.PROXY_CACHE_INACTIVE)) }} + +# Mitigate httpoxy attack (see README for details) +proxy_set_header Proxy ""; +{{ end }} + +{{ $enable_ipv6 := eq (or ($.Env.ENABLE_IPV6) "") "true" }} +server { + server_name _; # This is just an invalid value which will never trigger on a real hostname. + listen 80; + {{ if $enable_ipv6 }} + listen [::]:80; + {{ end }} + access_log /var/log/nginx/access.log vhost; + return 503; +} + +{{ if (and (exists "/etc/nginx/certs/default.crt") (exists "/etc/nginx/certs/default.key")) }} +server { + server_name _; # This is just an invalid value which will never trigger on a real hostname. + listen 443 ssl http2; + {{ if $enable_ipv6 }} + listen [::]:443 ssl http2; + {{ end }} + access_log /var/log/nginx/access.log vhost; + return 503; + + ssl_session_tickets off; + ssl_certificate /etc/nginx/certs/default.crt; + ssl_certificate_key /etc/nginx/certs/default.key; +} +{{ end }} + +{{ range $host, $containers := groupByMulti $ "Env.VIRTUAL_HOST" "," }} + +{{ $host := trim $host }} +{{ $is_regexp := hasPrefix "~" $host }} +{{ $upstream_name := when $is_regexp (sha1 $host) $host }} + +# {{ $host }} +upstream {{ $upstream_name }} { + +{{ range $container := $containers }} + {{ $addrLen := len $container.Addresses }} + + {{ range $knownNetwork := $CurrentContainer.Networks }} + {{ range $containerNetwork := $container.Networks }} + {{ if (and (ne $containerNetwork.Name "ingress") (or (eq $knownNetwork.Name $containerNetwork.Name) (eq $knownNetwork.Name "host"))) }} + ## Can be connected with "{{ $containerNetwork.Name }}" network + + {{/* If only 1 port exposed, use that */}} + {{ if eq $addrLen 1 }} + {{ $address := index $container.Addresses 0 }} + {{ template "upstream" (dict "Container" $container "Address" $address "Network" $containerNetwork) }} + {{/* If more than one port exposed, use the one matching VIRTUAL_PORT env var, falling back to standard web port 80 */}} + {{ else }} + {{ $port := coalesce $container.Env.VIRTUAL_PORT "80" }} + {{ $address := where $container.Addresses "Port" $port | first }} + {{ template "upstream" (dict "Container" $container "Address" $address "Network" $containerNetwork) }} + {{ end }} + {{ else }} + # Cannot connect to network of this container + server 127.0.0.1 down; + {{ end }} + {{ end }} + {{ end }} +{{ end }} +} + +{{ $default_host := or ($.Env.DEFAULT_HOST) "" }} +{{ $default_server := index (dict $host "" $default_host "default_server") $host }} + +{{/* Get the VIRTUAL_PROTO defined by containers w/ the same vhost, falling back to "http" */}} +{{ $proto := trim (or (first (groupByKeys $containers "Env.VIRTUAL_PROTO")) "http") }} + +{{/* Get the NETWORK_ACCESS defined by containers w/ the same vhost, falling back to "external" */}} +{{ $network_tag := or (first (groupByKeys $containers "Env.NETWORK_ACCESS")) "external" }} + +{{/* Get the HTTPS_METHOD defined by containers w/ the same vhost, falling back to "redirect" */}} +{{ $https_method := or (first (groupByKeys $containers "Env.HTTPS_METHOD")) "redirect" }} + +{{/* Get the SSL_POLICY defined by containers w/ the same vhost, falling back to "Mozilla-Intermediate" */}} +{{ $ssl_policy := or (first (groupByKeys $containers "Env.SSL_POLICY")) "Mozilla-Intermediate" }} + +{{/* Get the HSTS defined by containers w/ the same vhost, falling back to "max-age=31536000" */}} +{{ $hsts := or (first (groupByKeys $containers "Env.HSTS")) "max-age=31536000" }} + +{{/* Get the VIRTUAL_ROOT By containers w/ use fastcgi root */}} +{{ $vhost_root := or (first (groupByKeys $containers "Env.VIRTUAL_ROOT")) "/var/www/public" }} + + +{{/* Get the first cert name defined by containers w/ the same vhost */}} +{{ $certName := (first (groupByKeys $containers "Env.CERT_NAME")) }} + +{{/* Get the best matching cert by name for the vhost. */}} +{{ $vhostCert := (closest (dir "/etc/nginx/certs") (printf "%s.crt" $host))}} + +{{/* vhostCert is actually a filename so remove any suffixes since they are added later */}} +{{ $vhostCert := trimSuffix ".crt" $vhostCert }} +{{ $vhostCert := trimSuffix ".key" $vhostCert }} + +{{/* Use the cert specified on the container or fallback to the best vhost match */}} +{{ $cert := (coalesce $certName $vhostCert) }} + +{{ $is_https := (and (ne $https_method "nohttps") (ne $cert "") (exists (printf "/etc/nginx/certs/%s.crt" $cert)) (exists (printf "/etc/nginx/certs/%s.key" $cert))) }} + +{{ if $is_https }} + +{{ if eq $https_method "redirect" }} +server { + server_name {{ $host }}; + listen 80 {{ $default_server }}; + {{ if $enable_ipv6 }} + listen [::]:80 {{ $default_server }}; + {{ end }} + access_log /var/log/nginx/access.log vhost; + return 301 https://$host$request_uri; +} +{{ end }} + +server { + server_name {{ $host }}; + listen 443 ssl http2 {{ $default_server }}; + {{ if $enable_ipv6 }} + listen [::]:443 ssl http2 {{ $default_server }}; + {{ end }} + access_log /var/log/nginx/access.log vhost; + + {{ if eq $network_tag "internal" }} + # Only allow traffic from internal clients + include /etc/nginx/network_internal.conf; + {{ end }} + + {{ if eq $ssl_policy "Mozilla-Modern" }} + ssl_protocols TLSv1.2 TLSv1.3; + ssl_ciphers 'ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256'; + {{ else if eq $ssl_policy "Mozilla-Intermediate" }} + ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3; + ssl_ciphers 'ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:!DSS'; + {{ else if eq $ssl_policy "Mozilla-Old" }} + ssl_protocols SSLv3 TLSv1 TLSv1.1 TLSv1.2 TLSv1.3; + ssl_ciphers 'ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:ECDHE-RSA-DES-CBC3-SHA:ECDHE-ECDSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:DES-CBC3-SHA:HIGH:SEED:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!RSAPSK:!aDH:!aECDH:!EDH-DSS-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA:!SRP'; + {{ else if eq $ssl_policy "AWS-TLS-1-2-2017-01" }} + ssl_protocols TLSv1.2 TLSv1.3; + ssl_ciphers 'ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES128-SHA256:AES256-GCM-SHA384:AES256-SHA256'; + {{ else if eq $ssl_policy "AWS-TLS-1-1-2017-01" }} + ssl_protocols TLSv1.1 TLSv1.2 TLSv1.3; + ssl_ciphers 'ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:AES128-GCM-SHA256:AES128-SHA256:AES128-SHA:AES256-GCM-SHA384:AES256-SHA256:AES256-SHA'; + {{ else if eq $ssl_policy "AWS-2016-08" }} + ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3; + ssl_ciphers 'ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:AES128-GCM-SHA256:AES128-SHA256:AES128-SHA:AES256-GCM-SHA384:AES256-SHA256:AES256-SHA'; + {{ else if eq $ssl_policy "AWS-2015-05" }} + ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3; + ssl_ciphers 'ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:AES128-GCM-SHA256:AES128-SHA256:AES128-SHA:AES256-GCM-SHA384:AES256-SHA256:AES256-SHA:DES-CBC3-SHA'; + {{ else if eq $ssl_policy "AWS-2015-03" }} + ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3; + ssl_ciphers 'ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES128-SHA:DHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:AES128-GCM-SHA256:AES128-SHA256:AES128-SHA:AES256-GCM-SHA384:AES256-SHA256:AES256-SHA:DHE-DSS-AES128-SHA:DES-CBC3-SHA'; + {{ else if eq $ssl_policy "AWS-2015-02" }} + ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3; + ssl_ciphers 'ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES128-SHA:DHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:AES128-GCM-SHA256:AES128-SHA256:AES128-SHA:AES256-GCM-SHA384:AES256-SHA256:AES256-SHA:DHE-DSS-AES128-SHA'; + {{ end }} + + ssl_prefer_server_ciphers on; + ssl_session_timeout 5m; + ssl_session_cache shared:SSL:50m; + ssl_session_tickets off; + + ssl_certificate /etc/nginx/certs/{{ (printf "%s.crt" $cert) }}; + ssl_certificate_key /etc/nginx/certs/{{ (printf "%s.key" $cert) }}; + + {{ if (exists (printf "/etc/nginx/certs/%s.dhparam.pem" $cert)) }} + ssl_dhparam {{ printf "/etc/nginx/certs/%s.dhparam.pem" $cert }}; + {{ end }} + + {{ if (exists (printf "/etc/nginx/certs/%s.chain.pem" $cert)) }} + ssl_stapling on; + ssl_stapling_verify on; + ssl_trusted_certificate {{ printf "/etc/nginx/certs/%s.chain.pem" $cert }}; + {{ end }} + + {{ if (not (or (eq $https_method "noredirect") (eq $hsts "off"))) }} + add_header Strict-Transport-Security "{{ trim $hsts }}" always; + {{ end }} + + {{ if (exists (printf "/etc/nginx/vhost.d/%s" $host)) }} + include {{ printf "/etc/nginx/vhost.d/%s" $host }}; + {{ else if (exists "/etc/nginx/vhost.d/default") }} + include /etc/nginx/vhost.d/default; + {{ end }} + + {{ template "thumbor" (dict "Proto" $proto "UpstreamName" $upstream_name "Containers" $containers) }} + + location / { + {{ if eq $proto "uwsgi" }} + include uwsgi_params; + uwsgi_pass {{ trim $proto }}://{{ trim $upstream_name }}; + {{ else if eq $proto "fastcgi" }} + root {{ trim $vhost_root }}; + include fastcgi.conf; + fastcgi_pass {{ trim $upstream_name }}; + {{ else }} + proxy_pass {{ trim $proto }}://{{ trim $upstream_name }}; + {{ end }} + + {{ if (exists (printf "/etc/nginx/htpasswd/%s" $host)) }} + auth_basic "Restricted {{ $host }}"; + auth_basic_user_file {{ (printf "/etc/nginx/htpasswd/%s" $host) }}; + {{ end }} + {{ if (exists (printf "/etc/nginx/vhost.d/%s_location" $host)) }} + include {{ printf "/etc/nginx/vhost.d/%s_location" $host}}; + {{ else if (exists "/etc/nginx/vhost.d/default_location") }} + include /etc/nginx/vhost.d/default_location; + {{ end }} + } +} + +{{ end }} + +{{ if or (not $is_https) (eq $https_method "noredirect") }} + +server { + server_name {{ $host }}; + listen 80 {{ $default_server }}; + {{ if $enable_ipv6 }} + listen [::]:80 {{ $default_server }}; + {{ end }} + access_log /var/log/nginx/access.log vhost; + + {{ if eq $network_tag "internal" }} + # Only allow traffic from internal clients + include /etc/nginx/network_internal.conf; + {{ end }} + + {{ if (exists (printf "/etc/nginx/vhost.d/%s" $host)) }} + include {{ printf "/etc/nginx/vhost.d/%s" $host }}; + {{ else if (exists "/etc/nginx/vhost.d/default") }} + include /etc/nginx/vhost.d/default; + {{ end }} + + {{ template "thumbor" (dict "Proto" $proto "UpstreamName" $upstream_name "Containers" $containers) }} + + location / { + {{ if eq $proto "uwsgi" }} + include uwsgi_params; + uwsgi_pass {{ trim $proto }}://{{ trim $upstream_name }}; + {{ else if eq $proto "fastcgi" }} + root {{ trim $vhost_root }}; + include fastcgi.conf; + fastcgi_pass {{ trim $upstream_name }}; + {{ else }} + proxy_pass {{ trim $proto }}://{{ trim $upstream_name }}; + {{ end }} + {{ if (exists (printf "/etc/nginx/htpasswd/%s" $host)) }} + auth_basic "Restricted {{ $host }}"; + auth_basic_user_file {{ (printf "/etc/nginx/htpasswd/%s" $host) }}; + {{ end }} + {{ if (exists (printf "/etc/nginx/vhost.d/%s_location" $host)) }} + include {{ printf "/etc/nginx/vhost.d/%s_location" $host}}; + {{ else if (exists "/etc/nginx/vhost.d/default_location") }} + include /etc/nginx/vhost.d/default_location; + {{ end }} + } +} + +{{ if (and (not $is_https) (exists "/etc/nginx/certs/default.crt") (exists "/etc/nginx/certs/default.key")) }} +server { + server_name {{ $host }}; + listen 443 ssl http2 {{ $default_server }}; + {{ if $enable_ipv6 }} + listen [::]:443 ssl http2 {{ $default_server }}; + {{ end }} + access_log /var/log/nginx/access.log vhost; + return 500; + + ssl_certificate /etc/nginx/certs/default.crt; + ssl_certificate_key /etc/nginx/certs/default.key; +} +{{ end }} + +{{ end }} +{{ end }} diff --git a/push b/push index d884221a..bd3130a8 100755 --- a/push +++ b/push @@ -12,10 +12,14 @@ echo "--> pushing minimalcompact/thumbor:simdavx2" docker push minimalcompact/thumbor:latest-simd-avx2 docker push minimalcompact/thumbor:$THUMBOR_VERSION-simd-avx2 -echo "--> pushing minimalcompact/thumbor-nginx-proxy" +echo "--> pushing minimalcompact/thumbor-nginx-proxy (DEPRECATED)" docker push minimalcompact/thumbor-nginx-proxy:latest docker push minimalcompact/thumbor-nginx-proxy:$THUMBOR_VERSION +echo "--> pushing minimalcompact/thumbor-nginx-proxy-cache" +docker push minimalcompact/thumbor-nginx-proxy-cache:latest +docker push minimalcompact/thumbor-nginx-proxy-cache:$THUMBOR_VERSION + echo "--> pushing minimalcompact/remotecv" docker push minimalcompact/remotecv:latest docker push minimalcompact/remotecv:$THUMBOR_VERSION diff --git a/recipes/docker-compose/letsencrypt/docker-compose.yml b/recipes/docker-compose/letsencrypt/docker-compose.yml index 30a662bd..1debd57f 100644 --- a/recipes/docker-compose/letsencrypt/docker-compose.yml +++ b/recipes/docker-compose/letsencrypt/docker-compose.yml @@ -18,20 +18,16 @@ services: - CORS_ALLOW_ORIGIN=* # returns a webp image if browser Accept headers match - AUTO_WEBP=True - # Basic thumbor setup to cache both original images and results (after manipulation) - # nginx-proxy would detect if there's a cached version and serve it for you automatically + # nginx-proxy does caching automatically, so no need to store the result storage cache # (this greatly speeds up and saves on CPU) - - RESULT_STORAGE=thumbor.result_storages.file_storage + - RESULT_STORAGE=thumbor.result_storages.no_storage - RESULT_STORAGE_STORES_UNSAFE=True - STORAGE=thumbor.storages.file_storage - volumes: - # mounting a /data folder to store cached images - - ./data:/data restart: always networks: - app nginx-proxy: - image: minimalcompact/thumbor-nginx-proxy + image: minimalcompact/thumbor-nginx-proxy-cache environment: # setting the DEFAULT_HOST to the same as the VIRTUAL_HOST above. # Makes sure it works irrespective of the host name @@ -64,9 +60,6 @@ services: - ./nginx/vhost.d/:/etc/nginx/vhost.d:rw depends_on: - nginx-proxy -volumes: - data: - driver: local networks: app: driver: bridge diff --git a/recipes/docker-compose/remotecv/docker-compose.yml b/recipes/docker-compose/remotecv/docker-compose.yml index 3c719505..976688cd 100644 --- a/recipes/docker-compose/remotecv/docker-compose.yml +++ b/recipes/docker-compose/remotecv/docker-compose.yml @@ -15,10 +15,9 @@ services: - CORS_ALLOW_ORIGIN=* # returns a webp image if browser Accept headers match - AUTO_WEBP=True - # Basic thumbor setup to cache both original images and results (after manipulation) - # nginx-proxy would detect if there's a cached version and serve it for you automatically + # nginx-proxy does caching automatically, so no need to store the result storage cache # (this greatly speeds up and saves on CPU) - - RESULT_STORAGE=thumbor.result_storages.file_storage + - RESULT_STORAGE=thumbor.result_storages.no_storage - RESULT_STORAGE_STORES_UNSAFE=True # setting mixed storage, and detector storage using redis - STORAGE=thumbor.storages.mixed_storage @@ -32,27 +31,26 @@ services: - REDIS_QUEUE_SERVER_HOST=redis - REDIS_QUEUE_SERVER_PORT=6379 - REDIS_QUEUE_SERVER_DB=0 - volumes: - # mounting a /data folder to store cached images - - ./data:/data links: - redis:redis restart: always networks: - app nginx-proxy: - image: minimalcompact/thumbor-nginx-proxy + image: minimalcompact/thumbor-nginx-proxy-cache environment: # setting the DEFAULT_HOST to the same as the VIRTUAL_HOST above. # Makes sure it works irrespective of the host name # Normally this won't be necessary, but it helps for testing. - DEFAULT_HOST=localhost + # optional: control cache size (default 100m) and inactive (default 300m) + # see https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_cache_path + - PROXY_CACHE_SIZE=100m + - PROXY_CACHE_INACTIVE=300m volumes: # this is essential for nginx-proxy to detect docker containers, scaling etc # see https://github.com/jwilder/nginx-proxy - /var/run/docker.sock:/tmp/docker.sock:ro - # mapping the same data folder, to allow nginx-proxy to fetch images from cache - - ./data:/data ports: - "80:80" - "443:443" @@ -81,9 +79,6 @@ services: restart: always networks: - app -volumes: - data: - driver: local networks: app: driver: bridge diff --git a/recipes/docker-compose/simple/docker-compose.yml b/recipes/docker-compose/simple/docker-compose.yml index f2b5182b..28772583 100644 --- a/recipes/docker-compose/simple/docker-compose.yml +++ b/recipes/docker-compose/simple/docker-compose.yml @@ -24,12 +24,16 @@ services: networks: - app nginx-proxy: - image: minimalcompact/thumbor-nginx-proxy + image: minimalcompact/thumbor-nginx-proxy-cache environment: # setting the DEFAULT_HOST to the same as the VIRTUAL_HOST above. # Makes sure it works irrespective of the host name # Normally this won't be necessary, but it helps for testing. - DEFAULT_HOST=localhost + # optional: control cache size (default 100m) and inactive (default 300m) + # see https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_cache_path + - PROXY_CACHE_SIZE=100m + - PROXY_CACHE_INACTIVE=300m volumes: # this is essential for nginx-proxy to detect docker containers, scaling etc # see https://github.com/jwilder/nginx-proxy diff --git a/recipes/docker-compose/swarm/docker-compose.yml b/recipes/docker-compose/swarm/docker-compose.yml index a87653b5..6319ea2a 100644 --- a/recipes/docker-compose/swarm/docker-compose.yml +++ b/recipes/docker-compose/swarm/docker-compose.yml @@ -28,7 +28,7 @@ services: - STORAGE=thumbor.storages.mixed_storage - MIXED_STORAGE_FILE_STORAGE=thumbor.storages.file_storage - MIXED_STORAGE_DETECTOR_STORAGE=tc_redis.storages.redis_storage - - RESULT_STORAGE=thumbor.result_storages.file_storage + - RESULT_STORAGE=thumbor.result_storages.no_storage # Cache storage settings # # storage keeps the original image files cached @@ -43,7 +43,6 @@ services: - REDIS_QUEUE_SERVER_DB=0 volumes: - logs:/logs - - data:/data deploy: # Note this would create 3 instances, each with THUMBOR_NUM_PROCESSES # (for docker-compose, this is ignored, but can be controlled using @@ -55,7 +54,7 @@ services: networks: - frontend nginx-proxy: - image: minimalcompact/thumbor-nginx-proxy + image: minimalcompact/thumbor-nginx-proxy-cache environment: # setting the DEFAULT_HOST to the same as the VIRTUAL_HOST above. # Makes sure it works irrespective of the host name @@ -65,8 +64,6 @@ services: # this is essential for nginx-proxy to detect docker containers, scaling etc # see https://github.com/jwilder/nginx-proxy - /var/run/docker.sock:/tmp/docker.sock:ro - # mapping the same data folder, to allow nginx-proxy to fetch images from cache - - ./data:/data ports: - "80:80" - "443:443" @@ -99,8 +96,6 @@ services: - frontend - backend volumes: - data: - driver: local logs: driver: local networks: diff --git a/tests/nginx-proxy-autowebp.bats b/tests/nginx-proxy-autowebp.bats index 1c3584a8..5de58d34 100644 --- a/tests/nginx-proxy-autowebp.bats +++ b/tests/nginx-proxy-autowebp.bats @@ -12,14 +12,23 @@ load_thumbor () { } @test "no webp headers by default even if browser accepts" { + rm -rf $BASE/data/* load_thumbor run bash -c "curl -H 'Accept: image/webp' -sSL -D - http://localhost/unsafe/500x150/i.imgur.com/Nfn80ck.png -o /dev/null |grep 'Content-Type: image/png'" [ $status -eq 0 ] } @test "webp headers if AUTO_WEBP is set and browser accepts webp" { - export AUTO_WEB=True + export AUTO_WEBP=True + rm -rf $BASE/data/* load_thumbor - run bash -c "curl -H 'Accept: image/webp' -sSL -D - http://localhost/unsafe/500x150/i.imgur.com/Nfn80ck.png -o /dev/null |grep 'Content-Type: image/png'" + run bash -c "curl -H 'Accept: image/webp' -sSL -D - http://localhost/unsafe/500x150/i.imgur.com/Nfn80ck.png -o /dev/null |grep 'Content-Type: image/webp'" + [ $status -eq 0 ] +} + +@test "NOTE: nginx proxy cache can override AUTO_WEBP (Must clear the cache when you change the settings!)" { + export AUTO_WEBP=False + load_thumbor + run bash -c "curl -H 'Accept: image/webp' -sSL -D - http://localhost/unsafe/500x150/i.imgur.com/Nfn80ck.png -o /dev/null |grep 'Content-Type: image/webp'" [ $status -eq 0 ] } diff --git a/tests/nginx-proxy-cache.bats b/tests/nginx-proxy-cache.bats new file mode 100644 index 00000000..b55666d5 --- /dev/null +++ b/tests/nginx-proxy-cache.bats @@ -0,0 +1,38 @@ +#!/usr/bin/env bats + +BASE=tests/setup/basic + +teardown () { + docker-compose -f $BASE/docker-compose.yml down +} + +load_thumbor () { + docker-compose -f $BASE/docker-compose.yml up -d + timeout 2m bash -c 'while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost/healthcheck)" != "200" ]]; do sleep 5; done' || false +} + +@test "proxy cache size=100m inactive=300m by default" { + load_thumbor + run bash -c "docker-compose -f $BASE/docker-compose.yml exec nginx-proxy bash -c 'cat /etc/nginx/conf.d/default.conf' | grep 'keys_zone=thumbor:100m inactive=300m'" + [ $status -eq 0 ] +} + +@test "proxy cache size and inactive can be set by ENV" { + export PROXY_CACHE_SIZE=200m + export PROXY_CACHE_INACTIVE=24h + load_thumbor + run bash -c "docker-compose -f $BASE/docker-compose.yml exec nginx-proxy bash -c 'cat /etc/nginx/conf.d/default.conf' | grep 'keys_zone=thumbor:200m inactive=24h'" + [ $status -eq 0 ] +} + +@test "proxy cache is controlled by thumbor MAX_AGE" { + export MAX_AGE=123456789 + load_thumbor + rm -rf $BASE/data/* + # original (un-cached request) + run bash -c "curl -H -sSL -D - http://localhost/unsafe/500x150/i.imgur.com/Nfn80ck.png -o /dev/null |grep 'Cache-Control: max-age=123456789,public'" + [ $status -eq 0 ] + # cached request + run bash -c "curl -H -sSL -D - http://localhost/unsafe/500x150/i.imgur.com/Nfn80ck.png -o /dev/null |grep 'Cache-Control: max-age=123456789,public'" + [ $status -eq 0 ] +} diff --git a/tests/nginx-proxy.bats b/tests/nginx-proxy.bats index e8d2a37c..d96862e1 100644 --- a/tests/nginx-proxy.bats +++ b/tests/nginx-proxy.bats @@ -1,8 +1,10 @@ #!/usr/bin/env bats -BASE=recipes/docker-compose/simple +BASE=tests/setup/basic + setup () { if [[ "$BATS_TEST_NUMBER" -eq 1 ]]; then + export AUTO_WEBP=True docker-compose -f $BASE/docker-compose.yml up -d timeout 2m bash -c 'while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost/healthcheck)" != "200" ]]; do sleep 5; done' || false rm -rf $BASE/data/* @@ -20,60 +22,40 @@ teardown () { } @test "PNG result is cached" { - ls $BASE/data/result_storage/default/8b/83/b846d9575c01f23bff9f430d5f81f16c44cb + ls $BASE/data/d/a9/ad4b1bf58341edd5c957cf0aa94eba9d curl -sSL -D - http://localhost/unsafe/500x150/i.imgur.com/Nfn80ck.png -o /dev/null |grep 'Content-Type: image/png' } -@test "next PNG request is fetched from cache" { - echo 1234 > $BASE/data/result_storage/default/8b/83/b846d9575c01f23bff9f430d5f81f16c44cb - curl -H -sSL -D - http://localhost/unsafe/500x150/i.imgur.com/Nfn80ck.png |tail -1 |grep 1234 -} - @test "AUTO_WEBP returns an image/webp image" { curl -H 'Accept: image/webp' -sSL -D - http://localhost/unsafe/500x150/i.imgur.com/Nfn80ck.png -o /dev/null |grep 'Content-Type: image/webp' } @test "AUTO_WEBP result is cached" { - ls $BASE/data/result_storage/auto_webp/8b/83/b846d9575c01f23bff9f430d5f81f16c44cb + ls $BASE/data/6/fd/9d408b450fecb47d2a5370cfbbeccfd6 curl -H 'Accept: image/webp' -sSL -D - http://localhost/unsafe/500x150/i.imgur.com/Nfn80ck.png -o /dev/null |grep 'Content-Type: image/webp' } -@test "next WEBP request is fetched from cache" { - echo 1234 > $BASE/data/result_storage/auto_webp/8b/83/b846d9575c01f23bff9f430d5f81f16c44cb - curl -H 'Accept: image/webp' -sSL -D - http://localhost/unsafe/500x150/i.imgur.com/Nfn80ck.png |tail -1 |grep 1234 -} - @test "filters:format(jpeg) returns an image/jpeg content-type" { - rm -f $BASE/data/result_storage/default/8b/83/b846d9575c01f23bff9f430d5f81f16c44cb + rm -f $BASE/data/b/71/72931161fa8b05e9cf6ca882e1bc771b curl -sSL -D - 'http://localhost/unsafe/500x150/filters:format(jpeg)/i.imgur.com/Nfn80ck.png' -o /dev/null |grep 'Content-Type: image/jpeg' } @test "filters:format(jpeg) result is cached" { - ls $BASE/data/result_storage/default/77/1f/8c8786e4ad7ed364b2ea722fbe8f3ec043b6 + ls $BASE/data/b/71/72931161fa8b05e9cf6ca882e1bc771b curl -sSL -D - 'http://localhost/unsafe/500x150/filters:format(jpeg)/i.imgur.com/Nfn80ck.png' -o /dev/null |grep 'Content-Type: image/jpeg' } -@test "next filters:format(jpeg) request is fetched from cache" { - echo 1234 > $BASE/data/result_storage/default/77/1f/8c8786e4ad7ed364b2ea722fbe8f3ec043b6 - curl -sSL -D - 'http://localhost/unsafe/500x150/filters:format(jpeg)/i.imgur.com/Nfn80ck.png' |tail -1 |grep 1234 -} - @test "filters:format(webp) returns an image/webp content-type" { - rm -f $BASE/data/result_storage/default/a4/5a/9b7ed7290937d3c996c178141ae5517f8567 + rm -f $BASE/data/b/71/72931161fa8b05e9cf6ca882e1bc771b curl -sSL -D - 'http://localhost/unsafe/500x150/filters:format(webp)/i.imgur.com/Nfn80ck.png' -o /dev/null |grep 'Content-Type: image/webp' } @test "filters:format(webp) result is cached" { - ls $BASE/data/result_storage/default/a4/5a/9b7ed7290937d3c996c178141ae5517f8567 + ls $BASE/data/4/c7/2b3b73bd53c5d7e35f17b6d358ecbc74 curl -sSL -D - 'http://localhost/unsafe/500x150/filters:format(webp)/i.imgur.com/Nfn80ck.png' -o /dev/null |grep 'Content-Type: image/webp' } -@test "next filters:format(webp) request is fetched from cache" { - echo 1234 > $BASE/data/result_storage/default/a4/5a/9b7ed7290937d3c996c178141ae5517f8567 - curl -sSL -D - 'http://localhost/unsafe/500x150/filters:format(webp)/i.imgur.com/Nfn80ck.png' |tail -1 |grep 1234 -} - @test "only one content-type header is present" { count=`curl -H 'Accept: image/webp' -sSL -D - http://localhost/unsafe/500x150/i.imgur.com/Nfn80ck.png -o /dev/null |grep 'Content-Type:'|wc -l` [ $count -eq 1 ] -} \ No newline at end of file +} diff --git a/tests/setup/basic/docker-compose.yml b/tests/setup/basic/docker-compose.yml index dd0f7cd8..1c608052 100644 --- a/tests/setup/basic/docker-compose.yml +++ b/tests/setup/basic/docker-compose.yml @@ -18,28 +18,28 @@ services: # Basic thumbor setup to cache both original images and results (after manipulation) # nginx-proxy would detect if there's a cached version and serve it for you automatically # (this greatly speeds up and saves on CPU) - - RESULT_STORAGE=thumbor.result_storages.file_storage + - RESULT_STORAGE=thumbor.result_storages.no_storage - RESULT_STORAGE_STORES_UNSAFE=True - STORAGE=thumbor.storages.file_storage - volumes: - # mounting a /data folder to store cached images - - ./data:/data + - MAX_AGE restart: always networks: - app nginx-proxy: - image: minimalcompact/thumbor-nginx-proxy + image: minimalcompact/thumbor-nginx-proxy-cache environment: # setting the DEFAULT_HOST to the same as the VIRTUAL_HOST above. # Makes sure it works irrespective of the host name # Normally this won't be necessary, but it helps for testing. - DEFAULT_HOST=localhost + - PROXY_CACHE_SIZE + - PROXY_CACHE_INACTIVE volumes: # this is essential for nginx-proxy to detect docker containers, scaling etc # see https://github.com/jwilder/nginx-proxy - /var/run/docker.sock:/tmp/docker.sock:ro - # mapping the same data folder, to allow nginx-proxy to fetch images from cache - - ./data:/data + # mapping cache folder, only needed to test that cache is stored + - ./data:/var/cache/nginx ports: - "80:80" - "443:443" From 4790341d56999cdffd4e61709a67dac4cd27ba18 Mon Sep 17 00:00:00 2001 From: Yoav Date: Sun, 25 Aug 2019 10:25:30 +0200 Subject: [PATCH 3/7] revert changes to (deprecated) nginx-proxy ; adding deprecation warning --- nginx-proxy/Dockerfile | 3 ++ nginx-proxy/deprecated.sh | 42 +++++++++++++++++++++++++ nginx-proxy/imgpath.js | 5 +++ nginx-proxy/nginx.tmpl | 64 +++++++++++++++++++++++++++++++++++---- 4 files changed, 108 insertions(+), 6 deletions(-) create mode 100755 nginx-proxy/deprecated.sh create mode 100644 nginx-proxy/imgpath.js diff --git a/nginx-proxy/Dockerfile b/nginx-proxy/Dockerfile index a402b06e..cdec3a57 100644 --- a/nginx-proxy/Dockerfile +++ b/nginx-proxy/Dockerfile @@ -3,4 +3,7 @@ FROM jwilder/nginx-proxy LABEL maintainer="MinimalCompact" COPY ./nginx.conf /etc/nginx/ +COPY ./imgpath.js /etc/nginx/ COPY ./nginx.tmpl /app/ + +COPY ./deprecated.sh /app/docker-entrypoint.sh diff --git a/nginx-proxy/deprecated.sh b/nginx-proxy/deprecated.sh new file mode 100755 index 00000000..47fdd1b1 --- /dev/null +++ b/nginx-proxy/deprecated.sh @@ -0,0 +1,42 @@ +#!/bin/bash +set -e + +# Warn if the DOCKER_HOST socket does not exist +if [[ $DOCKER_HOST = unix://* ]]; then + socket_file=${DOCKER_HOST#unix://} + if ! [ -S $socket_file ]; then + cat >&2 <<-EOT + ERROR: you need to share your Docker host socket with a volume at $socket_file + Typically you should run your jwilder/nginx-proxy with: \`-v /var/run/docker.sock:$socket_file:ro\` + See the documentation at http://git.io/vZaGJ + EOT + socketMissing=1 + fi +fi + +# Generate dhparam file if required +# Note: if $DHPARAM_BITS is not defined, generate-dhparam.sh will use 2048 as a default +# Note2: if $DHPARAM_GENERATION is set to false in environment variable, dh param generator will skip completely +/app/generate-dhparam.sh $DHPARAM_BITS $DHPARAM_GENERATION + +# Compute the DNS resolvers for use in the templates - if the IP contains ":", it's IPv6 and must be enclosed in [] +export RESOLVERS=$(awk '$1 == "nameserver" {print ($2 ~ ":")? "["$2"]": $2}' ORS=' ' /etc/resolv.conf | sed 's/ *$//g') +if [ "x$RESOLVERS" = "x" ]; then + echo "Warning: unable to determine DNS resolvers for nginx" >&2 + unset RESOLVERS +fi + +# If the user has run the default command and the socket doesn't exist, fail +if [ "$socketMissing" = 1 -a "$1" = forego -a "$2" = start -a "$3" = '-r' ]; then + exit 1 +fi + +>&2 echo +>&2 echo +>&2 echo +>&2 echo "DEPRECATED. Please switch to thumbor-caching-proxy" +>&2 echo +>&2 echo +>&2 echo + +exec "$@" diff --git a/nginx-proxy/imgpath.js b/nginx-proxy/imgpath.js new file mode 100644 index 00000000..47950c53 --- /dev/null +++ b/nginx-proxy/imgpath.js @@ -0,0 +1,5 @@ +function file_path(r) { + var uri = decodeURI(r.uri); + var digest = require('crypto').createHash('sha1').update(uri).digest('hex'); + return digest.slice(NaN, 2) + "/" + digest.slice(2, 4) + "/" + digest.slice(4); +} diff --git a/nginx-proxy/nginx.tmpl b/nginx-proxy/nginx.tmpl index 8cecb7cf..be0de7a4 100644 --- a/nginx-proxy/nginx.tmpl +++ b/nginx-proxy/nginx.tmpl @@ -1,5 +1,35 @@ {{ $CurrentContainer := where $ "ID" .Docker.CurrentContainerID | first }} +{{ define "thumbor-include" }} + merge_slashes off; + js_include imgpath.js; + js_set $file_path file_path; +{{ end }} + +{{ define "thumbor-map" }} + map $http_accept $webp_prefix { + default "/default"; + "~*webp*" "/auto_webp"; + } + map $request_uri $format { + default ""; + "~*format\(jpeg\)" "image/jpeg"; + "~*format\(jpg\)" "image/jpeg"; + "~*format\(png\)" "image/png"; + "~*format\(gif\)" "image/gif"; + "~*format\(webp\)" "image/webp"; + "~*\.jpeg" "image/jpeg"; + "~*\.jpg" "image/jpeg"; + "~*\.png" "image/png"; + "~*\.gif" "image/gif"; + "~*\.webp" "image/webp"; + } + map $http_accept $auto_webp_content_type { + default $format; + "~*webp*" "image/webp"; + } +{{ end }} + {{ define "cors" }} add_header 'Access-Control-Allow-Origin' '{{ or .CorsOrigin "*" }}'; add_header 'Access-Control-Allow-Credentials' 'true'; @@ -10,23 +40,42 @@ {{ define "thumbor" }} {{ $proto := .Proto }} {{ $upstream_name := .UpstreamName }} + {{ $autoWebp := (first (groupByKeys .Containers "Env.AUTO_WEBP")) }} {{ $allowCors := (first (groupByKeys .Containers "Env.THUMBOR_ALLOW_CORS")) }} {{ $corsOrigin := (first (groupByKeys .Containers "Env.CORS_ALLOW_ORIGIN")) }} - location ~* "^/(.+)?$" { + location ~* "^/(?..)(?..)(.+)?$" { + root /data/result_storage; + expires 1M; + {{ if or $allowCors $corsOrigin }} {{ template "cors" (dict "CorsOrigin" $corsOrigin) }} {{ end }} - - proxy_cache thumbor; - proxy_buffering on; - proxy_pass {{ trim $proto }}://{{ trim $upstream_name }}; + {{ if $autoWebp }} + try_files $webp_prefix/$file_path =404; + add_header Vary Accept; + #if auto webp is on then content-type is based on accept header + set $format $auto_webp_content_type; + {{ else }} + try_files /default/$file_path =404; + {{ end }} + # if $format is empty (default) then this is ignored + add_header Content-Type $format; + error_page 404 = @fetch; } location = /healthcheck { proxy_pass {{ trim .Proto }}://{{ trim $upstream_name }}; access_log off; } + + location @fetch { + internal; + {{ if or $allowCors $corsOrigin }} + {{ template "cors" (dict "CorsOrigin" $corsOrigin) }} + {{ end }} + proxy_pass {{ trim $proto }}://{{ trim $upstream_name }}; + } {{ end }} {{ define "upstream" }} @@ -98,6 +147,9 @@ access_log off; resolver {{ $.Env.RESOLVERS }}; {{ end }} +{{ template "thumbor-include" }} +{{ template "thumbor-map" }} + {{ if (exists "/etc/nginx/proxy.conf") }} include /etc/nginx/proxy.conf; {{ else }} @@ -112,7 +164,7 @@ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $proxy_x_forwarded_proto; proxy_set_header X-Forwarded-Ssl $proxy_x_forwarded_ssl; proxy_set_header X-Forwarded-Port $proxy_x_forwarded_port; -proxy_cache_path /var/cache/nginx levels=1:2 keys_zone=thumbor:100m inactive=300m use_temp_path=off; + # Mitigate httpoxy attack (see README for details) proxy_set_header Proxy ""; {{ end }} From 02f35d7be5c4409852274c0b845f462a200d24c1 Mon Sep 17 00:00:00 2001 From: Yoav Date: Sun, 25 Aug 2019 10:40:24 +0200 Subject: [PATCH 4/7] added a note to README --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 4dc16172..e55d4def 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,11 @@ Check out the recipes folder for some examples (still work in progress). The recipes include comments to document how things should be set up and why. +## Changelog / Deprecation + +* `minimalcompact/thumbor-nginx-proxy` is deprecated and replaced by `minimalcompact/thumbor-nginx-proxy-cache` + see https://github.com/MinimalCompact/thumbor/pull/55 + ## History This project is a loose fork of `APSL/docker-thumbor`. It's not a direct fork, because lots has changed, and it's not From 5fd0a553af911b44b3c88b6cbb65eebc63d6c43b Mon Sep 17 00:00:00 2001 From: Yoav Date: Sun, 1 Sep 2019 13:53:27 +0200 Subject: [PATCH 5/7] fix wrong name in deprecated --- nginx-proxy/deprecated.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nginx-proxy/deprecated.sh b/nginx-proxy/deprecated.sh index 47fdd1b1..de7a0d4c 100755 --- a/nginx-proxy/deprecated.sh +++ b/nginx-proxy/deprecated.sh @@ -34,7 +34,7 @@ fi >&2 echo >&2 echo >&2 echo ->&2 echo "DEPRECATED. Please switch to thumbor-caching-proxy" +>&2 echo "DEPRECATED. Please switch to thumbor-nginx-proxy-cache" >&2 echo >&2 echo >&2 echo From c80b3456df0b4ef1d58d42339092dd7dc08607ca Mon Sep 17 00:00:00 2001 From: Yoav Date: Sun, 1 Sep 2019 14:18:39 +0200 Subject: [PATCH 6/7] updated default cache size (to 10g) and inactive time (to 48h) * added README for nginx-proxy-cache * updated recipes to add a persisted cache folder --- README.md | 2 +- nginx-proxy-cache/README.md | 24 +++++++++++++++++++ nginx-proxy-cache/nginx.tmpl | 4 ++-- .../letsencrypt/docker-compose.yml | 7 ++++-- .../remotecv/docker-compose.yml | 5 ++++ .../docker-compose/simple/docker-compose.yml | 5 ++++ .../docker-compose/swarm/docker-compose.yml | 4 ++++ 7 files changed, 46 insertions(+), 5 deletions(-) create mode 100644 nginx-proxy-cache/README.md diff --git a/README.md b/README.md index e55d4def..95a2d15c 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ Key Features and Goals: * The latest version of thumbor and dependencies, in a docker image * Supports both solo thumbor and multiprocess in one image -* Nginx frontend docker image with built-in caching, using [nginx-proxy](https://github.com/jwilder/nginx-proxy) +* [Nginx frontend docker image with built-in caching](nginx-proxy-cache/README.md), using [nginx-proxy](https://github.com/jwilder/nginx-proxy) * SIMD support via docker tags * remotecv docker image (for async smart cropping and feature detection) * Clear version tagging to match Thumbor versions diff --git a/nginx-proxy-cache/README.md b/nginx-proxy-cache/README.md new file mode 100644 index 00000000..4bcaa955 --- /dev/null +++ b/nginx-proxy-cache/README.md @@ -0,0 +1,24 @@ +# Minimal Compact thumbor nginx proxy cache + +A caching proxy for thumbor based on [nginx-proxy](https://github.com/jwilder/nginx-proxy) + +## Basics + +[nginx-proxy](https://github.com/jwilder/nginx-proxy) provides a way to dynamically attach docker images and get them proxied by Nginx. + +This image adds a proxy caching layer into `nginx-proxy`, using the built-in `proxy_cache` directive. + +## Defaults + +* Cache data is stored inside the container in `/var/cache/nginx` +* Cache size is set to 10Gb (`keys_zone=thumbor:10g`) +* Cached data that isn't accessed for more than 48 hours will be purged (`inactive=48h`) + +## Overriding defaults + +* You can/should mount a volume to persist cached data even when the container is recreated, e.g. `docker run -v /var/run/docker.sock:/tmp/docker.sock:ro -v /path/to/cache:/var/nginx/cache minimalcompact/thumbor-nginx-proxy-cache` +* use `PROXY_CACHE_SIZE`, `PROXY_CACHE_INACTIVE` to override the cache size and inactive time + +## Examples + +See recipes for usage examples diff --git a/nginx-proxy-cache/nginx.tmpl b/nginx-proxy-cache/nginx.tmpl index a828cb72..0dae1b38 100644 --- a/nginx-proxy-cache/nginx.tmpl +++ b/nginx-proxy-cache/nginx.tmpl @@ -30,8 +30,8 @@ {{ end }} {{ define "thumbor-caching" }} - {{ $proxyCacheSize := or .ProxyCacheSize "100m" }} - {{ $proxyCacheInactive := or .ProxyCacheInactive "300m" }} + {{ $proxyCacheSize := or .ProxyCacheSize "10g" }} + {{ $proxyCacheInactive := or .ProxyCacheInactive "48h" }} proxy_cache_path /var/cache/nginx levels=1:2 keys_zone=thumbor:{{ $proxyCacheSize }} inactive={{ $proxyCacheInactive }} use_temp_path=off; {{ end }} diff --git a/recipes/docker-compose/letsencrypt/docker-compose.yml b/recipes/docker-compose/letsencrypt/docker-compose.yml index 1debd57f..5191ea76 100644 --- a/recipes/docker-compose/letsencrypt/docker-compose.yml +++ b/recipes/docker-compose/letsencrypt/docker-compose.yml @@ -37,8 +37,8 @@ services: # this is essential for nginx-proxy to detect docker containers, scaling etc # see https://github.com/jwilder/nginx-proxy - /var/run/docker.sock:/tmp/docker.sock:ro - # mapping the same data folder, to allow nginx-proxy to fetch images from cache - - ./data:/data + # mapping cache folder, to persist it independently of the container + - ./cache:/var/cache/nginx # mapping for letsencrypt companion (note using the current folder) - ./nginx/vhost.d:/etc/nginx/vhost.d:ro - ./nginx/html:/usr/share/nginx/html @@ -60,6 +60,9 @@ services: - ./nginx/vhost.d/:/etc/nginx/vhost.d:rw depends_on: - nginx-proxy +volumes: + cache: + driver: local networks: app: driver: bridge diff --git a/recipes/docker-compose/remotecv/docker-compose.yml b/recipes/docker-compose/remotecv/docker-compose.yml index 976688cd..734596f8 100644 --- a/recipes/docker-compose/remotecv/docker-compose.yml +++ b/recipes/docker-compose/remotecv/docker-compose.yml @@ -51,6 +51,8 @@ services: # this is essential for nginx-proxy to detect docker containers, scaling etc # see https://github.com/jwilder/nginx-proxy - /var/run/docker.sock:/tmp/docker.sock:ro + # mapping cache folder, to persist it independently of the container + - ./cache:/var/cache/nginx ports: - "80:80" - "443:443" @@ -79,6 +81,9 @@ services: restart: always networks: - app +volumes: + cache: + driver: local networks: app: driver: bridge diff --git a/recipes/docker-compose/simple/docker-compose.yml b/recipes/docker-compose/simple/docker-compose.yml index 28772583..22952ed1 100644 --- a/recipes/docker-compose/simple/docker-compose.yml +++ b/recipes/docker-compose/simple/docker-compose.yml @@ -38,12 +38,17 @@ services: # this is essential for nginx-proxy to detect docker containers, scaling etc # see https://github.com/jwilder/nginx-proxy - /var/run/docker.sock:/tmp/docker.sock:ro + # mapping cache folder, to persist it independently of the container + - ./cache:/var/cache/nginx ports: - "80:80" - "443:443" restart: always networks: - app +volumes: + cache: + driver: local networks: app: driver: bridge diff --git a/recipes/docker-compose/swarm/docker-compose.yml b/recipes/docker-compose/swarm/docker-compose.yml index 6319ea2a..e23d0e8a 100644 --- a/recipes/docker-compose/swarm/docker-compose.yml +++ b/recipes/docker-compose/swarm/docker-compose.yml @@ -64,6 +64,8 @@ services: # this is essential for nginx-proxy to detect docker containers, scaling etc # see https://github.com/jwilder/nginx-proxy - /var/run/docker.sock:/tmp/docker.sock:ro + # mapping cache folder, to persist it independently of the container + - ./cache:/var/cache/nginx ports: - "80:80" - "443:443" @@ -96,6 +98,8 @@ services: - frontend - backend volumes: + cache: + driver: local logs: driver: local networks: From a22f45843a55ada66de187aa7632d5ede7a5eef5 Mon Sep 17 00:00:00 2001 From: Yoav Date: Sun, 1 Sep 2019 14:50:29 +0200 Subject: [PATCH 7/7] changed environment variables to distinguish between max_size (on disk) and memory size --- nginx-proxy-cache/README.md | 5 +++-- nginx-proxy-cache/nginx.tmpl | 5 +++-- recipes/docker-compose/remotecv/docker-compose.yml | 5 +++-- recipes/docker-compose/simple/docker-compose.yml | 5 +++-- tests/nginx-proxy-cache.bats | 12 +++++++----- tests/setup/basic/docker-compose.yml | 1 + 6 files changed, 20 insertions(+), 13 deletions(-) diff --git a/nginx-proxy-cache/README.md b/nginx-proxy-cache/README.md index 4bcaa955..4bc98af7 100644 --- a/nginx-proxy-cache/README.md +++ b/nginx-proxy-cache/README.md @@ -11,13 +11,14 @@ This image adds a proxy caching layer into `nginx-proxy`, using the built-in `pr ## Defaults * Cache data is stored inside the container in `/var/cache/nginx` -* Cache size is set to 10Gb (`keys_zone=thumbor:10g`) +* MAx cache size is set to 10Gb (`max_size=10g`) * Cached data that isn't accessed for more than 48 hours will be purged (`inactive=48h`) +* Cache memory size is set to 500m (`keys_zone=thumbor:500m`) holds all active keys and metadata of the cache in memory ## Overriding defaults * You can/should mount a volume to persist cached data even when the container is recreated, e.g. `docker run -v /var/run/docker.sock:/tmp/docker.sock:ro -v /path/to/cache:/var/nginx/cache minimalcompact/thumbor-nginx-proxy-cache` -* use `PROXY_CACHE_SIZE`, `PROXY_CACHE_INACTIVE` to override the cache size and inactive time +* use `PROXY_CACHE_SIZE`, `PROXY_CACHE_INACTIVE`, `PROXY_CACHE_MEMORY_SIZE` to override the max cache size, inactive time or memory size ## Examples diff --git a/nginx-proxy-cache/nginx.tmpl b/nginx-proxy-cache/nginx.tmpl index 0dae1b38..a434bb33 100644 --- a/nginx-proxy-cache/nginx.tmpl +++ b/nginx-proxy-cache/nginx.tmpl @@ -30,9 +30,10 @@ {{ end }} {{ define "thumbor-caching" }} + {{ $proxyCacheMemorySize := or .ProxyCacheMemorySize "500m" }} {{ $proxyCacheSize := or .ProxyCacheSize "10g" }} {{ $proxyCacheInactive := or .ProxyCacheInactive "48h" }} - proxy_cache_path /var/cache/nginx levels=1:2 keys_zone=thumbor:{{ $proxyCacheSize }} inactive={{ $proxyCacheInactive }} use_temp_path=off; + proxy_cache_path /var/cache/nginx levels=1:2 keys_zone=thumbor:{{ $proxyCacheMemorySize }} inactive={{ $proxyCacheInactive }} max_size={{ $proxyCacheSize }} use_temp_path=off; {{ end }} {{ define "upstream" }} @@ -119,7 +120,7 @@ proxy_set_header X-Forwarded-Proto $proxy_x_forwarded_proto; proxy_set_header X-Forwarded-Ssl $proxy_x_forwarded_ssl; proxy_set_header X-Forwarded-Port $proxy_x_forwarded_port; -{{ template "thumbor-caching" (dict "ProxyCacheSize" ($.Env.PROXY_CACHE_SIZE) "ProxyCacheInactive" ($.Env.PROXY_CACHE_INACTIVE)) }} +{{ template "thumbor-caching" (dict "ProxyCacheSize" ($.Env.PROXY_CACHE_SIZE) "ProxyCacheMemorySize" ($.Env.PROXY_CACHE_MEMORY_SIZE) "ProxyCacheInactive" ($.Env.PROXY_CACHE_INACTIVE)) }} # Mitigate httpoxy attack (see README for details) proxy_set_header Proxy ""; diff --git a/recipes/docker-compose/remotecv/docker-compose.yml b/recipes/docker-compose/remotecv/docker-compose.yml index 734596f8..e6eea37f 100644 --- a/recipes/docker-compose/remotecv/docker-compose.yml +++ b/recipes/docker-compose/remotecv/docker-compose.yml @@ -43,9 +43,10 @@ services: # Makes sure it works irrespective of the host name # Normally this won't be necessary, but it helps for testing. - DEFAULT_HOST=localhost - # optional: control cache size (default 100m) and inactive (default 300m) + # optional: control cache memory size (default 500m), cache size (default 10g) and inactive (default 300m) # see https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_cache_path - - PROXY_CACHE_SIZE=100m + - PROXY_CACHE_SIZE=10g + - PROXY_CACHE_MEMORY_SIZE=500m - PROXY_CACHE_INACTIVE=300m volumes: # this is essential for nginx-proxy to detect docker containers, scaling etc diff --git a/recipes/docker-compose/simple/docker-compose.yml b/recipes/docker-compose/simple/docker-compose.yml index 22952ed1..963d550d 100644 --- a/recipes/docker-compose/simple/docker-compose.yml +++ b/recipes/docker-compose/simple/docker-compose.yml @@ -30,9 +30,10 @@ services: # Makes sure it works irrespective of the host name # Normally this won't be necessary, but it helps for testing. - DEFAULT_HOST=localhost - # optional: control cache size (default 100m) and inactive (default 300m) + # optional: control cache memory size (default 500m), cache size (default 10g) and inactive (default 300m) # see https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_cache_path - - PROXY_CACHE_SIZE=100m + - PROXY_CACHE_SIZE=10g + - PROXY_CACHE_MEMORY_SIZE=500m - PROXY_CACHE_INACTIVE=300m volumes: # this is essential for nginx-proxy to detect docker containers, scaling etc diff --git a/tests/nginx-proxy-cache.bats b/tests/nginx-proxy-cache.bats index b55666d5..fe01c505 100644 --- a/tests/nginx-proxy-cache.bats +++ b/tests/nginx-proxy-cache.bats @@ -11,17 +11,19 @@ load_thumbor () { timeout 2m bash -c 'while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost/healthcheck)" != "200" ]]; do sleep 5; done' || false } -@test "proxy cache size=100m inactive=300m by default" { +@test "proxy cache memory size=500m, inactive=48h, max_size=10g by default" { load_thumbor - run bash -c "docker-compose -f $BASE/docker-compose.yml exec nginx-proxy bash -c 'cat /etc/nginx/conf.d/default.conf' | grep 'keys_zone=thumbor:100m inactive=300m'" + run bash -c "docker-compose -f $BASE/docker-compose.yml exec nginx-proxy bash -c 'cat /etc/nginx/conf.d/default.conf' | grep 'keys_zone=thumbor:500m inactive=48h max_size=10g'" [ $status -eq 0 ] } -@test "proxy cache size and inactive can be set by ENV" { - export PROXY_CACHE_SIZE=200m +@test "proxy cache size, memory size and inactive can be set by ENV" { + export PROXY_CACHE_SIZE=20g export PROXY_CACHE_INACTIVE=24h + export PROXY_CACHE_MEMORY_SIZE=10m load_thumbor - run bash -c "docker-compose -f $BASE/docker-compose.yml exec nginx-proxy bash -c 'cat /etc/nginx/conf.d/default.conf' | grep 'keys_zone=thumbor:200m inactive=24h'" + bash -c "docker-compose -f $BASE/docker-compose.yml exec nginx-proxy bash -c 'cat /etc/nginx/conf.d/default.conf'" + run bash -c "docker-compose -f $BASE/docker-compose.yml exec nginx-proxy bash -c 'cat /etc/nginx/conf.d/default.conf' | grep 'keys_zone=thumbor:10m inactive=24h max_size=20g'" [ $status -eq 0 ] } diff --git a/tests/setup/basic/docker-compose.yml b/tests/setup/basic/docker-compose.yml index 1c608052..9487548a 100644 --- a/tests/setup/basic/docker-compose.yml +++ b/tests/setup/basic/docker-compose.yml @@ -33,6 +33,7 @@ services: # Normally this won't be necessary, but it helps for testing. - DEFAULT_HOST=localhost - PROXY_CACHE_SIZE + - PROXY_CACHE_MEMORY_SIZE - PROXY_CACHE_INACTIVE volumes: # this is essential for nginx-proxy to detect docker containers, scaling etc