refs #54 nginx-proxy caching

README.md

@@ -8,7 +8,7 @@ Key Features and Goals:
 
 * The latest version of thumbor and dependencies, in a docker image
 * Supports both solo thumbor and multiprocess in one image
-* Nginx frontend docker image with built-in caching, using [nginx-proxy](https://github.com/jwilder/nginx-proxy)
+* [Nginx frontend docker image with built-in caching](nginx-proxy-cache/README.md), using [nginx-proxy](https://github.com/jwilder/nginx-proxy)
 * SIMD support via docker tags
 * remotecv docker image (for async smart cropping and feature detection)
 * Clear version tagging to match Thumbor versions
@@ -34,6 +34,11 @@ Check out the recipes folder for some examples (still work in progress).
 
 The recipes include comments to document how things should be set up and why.
 
+## Changelog / Deprecation
+
+* `minimalcompact/thumbor-nginx-proxy` is deprecated and replaced by `minimalcompact/thumbor-nginx-proxy-cache`
+  see https://github.com/MinimalCompact/thumbor/pull/55
+
 ## History
 
 This project is a loose fork of `APSL/docker-thumbor`. It's not a direct fork, because lots has changed, and it's not

build

@@ -26,13 +26,20 @@ docker tag minimalcompact/thumbor-simd-avx2 minimalcompact/thumbor:$THUMBOR_VERS
 echo "--> TAGGING minimalcompact/thumbor:latest-simd-avx2"
 docker tag minimalcompact/thumbor-simd-avx2 minimalcompact/thumbor:latest-simd-avx2
 
-echo "--> BUILDING minimalcompact/thumbor-nginx-proxy"
+echo "--> BUILDING minimalcompact/thumbor-nginx-proxy (DEPRECATED)"
 docker build --pull -f nginx-proxy/Dockerfile -t minimalcompact/thumbor-nginx-proxy nginx-proxy/
 echo "--> TAGGING minimalcompact/thumbor-nginx-proxy:$THUMBOR_VERSION"
 docker tag minimalcompact/thumbor-nginx-proxy minimalcompact/thumbor-nginx-proxy:$THUMBOR_VERSION
 echo "--> TAGGING minimalcompact/thumbor-nginx-proxy:latest"
 docker tag minimalcompact/thumbor-nginx-proxy minimalcompact/thumbor-nginx-proxy:latest
 
+echo "--> BUILDING minimalcompact/thumbor-nginx-proxy-cache"
+docker build --pull -f nginx-proxy-cache/Dockerfile -t minimalcompact/thumbor-nginx-proxy-cache nginx-proxy-cache/
+echo "--> TAGGING minimalcompact/thumbor-nginx-proxy-cache:$THUMBOR_VERSION"
+docker tag minimalcompact/thumbor-nginx-proxy-cache minimalcompact/thumbor-nginx-proxy-cache:$THUMBOR_VERSION
+echo "--> TAGGING minimalcompact/thumbor-nginx-proxy-cache:latest"
+docker tag minimalcompact/thumbor-nginx-proxy-cache minimalcompact/thumbor-nginx-proxy-cache:latest
+
 echo "--> BUILDING minimalcompact/remotecv"
 docker build --build-arg THUMBOR_TAG=latest -f remotecv/Dockerfile -t minimalcompact/remotecv remotecv/
 echo "--> TAGGING minimalcompact/remotecv:$THUMBOR_VERSION"

nginx-proxy-cache/Dockerfile

@@ -0,0 +1,5 @@
+FROM jwilder/nginx-proxy
+
+LABEL maintainer="MinimalCompact"
+
+COPY ./nginx.tmpl /app/

nginx-proxy-cache/README.md

@@ -0,0 +1,25 @@
+# Minimal Compact thumbor nginx proxy cache
+
+A caching proxy for thumbor based on [nginx-proxy](https://github.com/jwilder/nginx-proxy)
+
+## Basics
+
+[nginx-proxy](https://github.com/jwilder/nginx-proxy) provides a way to dynamically attach docker images and get them proxied by Nginx.
+
+This image adds a proxy caching layer into `nginx-proxy`, using the built-in `proxy_cache` directive.
+
+## Defaults
+
+* Cache data is stored inside the container in `/var/cache/nginx`
+* MAx cache size is set to 10Gb (`max_size=10g`)
+* Cached data that isn't accessed for more than 48 hours will be purged (`inactive=48h`)
+* Cache memory size is set to 500m (`keys_zone=thumbor:500m`) holds all active keys and metadata of the cache in memory
+
+## Overriding defaults
+
+* You can/should mount a volume to persist cached data even when the container is recreated, e.g. `docker run -v /var/run/docker.sock:/tmp/docker.sock:ro -v /path/to/cache:/var/nginx/cache minimalcompact/thumbor-nginx-proxy-cache`
+* use `PROXY_CACHE_SIZE`, `PROXY_CACHE_INACTIVE`, `PROXY_CACHE_MEMORY_SIZE` to override the max cache size, inactive time or memory size
+
+## Examples
+
+See recipes for usage examples

nginx-proxy/Dockerfile

@@ -5,3 +5,5 @@ LABEL maintainer="MinimalCompact"
 COPY ./nginx.conf /etc/nginx/
 COPY ./imgpath.js /etc/nginx/
 COPY ./nginx.tmpl /app/
+
+COPY ./deprecated.sh /app/docker-entrypoint.sh

nginx-proxy/deprecated.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+set -e
+
+# Warn if the DOCKER_HOST socket does not exist
+if [[ $DOCKER_HOST = unix://* ]]; then
+	socket_file=${DOCKER_HOST#unix://}
+	if ! [ -S $socket_file ]; then
+		cat >&2 <<-EOT
+			ERROR: you need to share your Docker host socket with a volume at $socket_file
+			Typically you should run your jwilder/nginx-proxy with: \`-v /var/run/docker.sock:$socket_file:ro\`
+			See the documentation at http://git.io/vZaGJ
+		EOT
+		socketMissing=1
+	fi
+fi
+
+# Generate dhparam file if required
+# Note: if $DHPARAM_BITS is not defined, generate-dhparam.sh will use 2048 as a default
+# Note2: if $DHPARAM_GENERATION is set to false in environment variable, dh param generator will skip completely
+/app/generate-dhparam.sh $DHPARAM_BITS $DHPARAM_GENERATION
+
+# Compute the DNS resolvers for use in the templates - if the IP contains ":", it's IPv6 and must be enclosed in []
+export RESOLVERS=$(awk '$1 == "nameserver" {print ($2 ~ ":")? "["$2"]": $2}' ORS=' ' /etc/resolv.conf | sed 's/ *$//g')
+if [ "x$RESOLVERS" = "x" ]; then
+    echo "Warning: unable to determine DNS resolvers for nginx" >&2
+    unset RESOLVERS
+fi
+
+# If the user has run the default command and the socket doesn't exist, fail
+if [ "$socketMissing" = 1 -a "$1" = forego -a "$2" = start -a "$3" = '-r' ]; then
+	exit 1
+fi
+
+>&2 echo
+>&2 echo
+>&2 echo
+>&2 echo "DEPRECATED. Please switch to thumbor-nginx-proxy-cache"
+>&2 echo
+>&2 echo
+>&2 echo
+
+exec "$@"

push

@@ -12,10 +12,14 @@ echo "--> pushing minimalcompact/thumbor:simdavx2"
 docker push minimalcompact/thumbor:latest-simd-avx2
 docker push minimalcompact/thumbor:$THUMBOR_VERSION-simd-avx2
 
-echo "--> pushing minimalcompact/thumbor-nginx-proxy"
+echo "--> pushing minimalcompact/thumbor-nginx-proxy (DEPRECATED)"
 docker push minimalcompact/thumbor-nginx-proxy:latest
 docker push minimalcompact/thumbor-nginx-proxy:$THUMBOR_VERSION
 
+echo "--> pushing minimalcompact/thumbor-nginx-proxy-cache"
+docker push minimalcompact/thumbor-nginx-proxy-cache:latest
+docker push minimalcompact/thumbor-nginx-proxy-cache:$THUMBOR_VERSION
+
 echo "--> pushing minimalcompact/remotecv"
 docker push minimalcompact/remotecv:latest
 docker push minimalcompact/remotecv:$THUMBOR_VERSION

recipes/docker-compose/letsencrypt/docker-compose.yml

@@ -18,20 +18,16 @@ services:
       - CORS_ALLOW_ORIGIN=*
       # returns a webp image if browser Accept headers match
       - AUTO_WEBP=True
-      # Basic thumbor setup to cache both original images and results (after manipulation)
-      # nginx-proxy would detect if there's a cached version and serve it for you automatically
+      # nginx-proxy does caching automatically, so no need to store the result storage cache
       # (this greatly speeds up and saves on CPU)
-      - RESULT_STORAGE=thumbor.result_storages.file_storage
+      - RESULT_STORAGE=thumbor.result_storages.no_storage
       - RESULT_STORAGE_STORES_UNSAFE=True
       - STORAGE=thumbor.storages.file_storage
-    volumes:
-      # mounting a /data folder to store cached images
-      - ./data:/data
     restart: always
     networks:
       - app
   nginx-proxy:
-    image: minimalcompact/thumbor-nginx-proxy
+    image: minimalcompact/thumbor-nginx-proxy-cache
     environment:
       # setting the DEFAULT_HOST to the same as the VIRTUAL_HOST above.
       # Makes sure it works irrespective of the host name
@@ -41,8 +37,8 @@ services:
       # this is essential for nginx-proxy to detect docker containers, scaling etc
       # see https://github.com/jwilder/nginx-proxy
       - /var/run/docker.sock:/tmp/docker.sock:ro
-      # mapping the same data folder, to allow nginx-proxy to fetch images from cache
-      - ./data:/data
+      # mapping cache folder, to persist it independently of the container
+      - ./cache:/var/cache/nginx
       # mapping for letsencrypt companion (note using the current folder)
       - ./nginx/vhost.d:/etc/nginx/vhost.d:ro
       - ./nginx/html:/usr/share/nginx/html
@@ -65,7 +61,7 @@ services:
     depends_on:
       - nginx-proxy
 volumes:
-  data:
+  cache:
     driver: local
 networks:
   app:

recipes/docker-compose/remotecv/docker-compose.yml

@@ -15,10 +15,9 @@ services:
       - CORS_ALLOW_ORIGIN=*
       # returns a webp image if browser Accept headers match
       - AUTO_WEBP=True
-      # Basic thumbor setup to cache both original images and results (after manipulation)
-      # nginx-proxy would detect if there's a cached version and serve it for you automatically
+      # nginx-proxy does caching automatically, so no need to store the result storage cache
       # (this greatly speeds up and saves on CPU)
-      - RESULT_STORAGE=thumbor.result_storages.file_storage
+      - RESULT_STORAGE=thumbor.result_storages.no_storage
       - RESULT_STORAGE_STORES_UNSAFE=True
       # setting mixed storage, and detector storage using redis
       - STORAGE=thumbor.storages.mixed_storage
@@ -32,27 +31,29 @@ services:
       - REDIS_QUEUE_SERVER_HOST=redis
       - REDIS_QUEUE_SERVER_PORT=6379
       - REDIS_QUEUE_SERVER_DB=0
-    volumes:
-      # mounting a /data folder to store cached images
-      - ./data:/data
     links:
       - redis:redis
     restart: always
     networks:
       - app
   nginx-proxy:
-    image: minimalcompact/thumbor-nginx-proxy
+    image: minimalcompact/thumbor-nginx-proxy-cache
     environment:
       # setting the DEFAULT_HOST to the same as the VIRTUAL_HOST above.
       # Makes sure it works irrespective of the host name
       # Normally this won't be necessary, but it helps for testing.
       - DEFAULT_HOST=localhost
+      # optional: control cache memory size (default 500m), cache size (default 10g) and inactive (default 300m)
+      #           see https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_cache_path
+      - PROXY_CACHE_SIZE=10g
+      - PROXY_CACHE_MEMORY_SIZE=500m
+      - PROXY_CACHE_INACTIVE=300m
     volumes:
       # this is essential for nginx-proxy to detect docker containers, scaling etc
       # see https://github.com/jwilder/nginx-proxy
       - /var/run/docker.sock:/tmp/docker.sock:ro
-      # mapping the same data folder, to allow nginx-proxy to fetch images from cache
-      - ./data:/data
+      # mapping cache folder, to persist it independently of the container
+      - ./cache:/var/cache/nginx
     ports:
       - "80:80"
       - "443:443"
@@ -82,7 +83,7 @@ services:
     networks:
       - app
 volumes:
-  data:
+  cache:
     driver: local
 networks:
   app:

recipes/docker-compose/simple/docker-compose.yml

@@ -15,39 +15,40 @@ services:
       - CORS_ALLOW_ORIGIN=*
       # returns a webp image if browser Accept headers match
       - AUTO_WEBP=True
-      # Basic thumbor setup to cache both original images and results (after manipulation)
-      # nginx-proxy would detect if there's a cached version and serve it for you automatically
+      # nginx-proxy does caching automatically, so no need to store the result storage cache
       # (this greatly speeds up and saves on CPU)
-      - RESULT_STORAGE=thumbor.result_storages.file_storage
+      - RESULT_STORAGE=thumbor.result_storages.no_storage
       - RESULT_STORAGE_STORES_UNSAFE=True
       - STORAGE=thumbor.storages.file_storage
-    volumes:
-      # mounting a /data folder to store cached images
-      - ./data:/data
     restart: always
     networks:
       - app
   nginx-proxy:
-    image: minimalcompact/thumbor-nginx-proxy
+    image: minimalcompact/thumbor-nginx-proxy-cache
     environment:
       # setting the DEFAULT_HOST to the same as the VIRTUAL_HOST above.
       # Makes sure it works irrespective of the host name
       # Normally this won't be necessary, but it helps for testing.
       - DEFAULT_HOST=localhost
+      # optional: control cache memory size (default 500m), cache size (default 10g) and inactive (default 300m)
+      #           see https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_cache_path
+      - PROXY_CACHE_SIZE=10g
+      - PROXY_CACHE_MEMORY_SIZE=500m
+      - PROXY_CACHE_INACTIVE=300m
     volumes:
       # this is essential for nginx-proxy to detect docker containers, scaling etc
       # see https://github.com/jwilder/nginx-proxy
       - /var/run/docker.sock:/tmp/docker.sock:ro
-      # mapping the same data folder, to allow nginx-proxy to fetch images from cache
-      - ./data:/data
+      # mapping cache folder, to persist it independently of the container
+      - ./cache:/var/cache/nginx
     ports:
       - "80:80"
       - "443:443"
     restart: always
     networks:
       - app
 volumes:
-  data:
+  cache:
     driver: local
 networks:
   app:

recipes/docker-compose/swarm/docker-compose.yml

@@ -28,7 +28,7 @@ services:
       - STORAGE=thumbor.storages.mixed_storage
       - MIXED_STORAGE_FILE_STORAGE=thumbor.storages.file_storage
       - MIXED_STORAGE_DETECTOR_STORAGE=tc_redis.storages.redis_storage
-      - RESULT_STORAGE=thumbor.result_storages.file_storage
+      - RESULT_STORAGE=thumbor.result_storages.no_storage
       # Cache storage settings
       #
       # storage keeps the original image files cached
@@ -43,7 +43,6 @@ services:
       - REDIS_QUEUE_SERVER_DB=0
     volumes:
       - logs:/logs
-      - data:/data
     deploy:
       # Note this would create 3 instances, each with THUMBOR_NUM_PROCESSES
       # (for docker-compose, this is ignored, but can be controlled using
@@ -55,7 +54,7 @@ services:
     networks:
       - frontend
   nginx-proxy:
-    image: minimalcompact/thumbor-nginx-proxy
+    image: minimalcompact/thumbor-nginx-proxy-cache
     environment:
       # setting the DEFAULT_HOST to the same as the VIRTUAL_HOST above.
       # Makes sure it works irrespective of the host name
@@ -65,8 +64,8 @@ services:
       # this is essential for nginx-proxy to detect docker containers, scaling etc
       # see https://github.com/jwilder/nginx-proxy
       - /var/run/docker.sock:/tmp/docker.sock:ro
-      # mapping the same data folder, to allow nginx-proxy to fetch images from cache
-      - ./data:/data
+      # mapping cache folder, to persist it independently of the container
+      - ./cache:/var/cache/nginx
     ports:
       - "80:80"
       - "443:443"
@@ -99,7 +98,7 @@ services:
       - frontend
       - backend
 volumes:
-  data:
+  cache:
     driver: local
   logs:
     driver: local

tests/nginx-proxy-autowebp.bats

@@ -12,14 +12,23 @@ load_thumbor () {
 }
 
 @test "no webp headers by default even if browser accepts" {
+    rm -rf $BASE/data/*
     load_thumbor
     run bash -c "curl -H 'Accept: image/webp' -sSL -D - http://localhost/unsafe/500x150/i.imgur.com/Nfn80ck.png -o /dev/null |grep 'Content-Type: image/png'"
     [ $status -eq 0 ]
 }
 
 @test "webp headers if AUTO_WEBP is set and browser accepts webp" {
-    export AUTO_WEB=True
+    export AUTO_WEBP=True
+    rm -rf $BASE/data/*
     load_thumbor
-    run bash -c "curl -H 'Accept: image/webp' -sSL -D - http://localhost/unsafe/500x150/i.imgur.com/Nfn80ck.png -o /dev/null |grep 'Content-Type: image/png'"
+    run bash -c "curl -H 'Accept: image/webp' -sSL -D - http://localhost/unsafe/500x150/i.imgur.com/Nfn80ck.png -o /dev/null |grep 'Content-Type: image/webp'"
+    [ $status -eq 0 ]
+}
+
+@test "NOTE: nginx proxy cache can override AUTO_WEBP (Must clear the cache when you change the settings!)" {
+    export AUTO_WEBP=False
+    load_thumbor
+    run bash -c "curl -H 'Accept: image/webp' -sSL -D - http://localhost/unsafe/500x150/i.imgur.com/Nfn80ck.png -o /dev/null |grep 'Content-Type: image/webp'"
     [ $status -eq 0 ]
 }