Add a bunch of tools to backup/restore/init-db + dedicated docker service

.gitignore

@@ -18,7 +18,9 @@ data/*.json
 !data/dictionary.json
 !data/networks.json
 
+*.bak
 
 # files generated for server-side purposes
 
-src/views/*.js
+src/views/*.js
+

README.md

@@ -18,11 +18,15 @@ Install:
 npm install
 ````
 
+#### Initialize the database
+
 Put your datafiles in `data`, then do:
 
 ```
-docker-compose -f docker-compose-init-db.yml rm -f
-docker-compose -f docker-compose-init-db.yml up
+npm run serve-dev # or npm run serve-prod in production environment
+
+# then in another window
+docker exec 6element_backup_1 tools/init-database.js
 docker-compose -f docker-compose-load-db.yml up
 ```
 
@@ -40,4 +44,34 @@ npm run start-dev
 
 ````
 npm start
-````
+````
+
+
+## Database management
+
+A backup-specific docker service is available to be used by a host machine to take care of backups
+
+### Create tables (and drop everything beforehand)
+
+```
+docker exec 6element_backup_1 tools/init-database.js
+```
+
+
+### Backup
+
+```
+docker exec 6element_backup_1 tools/backup.js > backups/.bak
+```
+
+`docker exec 6element_backup_1 tools/backup.js` outputs the backup to stdout.
+What to do with the backup data (file, send to the network, etc.) and when to generate it (daily basis in a cron tab, once, etc.) is left to the caller.
+
+### Restore
+
+The backup docker service has a directory where it expects a single backup file for restore use at `/usr/6element-backups/6element.bak` (in the container, so you don't need to know about this). However, you should provide in the docker-compose file which directory in the host machine corresponds to this directory
+
+```
+docker exec 6element_backup_1 tools/restore.js
+```
+

compose-dev.yml

@@ -11,7 +11,6 @@ api:
       - VIRTUAL_PORT=3500
       - POSTGRES_USER=postgres
       - POSTGRES_PASSWORD=postgres
-      - PGPASSWORD=postgres
       - NODE_ENV=development
    volumes:
       - ./:/6element
@@ -26,3 +25,16 @@ db:
       - POSTGRES_USER=postgres
       - POSTGRES_PASSWORD=postgres
    tty: true
+
+backup:
+   build: ./dockerfiles
+   dockerfile: node
+   links:
+      - db
+   environment:
+      - POSTGRES_USER=postgres
+      - PGPASSWORD=postgres
+   volumes:
+      - ./:/6element
+      - ./backups:/usr/6element-backups:ro
+   tty: true

compose-prod.yml

@@ -11,7 +11,6 @@ api:
       - VIRTUAL_PORT=3500
       - POSTGRES_USER=postgres
       - POSTGRES_PASSWORD=postgres
-      - PGPASSWORD=postgres
       - NODE_ENV=production
    volumes:
       - ./:/6element
@@ -26,3 +25,16 @@ db:
       - POSTGRES_USER=postgres
       - POSTGRES_PASSWORD=postgres
    tty: true
+
+backup:
+   build: ./dockerfiles
+   dockerfile: node
+   links:
+      - db
+   environment:
+      - POSTGRES_USER=postgres
+      - PGPASSWORD=postgres
+   volumes:
+      - ./:/6element
+      - ./backups:/usr/6element-backups:ro
+   tty: true

dockerfiles/node

@@ -1,4 +1,4 @@
-FROM node:4.1.2
+FROM node:4.2
 
 MAINTAINER Alexandre Vallette <alexandre.vallette@ants.builders>
 

package.json

@@ -15,9 +15,9 @@
     "build:jsx-for-server-side": "babel src/views/*.jsx --out-dir .",
     "stop-dev": "docker-compose -f compose-dev.yml stop",
     "stop-prod": "docker-compose -f compose-prod.yml stop",
-    "start-prod": "docker-compose -f compose-prod.yml up -d --force-recreate",
+    "start": "docker-compose -p 6element -f compose-prod.yml up --force-recreate",
     "prestart": "npm run build-prod",
-    "serve-dev": "docker-compose -f compose-dev.yml up --force-recreate",
+    "serve-dev": "docker-compose -p 6element -f compose-dev.yml up --force-recreate",
     "start-dev": "npm-run-all --parallel serve-dev watch"
   },
   "babel": {

server/database/management/connectToDB.js

@@ -2,7 +2,17 @@
 
 var pg = require('pg');
 
-var conString = 'postgres://postgres:' + process.env.POSTGRES_PASSWORD + '@' + process.env.DB_PORT_5432_TCP_ADDR + ':5432/postgres';
+var conString = [
+    'postgres://',
+    process.env.POSTGRES_USER,
+    ':', 
+    process.env.POSTGRES_PASSWORD,
+    '@',
+    process.env.DB_PORT_5432_TCP_ADDR,
+    ':',
+    process.env.DB_PORT_5432_TCP_PORT,
+    '/postgres'
+].join('');
 
 console.log('conString', conString);
 

server/database/management/createTables.js

@@ -1,41 +1,20 @@
 'use strict';
 
 var fs = require('fs');
-var zlib = require('zlib');
-var spawn = require('child_process').spawn;
 
 var connectToDB = require('./connectToDB.js');
 
+var createTableScript = fs.readFileSync( require.resolve('./createTables.sql') ).toString();
+
+
 module.exports = function(){
     return new Promise(function(resolve, reject){
         connectToDB()
         .then(function(db){
-            var sqlFile;
-            if (process.env.BACKUP){
-                // if a environement variable is set, use it to load some data
-                console.log('== A dump file is specified, loading the data. ==');
-                sqlFile = '/pheromon/data/' + process.env.BACKUP;
-                var gzip = zlib.createGunzip();
-                var readStream = fs.createReadStream(sqlFile);
-                var proc = spawn('psql', ['-p', process.env.DB_PORT_5432_TCP_PORT, '-h', process.env.DB_PORT_5432_TCP_ADDR, '-U', process.env.POSTGRES_USER, '-d', process.env.POSTGRES_USER]);
-                readStream
-                    .pipe(gzip)
-                    .pipe(proc.stdin);
-
-                // This code doesn't work because gzip is too slow
-                // readStream.on('close', function(){
-                //     resolve();
-                // })
-
-                setTimeout(resolve, 3000); // <-- not a beautiful patch, but it works
-
-            } else {
-                console.log('== Resetting the database ==');
-                var createTableScript = fs.readFileSync( require.resolve('./createTables.sql') ).toString();
-                db.query(createTableScript, function(err, result) {
-                    if(err) reject(err); else resolve(result);
-                });
-            }
+            console.log('== Creating the database tables ==');
+            db.query(createTableScript, function(err, result) {
+                if(err) reject(err); else resolve(result);
+            });
 
         })
         .catch(function(err){