diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..e3c31b6 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,5 @@ +* +!build-contract +!package.json +!parsetargets +!nodejs diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..fa3df9b --- /dev/null +++ b/.editorconfig @@ -0,0 +1,7 @@ +root = true + +[*] +charset = utf-8 +trim_trailing_whitespace = true +indent_style = space +indent_size = 2 diff --git a/Dockerfile b/Dockerfile index 3fbfd4f..3de80c1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM yolean/node@sha256:f033123ae2292d60769e5b8eff94c4b7b9d299648d0d23917319c0743029c5ef +FROM yolean/node@sha256:e591eac6f5d1f07876bd63bba2bbd1c1218521c5ed5312692851597b47089775 ENV docker_version=17.09.1~ce-0~debian ENV compose_version=1.21.0 compose_sha256=af639f5e9ca229442c8738135b5015450d56e2c1ae07c0aaa93b7da9fe09c2b0 @@ -23,8 +23,11 @@ RUN curl -L https://github.com/docker/compose/releases/download/$compose_version VOLUME /source WORKDIR /source -COPY package.json build-contract parsetargets /usr/src/app/ +COPY package.json /usr/src/app/ +RUN cd /usr/src/app/ && npm install --production +COPY build-contract parsetargets /usr/src/app/ COPY nodejs /usr/src/app/nodejs -RUN cd /usr/src/app/ && npm install && npm link +RUN cd /usr/src/app/ && npm link --only=production + ENTRYPOINT ["build-contract"] CMD ["push"] diff --git a/build-contract b/build-contract index acbffb1..bceb3b3 100755 --- a/build-contract +++ b/build-contract @@ -32,7 +32,7 @@ fi function wait_for_contract { sleep 3 - compose_name=$(echo "$1" | sed 's/[^A-Za-z0-9]//g') + compose_name=$(echo "$1" | sed 's/[^A-Za-z0-9_-]//g') # Count the number of failed containers # NOTE: Assumes no other build contract process is running at the same time filters="-f label=com.yolean.build-contract -f name=$compose_name" diff --git a/build-contracts/docker-compose.yml b/build-contracts/docker-compose.yml index 3a5d435..0b57cbe 100644 --- a/build-contracts/docker-compose.yml +++ b/build-contracts/docker-compose.yml @@ -1,12 +1,25 @@ version: "2" services: + build-contract: + image: yolean/build-contract + build: + context: ../ + labels: + - com.yolean.build-target + + unittest: + build: + context: ../ + dockerfile: ./build-contracts/unittest/Dockerfile + labels: + - com.yolean.build-contract + nginx: image: nginx + client: build: ./client - image: localhost:5000/build-contract-test/client:$PUSH_TAG environment: TEST_EXIT_CODE: $TEST_EXIT_CODE labels: - com.yolean.build-contract: "" - com.yolean.build-target: "" \ No newline at end of file + - com.yolean.build-contract diff --git a/build-contracts/unittest/Dockerfile b/build-contracts/unittest/Dockerfile new file mode 100644 index 0000000..edf0fdf --- /dev/null +++ b/build-contracts/unittest/Dockerfile @@ -0,0 +1,11 @@ +FROM yolean/node@sha256:e591eac6f5d1f07876bd63bba2bbd1c1218521c5ed5312692851597b47089775 + +WORKDIR /usr/src/app + +COPY package.json . + +RUN npm install + +COPY . . + +RUN npm run unittest diff --git a/nodejs/build-contract-packagelock b/nodejs/build-contract-packagelock index 9c7b431..16f07f8 100755 --- a/nodejs/build-contract-packagelock +++ b/nodejs/build-contract-packagelock @@ -6,5 +6,10 @@ $SCRIPTPATH/build-contract-predockerbuild cd npm-monorepo/ci npm install --production --package-lock-only --ignore-scripts +echo "------- packagelock details -------" +cat ../../npm-monorepo/ci/package.json +echo "" +shasum -a 256 ../../npm-monorepo/ci/npm-monorepo/* +echo "------- packagelock details -------" mv package-lock.json ../../ cd ../../ diff --git a/nodejs/build-contract-predockerbuild b/nodejs/build-contract-predockerbuild index fad3f35..2e4ee8d 100755 --- a/nodejs/build-contract-predockerbuild +++ b/nodejs/build-contract-predockerbuild @@ -1,6 +1,7 @@ #!/usr/bin/env node const path = require('path'); const fs = require('fs'); +const mnpm = require('./mnpm'); const npmLib = new Promise((resolve,reject) => { // ls -la $(which npm), or we could do something like https://stackoverflow.com/a/25106648/113009 @@ -43,8 +44,7 @@ function npmPackage(modulePath, cb) { npmLib.then(npm => { npm.commands.pack([], (err, result) => { if (err) return cb(err); - console.debug('# pack result', modulePath, JSON.stringify(result)); - const name = result[0]; + const name = result[0].filename; fs.stat(name, (err, stats) => { if (err) console.error('# npm pack failed to produce the result file', npm, process.cwd()); cb(err, err ? undefined : name); @@ -53,10 +53,6 @@ function npmPackage(modulePath, cb) { }); } -function stringifyPackageJson(package) { - return JSON.stringify(package, null, ' '); -} - let package = require(path.join(dir,'package.json')); let monorepoDeps = Object.keys(package.dependencies).filter( dep => /^file:\.\.\//.test(package.dependencies[dep])); @@ -76,10 +72,10 @@ mk(dir, mk.bind(null, mdir, mk.bind(null, cidir, mk.bind(null, cimdir, err => { const completed = () => { process.chdir(dir); // restore after npm fs.unlink(path.join(cimdir, 'package.json'), err => err && console.error('Failed to clean up after sourceless tgz pack', err)); - fs.writeFile(path.join(mdir, 'package.json'), stringifyPackageJson(package), + fs.writeFile(path.join(mdir, 'package.json'), mnpm.stringifyPackageJson(package), err => { if (err) throw err; }); const ciPackage = getCiPackage(package); - fs.writeFile(path.join(cidir, 'package.json'), stringifyPackageJson(ciPackage), + fs.writeFile(path.join(cidir, 'package.json'), mnpm.stringifyPackageJson(ciPackage), err => { if (err) throw err; }); fs.unlink(path.join(cimdir, '.npmignore'), err => err && console.error(err)); }; @@ -106,10 +102,21 @@ mk(dir, mk.bind(null, mdir, mk.bind(null, cidir, mk.bind(null, cimdir, err => { package.dependencies[dep] = `file:npm-monorepo/${tgzname}`; let depCiPackage = getCiPackage(depPackage); - fs.writeFile(path.join(cimdir, 'package.json'), stringifyPackageJson(depCiPackage), err => { - process.chdir(cimdir); // for npm - npmPackage(cimdir, (err, tgzname) => { + // We could probably speed things up here by using tar-stream, and maybe set permissions + let depPpJson = path.join(cimdir, 'package.json'); + fs.writeFile(depPpJson, mnpm.stringifyPackageJson(depCiPackage), err => { + mnpm.writeProdPackageTgzWithDeterministicHash({ + packageJsonObject: getCiPackage(depPackage), + filePath: path.join(cimdir, tgzname) + }).then(() => { console.log('# Created monorepo sourceless tarball for npm ci', cimdir, tgzname); + console.log('------- debug info -------'); + process.chdir(cimdir); // for npm + require('child_process').execSync('tar xvzf ' + tgzname, {stdio:[0,1,2]}); + require('child_process').execSync('ls -la package/', {stdio:[0,1,2]}); + require('child_process').execSync('shasum package/package.json', {stdio:[0,1,2]}); + require('child_process').execSync('rm -rf package/', {stdio:[0,1,2]}); + console.log('------- ---------- -------'); next(monorepoDeps.shift()); }); }); diff --git a/nodejs/mnpm.js b/nodejs/mnpm.js new file mode 100644 index 0000000..8674c82 --- /dev/null +++ b/nodejs/mnpm.js @@ -0,0 +1,32 @@ +const fs = require('fs'); +const tar = require('tar-stream'); +const zlib = require('zlib'); + +const tarContentMtime = new Date(946684800000); + +function stringifyPackageJson(packageJsonObject) { + return JSON.stringify(packageJsonObject, null, ' ') + '\n'; +} + +async function writeProdPackageTgzWithDeterministicHash({packageJsonObject, filePath}) { + const content = stringifyPackageJson(packageJsonObject); + const pack = tar.pack(); + const p = pack.entry({ + name: 'package/package.json', + mtime: tarContentMtime + }, content); + const fileStream = fs.createWriteStream(filePath); + const completed = new Promise((resolve, reject) => { + fileStream.on('close', resolve); + }); + pack.finalize(); + pack + .pipe(zlib.createGzip()) + .pipe(fileStream); + return completed; +}; + +module.exports = { + stringifyPackageJson, + writeProdPackageTgzWithDeterministicHash +}; diff --git a/nodejs/mnpm.spec.js b/nodejs/mnpm.spec.js new file mode 100644 index 0000000..b2c6461 --- /dev/null +++ b/nodejs/mnpm.spec.js @@ -0,0 +1,166 @@ +const os = require('os'); +const path = require('path'); +const fs = require('fs'); +const crypto = require('crypto'); +const zlib = require('zlib'); +const stream = require('stream'); +const tar = require('tar-stream'); + +const mnpm = require('./mnpm'); + +describe("stringifyPackageJson", () => { + + it("Uses two whitespaces to indent (the Yolean convention) and adds a trailing newline", () => { + const string = mnpm.stringifyPackageJson({name: 'test-module'}); + expect(string).toBe('{\n "name": "test-module"\n}\n'); + }); + +}); + +describe("Our choice of gzip function", () => { + + it("Is platform independent wrt result checksum", () => { + const blob = new stream.PassThrough(); + const sha256 = crypto.createHash('sha256'); + const result = new stream.PassThrough(); + result.on('data', d => sha256.update(d)); + result.on('end', () => expect(sha256.digest('hex')).toBe( + 'c5f9a2352dadba9488900ba6ede0133270e12350ffa6d6ebbdefef9ee6aa2238')); + // Note that this differs from `echo 'x' | gzip - | shasum -a 256 -` + blob.pipe(zlib.createGzip()).pipe(result); + blob.end('x\n'); + }); + + // https://github.com/nodejs/node/issues/12244 + it("Results may depend on zlib version", () => { + expect(process.versions.zlib).toBe('1.2.11'); + }); + + it("Results may depend on zlib options", () => { + const options = { + windowBits: 14, memLevel: 7, + level: zlib.constants.Z_BEST_SPEED, + strategy: zlib.constants.Z_FIXED + }; + const blob = new stream.PassThrough(); + const sha256 = crypto.createHash('sha256'); + const result = new stream.PassThrough(); + result.on('data', d => sha256.update(d)); + result.on('end', () => expect(sha256.digest('hex')).toBe( + '270b40b49af0dcc1de9631f231d022c93c07f29d2940e14d22ffdd797165e24f')); + // Note that this differs from `echo 'x' | gzip - | shasum -a 256 -` + blob.pipe(zlib.createGzip(options)).pipe(result); + blob.end('x\n'); + }); + + it("Results may depend on zlib compression level", () => { + const options = { + level: zlib.constants.Z_BEST_COMPRESSION + }; + const blob = new stream.PassThrough(); + const sha256 = crypto.createHash('sha256'); + const result = new stream.PassThrough(); + result.on('data', d => sha256.update(d)); + result.on('end', () => expect(sha256.digest('hex')).toBe( + '1437e4b499b5063c9530244d1655dba6986bcbd1258f81f122bdc4c983058ef4')); + // Note that this differs from `echo 'x' | gzip - | shasum -a 256 -` + blob.pipe(zlib.createGzip(options)).pipe(result); + blob.end('x\n'); + }); + + it("Results may be more platform independent with no compression", () => { + const options = { + level: zlib.constants.Z_NO_COMPRESSION + }; + const blob = new stream.PassThrough(); + const sha256 = crypto.createHash('sha256'); + const result = new stream.PassThrough(); + result.on('data', d => sha256.update(d)); + result.on('end', () => expect(sha256.digest('hex')).toBe( + '35932a249baf5fe47b9fecaa3482309e447ed8d9b01e207a6ce2846724006784')); + // Note that this differs from `echo 'x' | gzip - | shasum -a 256 -` + blob.pipe(zlib.createGzip(options)).pipe(result); + blob.end('x\n'); + }); + +}); + +describe("writeProdPackageTgzWithDeterministicHash", () => { + + it("Writes a file", async () => { + const filePath = path.join(os.tmpdir(), 'build-contract-test-mnpm-' + Date.now() + '.tgz'); + await mnpm.writeProdPackageTgzWithDeterministicHash({ + packageJsonObject: { + "dependencies": { + "build-contract": "1.5.0" + } + }, + filePath + }); + const stat = await fs.promises.stat(filePath); + // we base these assertions on a test result, not on npm pack output (which differs) + // and use the assertions to see if something changes over time or across platforms + expect(stat.size).toBe(154); + const tgz = await fs.promises.readFile(filePath); + const sha256 = crypto.createHash('sha256'); + sha256.update(tgz); + expect(sha256.digest('hex')).toBe('3be69fccaf4716df00adee93c219cfe44f1425aa968d33b6a3a4e725192586be'); + const sha512 = crypto.createHash('sha512'); + sha512.update(tgz); + expect(sha512.digest('base64')).toBe('M24fZ1mSsZYX8dSCGSd54842GgAKd80xInWqNUhSZH1/hx7syKOOx05qMhD8avcFdXNnDMG/N2i/YZJFJNW6rQ=='); + await fs.promises.unlink(filePath); + }); + + it("Entries are deterministic", done => { + const filePath = path.join(os.tmpdir(), 'build-contract-test-mnpm-' + Date.now() + '.tgz'); + const packageJsonObject = { + "dependencies": { + "build-contract": "1.5.0" + } + }; + mnpm.writeProdPackageTgzWithDeterministicHash({ + packageJsonObject, + filePath + }).then(() => { + const extract = tar.extract(); + let count = 0; + + extract.on('entry', function(header, stream, next) { + count++; + expect(header.name).toBe('package/package.json'); + expect(header.mode).toBe(parseInt('0644',8)); + expect(header.uid).toBe(0); + expect(header.gid).toBe(0); + expect(header.size).toBe(mnpm.stringifyPackageJson(packageJsonObject).length); + expect(header.type).toBe('file'); + expect(header.linkname).toBeNull(); + expect(header.uname).toBe(''); + expect(header.gname).toBe(''); + expect(header.devmajor).toBe(0); + expect(header.devminor).toBe(0); + expect(header.mtime).toBeInstanceOf(Date); + expect(header.mtime.getTime()).toBe(946684800000); + expect(Object.keys(header).length).toBe(12); + + stream.on('end', function() { + // previous test asserted tgz checksum so we don't need to check content here + next(); + }) + + stream.resume(); + }); + + extract.on('finish', () => { + expect(count).toBe(1); + fs.promises.unlink(filePath).then(done); + }); + + fs.createReadStream(filePath) + .pipe(zlib.createGunzip({ + + })) + .pipe(extract); + }); + }); + +}); diff --git a/nodejs/zlib-choice.js b/nodejs/zlib-choice.js new file mode 100644 index 0000000..b7a3939 --- /dev/null +++ b/nodejs/zlib-choice.js @@ -0,0 +1,6 @@ +const zlib = require('zlib'); +const pakoStreams = require('browserify-zlib'); + +module.exports.createGzip = pakoStreams.createGzip; +module.exports.createGunzip = zlib.createGunzip; +module.exports.constants = zlib.constants; diff --git a/package.json b/package.json index 316bab7..7eb81a8 100644 --- a/package.json +++ b/package.json @@ -2,12 +2,6 @@ "name": "build-contract", "version": "1.4.0", "description": "Defines a successful build and test run for a microservice, from source to docker push", - "main": "build-contract", - "bin": { - "build-contract": "./build-contract", - "build-contract-predockerbuild": "./nodejs/build-contract-predockerbuild", - "packagelock": "./nodejs/build-contract-packagelock" - }, "repository": { "type": "git", "url": "git+https://github.com/Yolean/build-contract.git" @@ -18,10 +12,27 @@ "url": "https://github.com/Yolean/build-contract/issues" }, "homepage": "https://github.com/Yolean/build-contract#readme", + "main": "build-contract", + "bin": { + "build-contract": "./build-contract", + "build-contract-predockerbuild": "./nodejs/build-contract-predockerbuild", + "build-contract-packagelock": "./nodejs/build-contract-packagelock" + }, + "scripts": { + "unittest": "./node_modules/.bin/jest", + "build-contract-predockerbuild": "./nodejs/build-contract-predockerbuild" + }, + "engines": { + "node": ">=10.1.0" + }, "dependencies": { + "tar-stream": "1.6.1", "yamljs": "0.2.8" }, "peerDependencies": { - "npm": "5.8.0" + "npm": "6.0.1" + }, + "devDependencies": { + "jest": "22.4.4" } }