Try to produce consistent monorepo tgz hashes #36

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged

atamon merged 18 commits into master from packagelock-hashes

May 22, 2018

.dockerignore

-Original file line number
+Diff line change
@@ -0,0 +1,5 @@
+    *
+    !build-contract
+    !package.json
+    !parsetargets
+    !nodejs

.editorconfig

-Original file line number
+Diff line change
@@ -0,0 +1,7 @@
+    root = true
+    [*]
+    charset = utf-8
+    trim_trailing_whitespace = true
+    indent_style = space
+    indent_size = 2

Dockerfile

-Original file line number
+Diff line change
@@ -1,4 +1,4 @@
-    FROM yolean/node@sha256:f033123ae2292d60769e5b8eff94c4b7b9d299648d0d23917319c0743029c5ef
+    FROM yolean/node@sha256:e591eac6f5d1f07876bd63bba2bbd1c1218521c5ed5312692851597b47089775
     ENV docker_version=17.09.1~ce-0~debian
     ENV compose_version=1.21.0 compose_sha256=af639f5e9ca229442c8738135b5015450d56e2c1ae07c0aaa93b7da9fe09c2b0
@@ Expand All @@
     VOLUME /source
     WORKDIR /source
-    COPY package.json build-contract parsetargets /usr/src/app/
+    COPY package.json /usr/src/app/
+    RUN cd /usr/src/app/ && npm install --production
+    COPY build-contract parsetargets /usr/src/app/
     COPY nodejs /usr/src/app/nodejs
-    RUN cd /usr/src/app/ && npm install && npm link
+    RUN cd /usr/src/app/ && npm link --only=production
     ENTRYPOINT ["build-contract"]
     CMD ["push"]

build-contract

-Original file line number
+Diff line change
@@ Expand Up / @@ -32,7 +32,7 @@ fi @@
     function wait_for_contract {
       sleep 3
-      compose_name=$(echo "$1" | sed 's/[^A-Za-z0-9]//g')
+      compose_name=$(echo "$1" | sed 's/[^A-Za-z0-9_-]//g')
       # Count the number of failed containers
       # NOTE: Assumes no other build contract process is running at the same time
       filters="-f label=com.yolean.build-contract -f name=$compose_name"
@@ Expand Down @@

build-contracts/docker-compose.yml

-Original file line number
+Diff line change
@@ -1,12 +1,25 @@
     version: "2"
     services:
+      build-contract:
+        image: yolean/build-contract
+        build:
+          context: ../
+        labels:
+        - com.yolean.build-target
+      unittest:
+        build:
+          context: ../
+          dockerfile: ./build-contracts/unittest/Dockerfile
+        labels:
+        - com.yolean.build-contract
       nginx:
         image: nginx
       client:
         build: ./client
-        image: localhost:5000/build-contract-test/client:$PUSH_TAG
         environment:
           TEST_EXIT_CODE: $TEST_EXIT_CODE
         labels:
-          com.yolean.build-contract: ""
-          com.yolean.build-target: ""
+        - com.yolean.build-contract

build-contracts/unittest/Dockerfile

-Original file line number
+Diff line change
@@ -0,0 +1,11 @@
+    FROM yolean/node@sha256:e591eac6f5d1f07876bd63bba2bbd1c1218521c5ed5312692851597b47089775
+    WORKDIR /usr/src/app
+    COPY package.json .
+    RUN npm install
+    COPY . .
+    RUN npm run unittest

nodejs/build-contract-packagelock

-Original file line number
+Diff line change
@@ Expand Up / @@ -6,5 +6,10 @@ $SCRIPTPATH/build-contract-predockerbuild @@
     cd npm-monorepo/ci
     npm install --production --package-lock-only --ignore-scripts
+    echo "------- packagelock details -------"
+    cat ../../npm-monorepo/ci/package.json
+    echo ""
+    shasum -a 256 ../../npm-monorepo/ci/npm-monorepo/*
+    echo "------- packagelock details -------"
     mv package-lock.json ../../
     cd ../../

nodejs/build-contract-predockerbuild

-Original file line number
+Diff line change
@@ -1,6 +1,7 @@
     #!/usr/bin/env node
     const path = require('path');
     const fs = require('fs');
+    const mnpm = require('./mnpm');
     const npmLib = new Promise((resolve,reject) => {
       // ls -la $(which npm), or we could do something like https://stackoverflow.com/a/25106648/113009
@@ Expand Down Expand Up / @@ -43,8 +44,7 @@ function npmPackage(modulePath, cb) { @@
       npmLib.then(npm => {
         npm.commands.pack([], (err, result) => {
           if (err) return cb(err);
-          console.debug('# pack result', modulePath, JSON.stringify(result));
-          const name = result[0];
+          const name = result[0].filename;
           fs.stat(name, (err, stats) => {
             if (err) console.error('# npm pack failed to produce the result file', npm, process.cwd());
             cb(err, err ? undefined : name);
@@ Expand All / @@ -53,10 +53,6 @@ function npmPackage(modulePath, cb) { @@
       });
     }
-    function stringifyPackageJson(package) {
-      return JSON.stringify(package, null, '  ');
-    }
     let package = require(path.join(dir,'package.json'));
     let monorepoDeps = Object.keys(package.dependencies).filter(
       dep => /^file:\.\.\//.test(package.dependencies[dep]));
@@ Expand All @@
       const completed = () => {
         process.chdir(dir); // restore after npm
         fs.unlink(path.join(cimdir, 'package.json'), err => err && console.error('Failed to clean up after sourceless tgz pack', err));
-        fs.writeFile(path.join(mdir, 'package.json'), stringifyPackageJson(package),
+        fs.writeFile(path.join(mdir, 'package.json'), mnpm.stringifyPackageJson(package),
           err => { if (err) throw err; });
         const ciPackage = getCiPackage(package);
-        fs.writeFile(path.join(cidir, 'package.json'), stringifyPackageJson(ciPackage),
+        fs.writeFile(path.join(cidir, 'package.json'), mnpm.stringifyPackageJson(ciPackage),
           err => { if (err) throw err; });
         fs.unlink(path.join(cimdir, '.npmignore'), err => err && console.error(err));
       };
@@ Expand All @@
             package.dependencies[dep] = `file:npm-monorepo/${tgzname}`;
             let depCiPackage = getCiPackage(depPackage);
-            fs.writeFile(path.join(cimdir, 'package.json'), stringifyPackageJson(depCiPackage), err => {
-              process.chdir(cimdir); // for npm
-              npmPackage(cimdir, (err, tgzname) => {
+            // We could probably speed things up here by using tar-stream, and maybe set permissions
+            let depPpJson = path.join(cimdir, 'package.json');
+            fs.writeFile(depPpJson, mnpm.stringifyPackageJson(depCiPackage), err => {
+              mnpm.writeProdPackageTgzWithDeterministicHash({
+                packageJsonObject: getCiPackage(depPackage),
+                filePath: path.join(cimdir, tgzname)
+              }).then(() => {
                 console.log('# Created monorepo sourceless tarball for npm ci', cimdir, tgzname);
+                console.log('------- debug info -------');
+                process.chdir(cimdir); // for npm
+                require('child_process').execSync('tar xvzf ' + tgzname, {stdio:[0,1,2]});
+                require('child_process').execSync('ls -la package/', {stdio:[0,1,2]});
+                require('child_process').execSync('shasum package/package.json', {stdio:[0,1,2]});
+                require('child_process').execSync('rm -rf package/', {stdio:[0,1,2]});
+                console.log('------- ---------- -------');
                 next(monorepoDeps.shift());
               });
             });
@@ Expand Down @@

nodejs/mnpm.js

-Original file line number
+Diff line change
@@ -0,0 +1,32 @@
+    const fs = require('fs');
+    const tar = require('tar-stream');
+    const zlib = require('zlib');
+    const tarContentMtime = new Date(946684800000);
+    function stringifyPackageJson(packageJsonObject) {
+      return JSON.stringify(packageJsonObject, null, '  ') + '\n';
+    }
+    async function writeProdPackageTgzWithDeterministicHash({packageJsonObject, filePath}) {
+      const content = stringifyPackageJson(packageJsonObject);
+      const pack = tar.pack();
+      const p = pack.entry({
+        name: 'package/package.json',
+        mtime: tarContentMtime
+      }, content);
+      const fileStream = fs.createWriteStream(filePath);
+      const completed = new Promise((resolve, reject) => {
+        fileStream.on('close', resolve);
+      });
+      pack.finalize();
+      pack
+        .pipe(zlib.createGzip())
+        .pipe(fileStream);
+      return completed;
+    };
+    module.exports = {
+      stringifyPackageJson,
+      writeProdPackageTgzWithDeterministicHash
+    };

nodejs/mnpm.spec.js

-Original file line number
+Diff line change
@@ -0,0 +1,166 @@
+    const os = require('os');
+    const path = require('path');
+    const fs = require('fs');
+    const crypto = require('crypto');
+    const zlib = require('zlib');
+    const stream = require('stream');
+    const tar = require('tar-stream');
+    const mnpm = require('./mnpm');
+    describe("stringifyPackageJson", () => {
+      it("Uses two whitespaces to indent (the Yolean convention) and adds a trailing newline", () => {
+        const string = mnpm.stringifyPackageJson({name: 'test-module'});
+        expect(string).toBe('{\n  "name": "test-module"\n}\n');
+      });
+    });
+    describe("Our choice of gzip function", () => {
+      it("Is platform independent wrt result checksum", () => {
+        const blob = new stream.PassThrough();
+        const sha256 = crypto.createHash('sha256');
+        const result = new stream.PassThrough();
+        result.on('data', d => sha256.update(d));
+        result.on('end', () => expect(sha256.digest('hex')).toBe(
+          'c5f9a2352dadba9488900ba6ede0133270e12350ffa6d6ebbdefef9ee6aa2238'));
+        // Note that this differs from `echo 'x' | gzip - | shasum -a 256 -`
+        blob.pipe(zlib.createGzip()).pipe(result);
+        blob.end('x\n');
+      });
+      // https://github.com/nodejs/node/issues/12244
+      it("Results may depend on zlib version", () => {
+        expect(process.versions.zlib).toBe('1.2.11');
+      });
+      it("Results may depend on zlib options", () => {
+        const options = {
+          windowBits: 14, memLevel: 7,
+          level: zlib.constants.Z_BEST_SPEED,
+          strategy: zlib.constants.Z_FIXED
+        };
+        const blob = new stream.PassThrough();
+        const sha256 = crypto.createHash('sha256');
+        const result = new stream.PassThrough();
+        result.on('data', d => sha256.update(d));
+        result.on('end', () => expect(sha256.digest('hex')).toBe(
+          '270b40b49af0dcc1de9631f231d022c93c07f29d2940e14d22ffdd797165e24f'));
+        // Note that this differs from `echo 'x' | gzip - | shasum -a 256 -`
+        blob.pipe(zlib.createGzip(options)).pipe(result);
+        blob.end('x\n');
+      });
+      it("Results may depend on zlib compression level", () => {
+        const options = {
+          level: zlib.constants.Z_BEST_COMPRESSION
+        };
+        const blob = new stream.PassThrough();
+        const sha256 = crypto.createHash('sha256');
+        const result = new stream.PassThrough();
+        result.on('data', d => sha256.update(d));
+        result.on('end', () => expect(sha256.digest('hex')).toBe(
+          '1437e4b499b5063c9530244d1655dba6986bcbd1258f81f122bdc4c983058ef4'));
+        // Note that this differs from `echo 'x' | gzip - | shasum -a 256 -`
+        blob.pipe(zlib.createGzip(options)).pipe(result);
+        blob.end('x\n');
+      });
+      it("Results may be more platform independent with no compression", () => {
+        const options = {
+          level: zlib.constants.Z_NO_COMPRESSION
+        };
+        const blob = new stream.PassThrough();
+        const sha256 = crypto.createHash('sha256');
+        const result = new stream.PassThrough();
+        result.on('data', d => sha256.update(d));
+        result.on('end', () => expect(sha256.digest('hex')).toBe(
+          '35932a249baf5fe47b9fecaa3482309e447ed8d9b01e207a6ce2846724006784'));
+        // Note that this differs from `echo 'x' | gzip - | shasum -a 256 -`
+        blob.pipe(zlib.createGzip(options)).pipe(result);
+        blob.end('x\n');
+      });
+    });
+    describe("writeProdPackageTgzWithDeterministicHash", () => {
+      it("Writes a file", async () => {
+        const filePath = path.join(os.tmpdir(), 'build-contract-test-mnpm-' + Date.now() + '.tgz');
+        await mnpm.writeProdPackageTgzWithDeterministicHash({
+          packageJsonObject: {
+            "dependencies": {
+              "build-contract": "1.5.0"
+            }
+          },
+          filePath
+        });
+        const stat = await fs.promises.stat(filePath);
+        // we base these assertions on a test result, not on npm pack output (which differs)
+        // and use the assertions to see if something changes over time or across platforms
+        expect(stat.size).toBe(154);
+        const tgz = await fs.promises.readFile(filePath);
+        const sha256 = crypto.createHash('sha256');
+        sha256.update(tgz);
+        expect(sha256.digest('hex')).toBe('3be69fccaf4716df00adee93c219cfe44f1425aa968d33b6a3a4e725192586be');
+        const sha512 = crypto.createHash('sha512');
+        sha512.update(tgz);
+        expect(sha512.digest('base64')).toBe('M24fZ1mSsZYX8dSCGSd54842GgAKd80xInWqNUhSZH1/hx7syKOOx05qMhD8avcFdXNnDMG/N2i/YZJFJNW6rQ==');
+        await fs.promises.unlink(filePath);
+      });
+      it("Entries are deterministic", done => {
+        const filePath = path.join(os.tmpdir(), 'build-contract-test-mnpm-' + Date.now() + '.tgz');
+        const packageJsonObject = {
+          "dependencies": {
+            "build-contract": "1.5.0"
+          }
+        };
+        mnpm.writeProdPackageTgzWithDeterministicHash({
+          packageJsonObject,
+          filePath
+        }).then(() => {
+          const extract = tar.extract();
+          let count = 0;
+          extract.on('entry', function(header, stream, next) {
+            count++;
+            expect(header.name).toBe('package/package.json');
+            expect(header.mode).toBe(parseInt('0644',8));
+            expect(header.uid).toBe(0);
+            expect(header.gid).toBe(0);
+            expect(header.size).toBe(mnpm.stringifyPackageJson(packageJsonObject).length);
+            expect(header.type).toBe('file');
+            expect(header.linkname).toBeNull();
+            expect(header.uname).toBe('');
+            expect(header.gname).toBe('');
+            expect(header.devmajor).toBe(0);
+            expect(header.devminor).toBe(0);
+            expect(header.mtime).toBeInstanceOf(Date);
+            expect(header.mtime.getTime()).toBe(946684800000);
+            expect(Object.keys(header).length).toBe(12);
+            stream.on('end', function() {
+              // previous test asserted tgz checksum so we don't need to check content here
+              next();
+            })
+            stream.resume();
+          });
+          extract.on('finish', () => {
+            expect(count).toBe(1);
+            fs.promises.unlink(filePath).then(done);
+          });
+          fs.createReadStream(filePath)
+            .pipe(zlib.createGunzip({
+            }))
+            .pipe(extract);
+        });
+      });
+    });

nodejs/zlib-choice.js

-Original file line number
+Diff line change
@@ -0,0 +1,6 @@
+    const zlib = require('zlib');
+    const pakoStreams = require('browserify-zlib');
+    module.exports.createGzip = pakoStreams.createGzip;
+    module.exports.createGunzip = zlib.createGunzip;
+    module.exports.constants = zlib.constants;

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Try to produce consistent monorepo tgz hashes #36

Uh oh!

Diff view

Diff view

There are no files selected for viewing

Uh oh!