fix(recorder): recompress fetch body on record (#2810)

Author: Michael Solomon

lib/recorder.js

@@ -6,21 +6,18 @@ const { inspect } = require('util')
 
 const common = require('./common')
 const { restoreOverriddenClientRequest } = require('./intercept')
-const { BatchInterceptor } = require('@mswjs/interceptors')
+const { EventEmitter } = require('stream')
+const { gzipSync, brotliCompressSync, deflateSync } = require('zlib')
 const {
   default: nodeInterceptors,
 } = require('@mswjs/interceptors/presets/node')
-const { EventEmitter } = require('stream')
-
 const SEPARATOR = '\n<<<<<<-- cut here -->>>>>>\n'
 let recordingInProgress = false
 let outputs = []
 
-// TODO: Consider use one BatchInterceptor (and not one for intercept and one for record)
-const interceptor = new BatchInterceptor({
-  name: 'nock-interceptor',
-  interceptors: nodeInterceptors,
-})
+// TODO: don't reuse the nodeInterceptors, create new ones.
+const clientRequestInterceptor = nodeInterceptors[0]
+const fetchRequestInterceptor = nodeInterceptors[2]
 
 function getScope(options) {
   const { proto, host, port } = common.normalizeRequestOptions(options)
@@ -226,109 +223,137 @@ function record(recOptions) {
   restoreOverriddenClientRequest()
 
   //  We override the requests so that we can save information on them before executing.
-  interceptor.apply()
-  interceptor.on(
-    'request',
-    async function ({ request: mswRequest, requestId }) {
-      const request = mswRequest.clone()
-      const { options } = common.normalizeClientRequestArgs(request.url)
-      options.method = request.method
-      const proto = options.protocol.slice(0, -1)
-
-      // Node 0.11 https.request calls http.request -- don't want to record things
-      // twice.
-      /* istanbul ignore if */
-      if (options._recording) {
-        return
-      }
-      options._recording = true
-
-      const req = new EventEmitter()
-      req.on('response', function () {
-        debug(thisRecordingId, 'intercepting', proto, 'request to record')
-
-        // Intercept "res.once('end', ...)"-like event
-        interceptor.once(
-          'response',
-          async function ({ response: mswResponse }) {
-            const response = mswResponse.clone()
-            debug(thisRecordingId, proto, 'intercepted request ended')
-
-            let reqheaders
-            // Ignore request headers completely unless it was explicitly enabled by the user (see README)
-            if (enableReqHeadersRecording) {
-              // We never record user-agent headers as they are worse than useless -
-              // they actually make testing more difficult without providing any benefit (see README)
-              reqheaders = Object.fromEntries(request.headers.entries())
-              common.deleteHeadersField(reqheaders, 'user-agent')
-            }
+  clientRequestInterceptor.apply()
+  fetchRequestInterceptor.apply()
+  clientRequestInterceptor.on('request', async function ({ request }) {
+    await recordRequest(request)
+  })
+  fetchRequestInterceptor.on('request', async function ({ request }) {
+    await recordRequest(request)
+  })
 
-            const headers = Object.fromEntries(response.headers.entries())
-            const res = {
-              statusCode: response.status,
-              headers,
-              rawHeaders: headers,
-            }
+  async function recordRequest(mswRequest) {
+    const request = mswRequest.clone()
+    const { options } = common.normalizeClientRequestArgs(request.url)
+    options.method = request.method
+    const proto = options.protocol.slice(0, -1)
+
+    // Node 0.11 https.request calls http.request -- don't want to record things
+    // twice.
+    /* istanbul ignore if */
+    if (options._recording) {
+      return
+    }
+    options._recording = true
 
-            const generateFn = outputObjects
-              ? generateRequestAndResponseObject
-              : generateRequestAndResponse
-            let out = generateFn({
-              req: options,
-              bodyChunks: [Buffer.from(await request.arrayBuffer())],
-              options,
-              res,
-              dataChunks: [Buffer.from(await response.arrayBuffer())],
-              reqheaders,
-            })
-
-            debug('out:', out)
-
-            //  Check that the request was made during the current recording.
-            //  If it hasn't then skip it. There is no other simple way to handle
-            //  this as it depends on the timing of requests and responses. Throwing
-            //  will make some recordings/unit tests fail randomly depending on how
-            //  fast/slow the response arrived.
-            //  If you are seeing this error then you need to make sure that all
-            //  the requests made during a single recording session finish before
-            //  ending the same recording session.
-            if (thisRecordingId !== currentRecordingId) {
-              debug('skipping recording of an out-of-order request', out)
-              return
-            }
+    const req = new EventEmitter()
+    req.on('response', function () {
+      debug(thisRecordingId, 'intercepting', proto, 'request to record')
 
-            outputs.push(out)
-
-            if (!dontPrint) {
-              if (useSeparator) {
-                if (typeof out !== 'string') {
-                  out = JSON.stringify(out, null, 2)
-                }
-                logging(SEPARATOR + out + SEPARATOR)
-              } else {
-                logging(out)
-              }
-            }
-          },
-        )
+      clientRequestInterceptor.once('response', async function ({ response }) {
+        await recordResponse(response)
+      })
+      fetchRequestInterceptor.once('response', async function ({ response }) {
+        // fetch decompresses the body automatically, so we need to recompress it
+        const codings =
+          response.headers
+            .get('content-encoding')
+            ?.toLowerCase()
+            .split(',')
+            .map(c => c.trim()) || []
+
+        let body = await response.arrayBuffer()
+        for (const coding of codings) {
+          if (coding === 'gzip') {
+            body = gzipSync(body)
+          } else if (coding === 'deflate') {
+            body = deflateSync(body)
+          } else if (coding === 'br') {
+            body = brotliCompressSync(body)
+          }
+        }
 
-        debug('finished setting up intercepting')
+        await recordResponse(new Response(body, response))
+      })
 
-        // We override both the http and the https modules; when we are
-        // serializing the request, we need to know which was called.
-        // By stuffing the state, we can make sure that nock records
-        // the intended protocol.
-        if (proto === 'https') {
-          options.proto = 'https'
+      // Intercept "res.once('end', ...)"-like event
+      async function recordResponse(mswResponse) {
+        const response = mswResponse.clone()
+        debug(thisRecordingId, proto, 'intercepted request ended')
+
+        let reqheaders
+        // Ignore request headers completely unless it was explicitly enabled by the user (see README)
+        if (enableReqHeadersRecording) {
+          // We never record user-agent headers as they are worse than useless -
+          // they actually make testing more difficult without providing any benefit (see README)
+          reqheaders = Object.fromEntries(request.headers.entries())
+          common.deleteHeadersField(reqheaders, 'user-agent')
         }
-      })
 
-      // This is a massive change, we are trying to change minimum code, so we emit end event here
-      // because mswjs take care for these events
-      // TODO: refactor the recorder, we no longer need all the listeners and can just record the request we get from MSW
-      req.emit('response')
-    },
-  )
+        const headers = Object.fromEntries(response.headers.entries())
+        const res = {
+          statusCode: response.status,
+          headers,
+          rawHeaders: headers,
+        }
+
+        const generateFn = outputObjects
+          ? generateRequestAndResponseObject
+          : generateRequestAndResponse
+        let out = generateFn({
+          req: options,
+          bodyChunks: [Buffer.from(await request.arrayBuffer())],
+          options,
+          res,
+          dataChunks: [Buffer.from(await response.arrayBuffer())],
+          reqheaders,
+        })
+
+        debug('out:', out)
+
+        //  Check that the request was made during the current recording.
+        //  If it hasn't then skip it. There is no other simple way to handle
+        //  this as it depends on the timing of requests and responses. Throwing
+        //  will make some recordings/unit tests fail randomly depending on how
+        //  fast/slow the response arrived.
+        //  If you are seeing this error then you need to make sure that all
+        //  the requests made during a single recording session finish before
+        //  ending the same recording session.
+        if (thisRecordingId !== currentRecordingId) {
+          debug('skipping recording of an out-of-order request', out)
+          return
+        }
+
+        outputs.push(out)
+
+        if (!dontPrint) {
+          if (useSeparator) {
+            if (typeof out !== 'string') {
+              out = JSON.stringify(out, null, 2)
+            }
+            logging(SEPARATOR + out + SEPARATOR)
+          } else {
+            logging(out)
+          }
+        }
+      }
+
+      debug('finished setting up intercepting')
+
+      // We override both the http and the https modules; when we are
+      // serializing the request, we need to know which was called.
+      // By stuffing the state, we can make sure that nock records
+      // the intended protocol.
+      if (proto === 'https') {
+        options.proto = 'https'
+      }
+    })
+
+    // This is a massive change, we are trying to change minimum code, so we emit end event here
+    // because mswjs take care for these events
+    // TODO: refactor the recorder, we no longer need all the listeners and can just record the request we get from MSW
+    req.emit('response')
+  }
 }
 
 // Restore *all* the overridden http/https modules' properties.
@@ -338,7 +363,8 @@ function restore() {
     'restoring all the overridden http/https properties',
   )
 
-  interceptor.dispose()
+  clientRequestInterceptor.dispose()
+  fetchRequestInterceptor.dispose()
   restoreOverriddenClientRequest()
   recordingInProgress = false
 }

tests/test_fetch.js

@@ -509,4 +509,84 @@ describe('Native Fetch', () => {
       expect(body).to.be.empty()
     })
   })
+
+  describe('recording', () => {
+    it('records and replays gzipped nocks correctly', async () => {
+      const exampleText = '<html><body>example</body></html>'
+
+      const { origin } = await startHttpServer((request, response) => {
+        // TODO: flip the order of the encoding, this is a bug in fetch
+        // const body = zlib.brotliCompressSync(zlib.gzipSync(exampleText))
+        const body = zlib.gzipSync(zlib.brotliCompressSync(exampleText))
+
+        response.writeHead(200, { 'content-encoding': 'gzip, br' })
+        response.end(body)
+      })
+
+      nock.restore()
+      nock.recorder.clear()
+      expect(nock.recorder.play()).to.be.empty()
+
+      nock.recorder.rec({
+        dont_print: true,
+        output_objects: true,
+      })
+
+      const response1 = await fetch(origin)
+      expect(await response1.text()).to.equal(exampleText)
+      expect(response1.headers.get('content-encoding')).to.equal('gzip, br')
+
+      nock.restore()
+      const recorded = nock.recorder.play()
+      nock.recorder.clear()
+      nock.activate()
+
+      expect(recorded).to.have.lengthOf(1)
+      const nocks = nock.define(recorded)
+
+      const response2 = await fetch(origin)
+      expect(await response2.text()).to.equal(exampleText)
+      expect(response1.headers.get('content-encoding')).to.equal('gzip, br')
+
+      nocks.forEach(nock => nock.done())
+    })
+
+    it('records and replays deflated nocks correctly', async () => {
+      const exampleText = '<html><body>example</body></html>'
+
+      const { origin } = await startHttpServer((request, response) => {
+        const body = zlib.deflateSync(exampleText)
+
+        response.writeHead(200, { 'content-encoding': 'deflate' })
+        response.end(body)
+      })
+
+      nock.restore()
+      nock.recorder.clear()
+      expect(nock.recorder.play()).to.be.empty()
+
+      nock.recorder.rec({
+        dont_print: true,
+        output_objects: true,
+      })
+
+      const response1 = await fetch(origin)
+      expect(await response1.text()).to.equal(exampleText)
+      expect(response1.headers.get('content-encoding')).to.equal('deflate')
+
+      nock.restore()
+      const recorded = nock.recorder.play()
+      nock.recorder.clear()
+      nock.activate()
+
+      expect(recorded).to.have.lengthOf(1)
+      const nocks = nock.define(recorded)
+
+      const response2 = await fetch(origin)
+      expect(await response2.text()).to.equal(exampleText)
+      expect(response1.headers.get('content-encoding')).to.equal('deflate')
+
+      nocks.forEach(nock => nock.done())
+    })
+  })
 })