Replace node-fetch with axios to support using proxy (#1200)

* replace node-fetch with axios

* fix indent

* fix type annotations and indentations

* fix imports and lint

* remove a console statement

* fix type checking

* fix type and return type

* retrigger

* fix as any

* updaye pnpm-lock

* retrigger

Author: Shuai Wang

common/config/rush/pnpm-lock.yaml

@@ -18,7 +18,8 @@
     "latest-version": "^4.0.0",
     "mime-types": "^2.1.18",
     "minimist": "^1.2.0",
-    "node-fetch": "~2.6.0",
+    "axios":"~0.21.1",
+    "https-proxy-agent": "^5.0.0",
     "please-upgrade-node": "^3.0.1",
     "semver": "^5.5.1",
     "tslib": "^2.0.3",

packages/chatdown/package.json

@@ -0,0 +1,37 @@
+/**
+ * Copyright (c) Microsoft Corporation. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+const url = require('url')
+const httpsProxyAgent = require('https-proxy-agent')
+
+const httpsProxy = function(config) {
+  const parsed = url.parse(config.url)
+  const protocol = parsed.protocol
+  if (protocol !== 'https:') {
+      return config
+  }
+
+  const envProxy = process.env['HTTPS_PROXY'] || process.env['https_proxy']
+  if (envProxy) {
+    const parsed = url.parse(envProxy)
+    const proxyOpt = 
+    {
+      hostname: parsed.hostname,
+      port: parsed.port
+    }
+
+    if (parsed.auth) {
+    proxyOpt.auth = parsed.auth
+    }
+    
+    config.httpsAgent = httpsProxyAgent(proxyOpt)
+    //Disable direct proxy
+    config.proxy = false
+  }
+
+  return config
+}
+
+module.exports = httpsProxy

packages/chatdown/utils/httpsProxy.js

@@ -17,9 +17,11 @@ const ChannelAccount = require('./serializable/channelAccount');
 const ConversationAccount = require('./serializable/conversationAccount');
 const Attachment = require('./serializable/attachment');
 const chalk = require('chalk');
-const request = require('node-fetch');
+const axios = require('axios');
+const httpsProxy = require('./httpsProxy');
 const NEWLINE = require('os').EOL;
 let activityId = 1;
+axios.interceptors.request.use(httpsProxy);
 
 // Matches [someActivityOrInstruction=value]
 const commandRegExp = /(?:\[)([\s\S]*?)(?:])/i;
@@ -482,8 +484,8 @@ async function addAttachment(activity, arg) {
         contentType = mime.lookup(contentUrl) || cardContentTypes[path.extname(contentUrl)];
 
         if (!contentType && contentUrl && contentUrl.indexOf('http') == 0) {
-            let response = await request(contentUrl, {method: 'HEAD'});
-            contentType = response.headers.get('content-type').split(';')[0];
+            let response = await axios.get(contentUrl, {method: 'HEAD'});
+            contentType = response.headers['content-type'].split(';')[0];
         }
     }
 

packages/chatdown/utils/index.js

@@ -17,7 +17,8 @@
     "tslib": "^2.0.3",
     "read-text-file": "^1.1.0",
     "readline-sync": "^1.4.10",
-    "node-fetch": "^2.1.2"
+    "axios":"~0.21.1",
+    "https-proxy-agent": "^5.0.0"
   },
   "devDependencies": {
     "@oclif/dev-cli": "^1.22.2",

packages/lg/package.json

@@ -9,8 +9,10 @@
 import * as path from 'path'
 import * as fs from 'fs-extra'
 import {CLIError} from '@microsoft/bf-cli-command'
+import {httpsProxy} from './https-proxy'
 // eslint-disable-next-line node/no-extraneous-require
-const fetch = require('node-fetch')
+const axios = require('axios')
+axios.interceptors.request.use(httpsProxy)
 const NEWLINE = require('os').EOL
 const ANY_NEWLINE = /\r\n|\r|\n/g
 
@@ -118,21 +120,20 @@ export class Helper {
     let tUri = `${baseUri}?api-version=3.0&to=${translateOption.to_lang}&includeAlignment=true`
     if (translateOption.src_lang) tUri += `&from=${translateOption.src_lang}`
     const options = {
-      method: 'POST',
-      body: JSON.stringify(payload),
       headers: {
         'Content-Type': 'application/json',
         'Ocp-Apim-Subscription-Key': translateOption.subscriptionKey,
         'Ocp-Apim-Subscription-Region': translateOption.region,
         'X-ClientTraceId': Helper.get_guid(),
       },
     }
-    const res = await fetch(tUri, options)
-    if (!res.ok) {
+
+    const res = await axios.post(tUri, payload, options)
+    if (res.status !== 200) {
       throw (new CLIError('Text translator service call failed with [' + res.status + '] : ' + res.statusText + '.\nPlease check key & language code validity'))
     }
 
-    const data = await res.json()
+    const data = res.data
     return data
   }
 

packages/lg/src/utils/helper.ts

@@ -0,0 +1,36 @@
+/**
+ * Copyright (c) Microsoft Corporation. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import httpsProxyAgent from 'https-proxy-agent'
+import {AxiosRequestConfig} from 'axios'
+import {Url, parse} from 'url'
+
+export function httpsProxy(config: AxiosRequestConfig) {
+  const parsed: Url = parse(config.url || '')
+  const protocol = parsed.protocol
+  if (protocol !== 'https:') {
+    return config
+  }
+
+  const envProxy = process.env.HTTPS_PROXY || process.env.https_proxy
+  if (envProxy) {
+    const parsed: Url = parse(envProxy)
+    const proxyOpt: Record<string, string | undefined> =
+    {
+      hostname: parsed.hostname,
+      port: parsed.port,
+    }
+
+    if (parsed.auth) {
+      proxyOpt.auth = parsed.auth
+    }
+
+    config.httpsAgent = httpsProxyAgent(proxyOpt)
+    // Disable direct proxy
+    config.proxy = false
+  }
+
+  return config
+}

packages/lg/src/utils/https-proxy.ts

@@ -46,7 +46,6 @@
     "globby": "^10.0.1",
     "intercept-stdout": "^0.1.2",
     "lodash": "^4.17.19",
-    "node-fetch": "~2.6.0",
     "semver": "^5.5.1",
     "tslib": "^2.0.3"
   },

packages/lu/package.json

@@ -16,7 +16,9 @@ const parserObject = require('./../lufile/classes/parserObject');
 const txtfile = require('./../lufile/read-text-file');
 const BuildDiagnostic = require('./../lufile/diagnostic').BuildDiagnostic;
 const LUISObjNameEnum = require('./../utils/enums/luisobjenum');
-const fetch = require('node-fetch');
+const axios = require('axios');
+const httpsProxy = require('../utils/httpsProxy')
+axios.interceptors.request.use(httpsProxy)
 
 module.exports = {
     /**
@@ -455,7 +457,7 @@ const resolveLuUriContent = async function(srcId, toResolve, luObjects) {
     if (uri !== undefined) {
         let response;
         try {
-            response = await fetch(uri, { method: 'GET' });
+            response = await axios.get(uri);
         } catch (err) {
             // throw, invalid URI
             let errorMsg = `URI: "${uri}" appears to be invalid. Please double check the URI or re-try this parse when you are connected to the internet.`;
@@ -465,7 +467,7 @@ const resolveLuUriContent = async function(srcId, toResolve, luObjects) {
 
             throw (new exception(retCode.errorCode.INVALID_URI, error.toString(), [error]));
         }
-        var res = await response.buffer();
+        var res = await response.data;
         var encodedRes = helpers.fixBuffer(res);
         luObjects.push(new luObject(encodedRes, new luOptions(toResolve.filePath, toResolve.includeInCollate)));
     }