From ae7d1a7811e30d06640833fd9bfaa162a8467304 Mon Sep 17 00:00:00 2001
From: Marcus Pasell <rickyrombo@users.noreply.github.com>
Date: Thu, 5 May 2022 04:34:22 +0000
Subject: [PATCH 01/19] Speed up "all up" command

Updates the `A up` command to build _all_ of the services before attempting health checks/registers
---
 service-commands/src/setup.js | 175 ++++++++++++++++++++--------------
 1 file changed, 104 insertions(+), 71 deletions(-)

diff --git a/service-commands/src/setup.js b/service-commands/src/setup.js
index 1a278009ca3..28a4d9de7af 100644
--- a/service-commands/src/setup.js
+++ b/service-commands/src/setup.js
@@ -195,7 +195,7 @@ const runSetupCommand = async (
   setupCommand,
   { serviceNumber, verbose = false, waitSec, retries } = { verbose: false }
 ) => {
-  console.log(`${service} - ${setupCommand}`.info)
+  console.log(`${service} ${serviceNumber || ''} - ${setupCommand}`.info)
   const start = Date.now()
   const commands = getServiceCommands(service, serviceNumber)
   if (!commands) {
@@ -232,7 +232,9 @@ const runSetupCommand = async (
       }
       const durationSeconds = Math.abs((Date.now() - start) / 1000)
       console.log(
-        `${service} - ${setupCommand} | executed in ${durationSeconds}s`.info
+        `${service} ${
+          serviceNumber || ''
+        } - ${setupCommand} | executed in ${durationSeconds}s`.info
       )
       return outputs
     } catch (err) {
@@ -280,12 +282,12 @@ const performHealthCheckWithRetry = async (
   let attempts = retries
   while (attempts > 0) {
     try {
-      await wait(10000)
       await performHealthCheck(service, serviceNumber)
       return
     } catch (e) {
       console.log(`${e}`)
     }
+    await wait(10000)
     attempts -= 1
   }
   const serviceNumberString = serviceNumber ? `, spId=${serviceNumber}` : ''
@@ -640,100 +642,131 @@ const allUp = async ({
     ipfsAndContractsCommands.push([Service.SOLANA_PROGRAMS, SetupCommand.UP])
   }
 
-  let creatorNodesCommands = _.range(1, numCreatorNodes + 1).map(
-    serviceNumber => {
-      return [
-        [
-          Service.CREATOR_NODE,
-          SetupCommand.UPDATE_DELEGATE_WALLET,
-          { serviceNumber, ...options }
-        ],
-        [
-          Service.CREATOR_NODE,
-          SetupCommand.UP,
-          { serviceNumber, ...options, waitSec: 10 }
-        ],
-        [
-          Service.CREATOR_NODE,
-          SetupCommand.HEALTH_CHECK_RETRY,
-          { serviceNumber, ...options }
-        ],
-        [
-          Service.CREATOR_NODE,
-          SetupCommand.REGISTER,
-          { serviceNumber, ...options }
-        ]
+  const nodeUpCommands = []
+  const nodeHealthCheckCommands = []
+  const nodeRegisterCommands = []
+  // for (
+  //   let serviceNumber = 1;
+  //   serviceNumber < numDiscoveryNodes + 1;
+  //   serviceNumber++
+  // ) {
+  //   nodeUpCommands.push([
+  //     [
+  //       Service.DISCOVERY_PROVIDER,
+  //       SetupCommand.UP,
+  //       { serviceNumber, ...options }
+  //     ]
+  //   ])
+  //   nodeHealthCheckCommands.push([
+  //     [
+  //       Service.DISCOVERY_PROVIDER,
+  //       SetupCommand.HEALTH_CHECK_RETRY,
+  //       { serviceNumber, ...options }
+  //     ]
+  //   ])
+  //   nodeRegisterCommands.push([
+  //     [
+  //       Service.DISCOVERY_PROVIDER,
+  //       SetupCommand.REGISTER,
+  //       { retries: 2, serviceNumber, ...options }
+  //     ]
+  //   ])
+  // }
+  for (
+    let serviceNumber = 1;
+    serviceNumber < numCreatorNodes + 1;
+    serviceNumber++
+  ) {
+    nodeUpCommands.push([
+      [
+        Service.CREATOR_NODE,
+        SetupCommand.UPDATE_DELEGATE_WALLET,
+        { serviceNumber, ...options }
+      ],
+      [Service.CREATOR_NODE, SetupCommand.UP, { serviceNumber, ...options }]
+    ])
+    nodeHealthCheckCommands.push([
+      [
+        Service.CREATOR_NODE,
+        SetupCommand.HEALTH_CHECK_RETRY,
+        { serviceNumber, ...options }
       ]
-    }
-  )
-
-  let discoveryNodesCommands = _.range(1, numDiscoveryNodes + 1).map(
-    serviceNumber => {
-      return [
-        [
-          Service.DISCOVERY_PROVIDER,
-          SetupCommand.UP,
-          { serviceNumber, ...options }
-        ],
-        [
-          Service.DISCOVERY_PROVIDER,
-          SetupCommand.HEALTH_CHECK_RETRY,
-          { serviceNumber, ...options }
-        ],
-        [
-          Service.DISCOVERY_PROVIDER,
-          SetupCommand.REGISTER,
-          { retries: 2, serviceNumber, ...options }
-        ]
+    ])
+    nodeRegisterCommands.push([
+      [
+        Service.CREATOR_NODE,
+        SetupCommand.REGISTER,
+        { serviceNumber, ...options }
       ]
-    }
-  )
+    ])
+  }
 
-  const sequential1 = [
+  const prereqs = [
     [Service.INIT_CONTRACTS_INFO, SetupCommand.UP],
-    [Service.INIT_TOKEN_VERSIONS, SetupCommand.UP]
-  ]
-  const sequential2 = [
-    [Service.IDENTITY_SERVICE, SetupCommand.UP],
-    [Service.IDENTITY_SERVICE, SetupCommand.HEALTH_CHECK_RETRY],
+    [Service.INIT_TOKEN_VERSIONS, SetupCommand.UP],
     [Service.USER_REPLICA_SET_MANAGER, SetupCommand.UP]
   ]
-  if (withAAO) {
-    sequential2.push([Service.AAO, SetupCommand.REGISTER])
-    sequential2.push([Service.AAO, SetupCommand.UP])
-  }
+  // nodeUpCommands.push([[Service.IDENTITY_SERVICE, SetupCommand.UP]])
+  // nodeHealthCheckCommands.push([
+  //   [Service.IDENTITY_SERVICE, SetupCommand.HEALTH_CHECK_RETRY]
+  // ])
+  // if (withAAO) {
+  //   nodeUpCommands.push([Service.AAO, SetupCommand.UP])
+  //   nodeRegisterCommands.push([Service.AAO, SetupCommand.REGISTER])
+  // }
 
   const start = Date.now()
 
   // Start up the docker network `audius_dev` and the Solana test validator
-  await runInSequence(setup, options)
+  // await runInSequence(setup, options)
   // Run parallel ops
-  await runInParallel(ipfsAndContractsCommands, options)
+  // await runInParallel(ipfsAndContractsCommands, options)
 
   // Run sequential ops
-  await runInSequence(sequential1, options)
+  // await runInSequence(prereqs, options)
 
   if (parallel) {
+    const startProv = Date.now()
+    console.log('Provisioning services in parallel'.info)
+    await Promise.all(
+      nodeUpCommands.map(commandGroup => runInSequence(commandGroup, options))
+    )
+    console.log(
+      `Services provisioned in ${Math.abs((Date.now() - startProv) / 1000.0)}s`
+    )
+    const startHealth = Date.now()
+    console.log('Health checking services'.info)
     await Promise.all(
-      discoveryNodesCommands.map(commandGroup =>
+      nodeHealthCheckCommands.map(commandGroup =>
         runInSequence(commandGroup, options)
       )
     )
+    console.log(
+      `Services health check complete in ${Math.abs(
+        (Date.now() - startHealth) / 1000.0
+      )}s`
+    )
+    const startRegister = Date.now()
+    console.log('Registering services'.info)
     await Promise.all(
-      creatorNodesCommands.map(commandGroup =>
+      nodeRegisterCommands.map(commandGroup =>
         runInSequence(commandGroup, options)
       )
     )
+    console.log(
+      `Services registered in ${Math.abs(
+        (Date.now() - startRegister) / 1000.0
+      )}s`
+    )
   } else {
-    console.log('Provisioning DNs and CNs in sequence.'.info)
-    creatorNodesCommands = creatorNodesCommands.flat()
-    discoveryNodesCommands = discoveryNodesCommands.flat()
-    await runInSequence(discoveryNodesCommands)
-    await runInSequence(creatorNodesCommands)
+    console.log('Provisioning services in sequence.'.info)
+    await runInSequence(nodeUpCommands.flat())
+    console.log('Health checking services'.info)
+    await runInSequence(nodeHealthCheckCommands.flat())
+    console.log('Registering services'.info)
+    await runInSequence(nodeRegisterCommands.flat())
   }
 
-  await runInSequence(sequential2, options)
-
   const durationSeconds = Math.abs((Date.now() - start) / 1000)
   console.log(`All services brought up in ${durationSeconds}s`.happy)
 }

From 6b4ef54615a22bd8cebbc9868e8032e0273940d8 Mon Sep 17 00:00:00 2001
From: Marcus Pasell <rickyrombo@users.noreply.github.com>
Date: Thu, 5 May 2022 04:35:06 +0000
Subject: [PATCH 02/19] Rename 'parallel' option to '--parallel'

---
 service-commands/scripts/setup.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/service-commands/scripts/setup.js b/service-commands/scripts/setup.js
index 0b17103e229..b866300dbfc 100644
--- a/service-commands/scripts/setup.js
+++ b/service-commands/scripts/setup.js
@@ -61,7 +61,7 @@ program
     false
   )
   .option(
-    'parallel',
+    '--parallel',
     'Parallel provisioning - whether to provision CNs and DNs in parallel.',
     false
   )

From b25599b7367a8e4e4df4a8bc3cef3350d374cc3f Mon Sep 17 00:00:00 2001
From: Marcus Pasell <rickyrombo@users.noreply.github.com>
Date: Thu, 5 May 2022 04:37:36 +0000
Subject: [PATCH 03/19] Uncomment things i had commented for testing oops

---
 service-commands/src/setup.js | 81 +++++++++++++++++++----------------
 1 file changed, 43 insertions(+), 38 deletions(-)

diff --git a/service-commands/src/setup.js b/service-commands/src/setup.js
index 28a4d9de7af..9fee8792454 100644
--- a/service-commands/src/setup.js
+++ b/service-commands/src/setup.js
@@ -645,33 +645,35 @@ const allUp = async ({
   const nodeUpCommands = []
   const nodeHealthCheckCommands = []
   const nodeRegisterCommands = []
-  // for (
-  //   let serviceNumber = 1;
-  //   serviceNumber < numDiscoveryNodes + 1;
-  //   serviceNumber++
-  // ) {
-  //   nodeUpCommands.push([
-  //     [
-  //       Service.DISCOVERY_PROVIDER,
-  //       SetupCommand.UP,
-  //       { serviceNumber, ...options }
-  //     ]
-  //   ])
-  //   nodeHealthCheckCommands.push([
-  //     [
-  //       Service.DISCOVERY_PROVIDER,
-  //       SetupCommand.HEALTH_CHECK_RETRY,
-  //       { serviceNumber, ...options }
-  //     ]
-  //   ])
-  //   nodeRegisterCommands.push([
-  //     [
-  //       Service.DISCOVERY_PROVIDER,
-  //       SetupCommand.REGISTER,
-  //       { retries: 2, serviceNumber, ...options }
-  //     ]
-  //   ])
-  // }
+  // Add discovery node commands
+  for (
+    let serviceNumber = 1;
+    serviceNumber < numDiscoveryNodes + 1;
+    serviceNumber++
+  ) {
+    nodeUpCommands.push([
+      [
+        Service.DISCOVERY_PROVIDER,
+        SetupCommand.UP,
+        { serviceNumber, ...options }
+      ]
+    ])
+    nodeHealthCheckCommands.push([
+      [
+        Service.DISCOVERY_PROVIDER,
+        SetupCommand.HEALTH_CHECK_RETRY,
+        { serviceNumber, ...options }
+      ]
+    ])
+    nodeRegisterCommands.push([
+      [
+        Service.DISCOVERY_PROVIDER,
+        SetupCommand.REGISTER,
+        { retries: 2, serviceNumber, ...options }
+      ]
+    ])
+  }
+  // Add creator node commands
   for (
     let serviceNumber = 1;
     serviceNumber < numCreatorNodes + 1;
@@ -706,24 +708,27 @@ const allUp = async ({
     [Service.INIT_TOKEN_VERSIONS, SetupCommand.UP],
     [Service.USER_REPLICA_SET_MANAGER, SetupCommand.UP]
   ]
-  // nodeUpCommands.push([[Service.IDENTITY_SERVICE, SetupCommand.UP]])
-  // nodeHealthCheckCommands.push([
-  //   [Service.IDENTITY_SERVICE, SetupCommand.HEALTH_CHECK_RETRY]
-  // ])
-  // if (withAAO) {
-  //   nodeUpCommands.push([Service.AAO, SetupCommand.UP])
-  //   nodeRegisterCommands.push([Service.AAO, SetupCommand.REGISTER])
-  // }
+  // Add Identity Service commands
+  nodeUpCommands.push([[Service.IDENTITY_SERVICE, SetupCommand.UP]])
+  nodeHealthCheckCommands.push([
+    [Service.IDENTITY_SERVICE, SetupCommand.HEALTH_CHECK_RETRY]
+  ])
+
+  // Add AAO commands
+  if (withAAO) {
+    nodeUpCommands.push([Service.AAO, SetupCommand.UP])
+    nodeRegisterCommands.push([Service.AAO, SetupCommand.REGISTER])
+  }
 
   const start = Date.now()
 
   // Start up the docker network `audius_dev` and the Solana test validator
-  // await runInSequence(setup, options)
+  await runInSequence(setup, options)
   // Run parallel ops
-  // await runInParallel(ipfsAndContractsCommands, options)
+  await runInParallel(ipfsAndContractsCommands, options)
 
   // Run sequential ops
-  // await runInSequence(prereqs, options)
+  await runInSequence(prereqs, options)
 
   if (parallel) {
     const startProv = Date.now()

From ba3c885fa2a6fab70e96bf457384c81b5f119015 Mon Sep 17 00:00:00 2001
From: Marcus Pasell <rickyrombo@users.noreply.github.com>
Date: Thu, 5 May 2022 04:39:46 +0000
Subject: [PATCH 04/19] Friendly log formatting

---
 service-commands/src/setup.js | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/service-commands/src/setup.js b/service-commands/src/setup.js
index 9fee8792454..96087db5251 100644
--- a/service-commands/src/setup.js
+++ b/service-commands/src/setup.js
@@ -738,6 +738,7 @@ const allUp = async ({
     )
     console.log(
       `Services provisioned in ${Math.abs((Date.now() - startProv) / 1000.0)}s`
+        .info
     )
     const startHealth = Date.now()
     console.log('Health checking services'.info)
@@ -749,7 +750,7 @@ const allUp = async ({
     console.log(
       `Services health check complete in ${Math.abs(
         (Date.now() - startHealth) / 1000.0
-      )}s`
+      )}s`.info
     )
     const startRegister = Date.now()
     console.log('Registering services'.info)
@@ -761,7 +762,7 @@ const allUp = async ({
     console.log(
       `Services registered in ${Math.abs(
         (Date.now() - startRegister) / 1000.0
-      )}s`
+      )}s`.info
     )
   } else {
     console.log('Provisioning services in sequence.'.info)

From affa98ce75ad56b61fdfd34537a42d8f238df82e Mon Sep 17 00:00:00 2001
From: Marcus Pasell <rickyrombo@users.noreply.github.com>
Date: Thu, 5 May 2022 06:21:14 +0000
Subject: [PATCH 05/19] Don't build libs in containers

---
 creator-node/scripts/start.sh          | 1 -
 identity-service/scripts/dev-server.sh | 1 -
 2 files changed, 2 deletions(-)

diff --git a/creator-node/scripts/start.sh b/creator-node/scripts/start.sh
index 79cfaaadf41..15ed17c2252 100644
--- a/creator-node/scripts/start.sh
+++ b/creator-node/scripts/start.sh
@@ -64,7 +64,6 @@ if [[ "$devMode" == "true" ]]; then
     if [ "$link_libs" = true ]
     then
         cd ../audius-libs
-        npm run dev &
         npm link
         cd ../app
         npm link @audius/libs
diff --git a/identity-service/scripts/dev-server.sh b/identity-service/scripts/dev-server.sh
index e0b454a1519..ca3990b5503 100755
--- a/identity-service/scripts/dev-server.sh
+++ b/identity-service/scripts/dev-server.sh
@@ -7,7 +7,6 @@ link_libs=true
 if [ "$link_libs" = true ]
 then
     cd ../audius-libs
-    npm run dev &
     npm link
     cd ../app
     npm link @audius/libs

From 7c81e51a03a62b3537099a99216975d5ac0525ea Mon Sep 17 00:00:00 2001
From: Marcus Pasell <rickyrombo@users.noreply.github.com>
Date: Sat, 7 May 2022 11:14:34 +0000
Subject: [PATCH 06/19] Warn users about --parallel, run libs in bg

---
 service-commands/.gitignore                   |  1 +
 service-commands/scripts/run-libs.sh          |  2 ++
 service-commands/scripts/setup.js             |  6 ++++
 .../src/commands/service-commands.json        |  7 +++++
 service-commands/src/setup.js                 | 29 ++++++++++++++++---
 5 files changed, 41 insertions(+), 4 deletions(-)
 create mode 100755 service-commands/scripts/run-libs.sh

diff --git a/service-commands/.gitignore b/service-commands/.gitignore
index c5695e55349..72ee5c26dc9 100644
--- a/service-commands/.gitignore
+++ b/service-commands/.gitignore
@@ -1,3 +1,4 @@
 output.log
 error.log
+libs.log
 local-storage
diff --git a/service-commands/scripts/run-libs.sh b/service-commands/scripts/run-libs.sh
new file mode 100755
index 00000000000..52afb9c83c9
--- /dev/null
+++ b/service-commands/scripts/run-libs.sh
@@ -0,0 +1,2 @@
+# Convenience script to make it easier to grep for exactly what service command runs
+npm run dev
\ No newline at end of file
diff --git a/service-commands/scripts/setup.js b/service-commands/scripts/setup.js
index b866300dbfc..fd6f55e7f13 100644
--- a/service-commands/scripts/setup.js
+++ b/service-commands/scripts/setup.js
@@ -91,6 +91,12 @@ program
     const numCreatorNodes = parseInt(opts.numCnodes)
     const numDiscoveryNodes = parseInt(opts.numDn)
     const { verbose, parallel, withAao: withAAO, withSolanaProgramsBuild: buildSolana } = opts
+    if (parallel) {
+      // Trying to bring up services in parallel will cause potential problems as the Docker compose files
+      // for various services, CN in particular, requires ENV vars to be set. If they override each other due to race
+      // conditions, then the services will be misconfigured or conflict.
+      console.log('Warning: --parallel is not yet supported.'.error)
+    }
     await allUp({ numCreatorNodes, numDiscoveryNodes, withAAO, verbose, parallel, buildSolana, opts })
   })
 
diff --git a/service-commands/src/commands/service-commands.json b/service-commands/src/commands/service-commands.json
index 3a97761b03e..8e98824db53 100644
--- a/service-commands/src/commands/service-commands.json
+++ b/service-commands/src/commands/service-commands.json
@@ -1,6 +1,7 @@
 {
   "all": {
     "down": [
+      "A run libs down",
       "SERVICES=$(docker ps -aq); if [ \"$SERVICES\" != \"\" ]; then docker stop ${SERVICES} && docker rm ${SERVICES}; fi",
       "docker container prune -f",
       "docker volume prune -f",
@@ -300,5 +301,11 @@
       "echo 'Stopping TN...'",
       "cd $TN_DIR; . scripts/stop.sh dev"
     ]
+  },
+  "libs": {
+    "up": ["echo 'Process spawned in background using Node.'"],
+    "down": [
+      "kill -9 $(ps x -o \"%r %a\" | grep '.*/audius-protocol/service-commands/scripts/run-libs.sh$' | awk '{print -$1}')"
+    ]
   }
 }
diff --git a/service-commands/src/setup.js b/service-commands/src/setup.js
index 96087db5251..b3d3daab2c5 100644
--- a/service-commands/src/setup.js
+++ b/service-commands/src/setup.js
@@ -1,4 +1,4 @@
-const { exec } = require('child_process')
+const { exec, spawn } = require('child_process')
 const fs = require('fs')
 const colors = require('colors')
 const config = require('../config/config.js')
@@ -161,7 +161,8 @@ const Service = Object.freeze({
   INIT_REPOS: 'init-repos',
   USER_REPLICA_SET_MANAGER: 'user-replica-set-manager',
   AAO: 'aao',
-  TN: 'tn'
+  TN: 'tn',
+  LIBS: 'libs'
 })
 
 // gets a service command, interpolating service names
@@ -212,6 +213,25 @@ const runSetupCommand = async (
     return
   }
 
+  if (service === Service.LIBS && setupCommand === SetupCommand.UP) {
+    const libsLog = await fs.promises.open(
+      `${PROTOCOL_DIR}/service-commands/libs.log`,
+      'w'
+    )
+    const subprocess = await spawn(
+      `${PROTOCOL_DIR}/service-commands/scripts/run-libs.sh`,
+      [],
+      {
+        detached: true,
+        stdio: ['ignore', libsLog, libsLog],
+        cwd: `${PROTOCOL_DIR}/libs`
+      }
+    )
+    subprocess.unref()
+    console.log(`Spawned libs watcher. PID: ${subprocess.pid}`.info)
+    libsLog.close()
+  }
+
   const command = commands[setupCommand]
   if (!command) {
     throw new Error(
@@ -627,7 +647,7 @@ const allUp = async ({
 
   const setup = [
     [Service.NETWORK, SetupCommand.UP],
-    [Service.SOLANA_VALIDATOR_PREDEPLOYED, SetupCommand.UP],
+    [Service.SOLANA_VALIDATOR_PREDEPLOYED, SetupCommand.UP, { waitSec: 1 }],
     [Service.SOLANA_VALIDATOR_PREDEPLOYED, SetupCommand.HEALTH_CHECK_RETRY]
   ]
 
@@ -635,7 +655,8 @@ const allUp = async ({
     [Service.IPFS, SetupCommand.UP],
     [Service.IPFS_2, SetupCommand.UP],
     [Service.CONTRACTS, SetupCommand.UP],
-    [Service.ETH_CONTRACTS, SetupCommand.UP]
+    [Service.ETH_CONTRACTS, SetupCommand.UP],
+    [Service.LIBS, SetupCommand.UP]
   ]
 
   if (buildSolana) {

From db85bcce980cf7c87b8fcb55232bf15106479d55 Mon Sep 17 00:00:00 2001
From: Marcus Pasell <rickyrombo@users.noreply.github.com>
Date: Sat, 7 May 2022 11:39:13 +0000
Subject: [PATCH 07/19] Wait a bit more for solana-validator

---
 service-commands/src/setup.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/service-commands/src/setup.js b/service-commands/src/setup.js
index b3d3daab2c5..5607240db45 100644
--- a/service-commands/src/setup.js
+++ b/service-commands/src/setup.js
@@ -647,7 +647,7 @@ const allUp = async ({
 
   const setup = [
     [Service.NETWORK, SetupCommand.UP],
-    [Service.SOLANA_VALIDATOR_PREDEPLOYED, SetupCommand.UP, { waitSec: 1 }],
+    [Service.SOLANA_VALIDATOR_PREDEPLOYED, SetupCommand.UP, { waitSec: 3 }],
     [Service.SOLANA_VALIDATOR_PREDEPLOYED, SetupCommand.HEALTH_CHECK_RETRY]
   ]
 

From fdc0c9ea174dd54f7a4fadf0b5eb4eed9fb60e4f Mon Sep 17 00:00:00 2001
From: Marcus Pasell <rickyrombo@users.noreply.github.com>
Date: Sun, 8 May 2022 00:12:56 +0000
Subject: [PATCH 08/19] Change comment

---
 service-commands/scripts/setup.js | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/service-commands/scripts/setup.js b/service-commands/scripts/setup.js
index fd6f55e7f13..2fa26a938ec 100644
--- a/service-commands/scripts/setup.js
+++ b/service-commands/scripts/setup.js
@@ -92,9 +92,7 @@ program
     const numDiscoveryNodes = parseInt(opts.numDn)
     const { verbose, parallel, withAao: withAAO, withSolanaProgramsBuild: buildSolana } = opts
     if (parallel) {
-      // Trying to bring up services in parallel will cause potential problems as the Docker compose files
-      // for various services, CN in particular, requires ENV vars to be set. If they override each other due to race
-      // conditions, then the services will be misconfigured or conflict.
+      // TODO: Further testing to prove --parallel works before recommending it
       console.log('Warning: --parallel is not yet supported.'.error)
     }
     await allUp({ numCreatorNodes, numDiscoveryNodes, withAAO, verbose, parallel, buildSolana, opts })

From ee319b5ae11e0671a67c16ca1096a76c92e7e182 Mon Sep 17 00:00:00 2001
From: Marcus Pasell <rickyrombo@users.noreply.github.com>
Date: Mon, 9 May 2022 20:04:13 +0000
Subject: [PATCH 09/19] Kill command idempotent

---
 service-commands/src/commands/service-commands.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/service-commands/src/commands/service-commands.json b/service-commands/src/commands/service-commands.json
index 8e98824db53..ca4e265a1c8 100644
--- a/service-commands/src/commands/service-commands.json
+++ b/service-commands/src/commands/service-commands.json
@@ -305,7 +305,7 @@
   "libs": {
     "up": ["echo 'Process spawned in background using Node.'"],
     "down": [
-      "kill -9 $(ps x -o \"%r %a\" | grep '.*/audius-protocol/service-commands/scripts/run-libs.sh$' | awk '{print -$1}')"
+      "ps x -o '%r %a' | grep '.*/audius-protocol/service-commands/scripts/run-libs.sh$' | awk '{print -$1}' | xargs -r kill -- -9"
     ]
   }
 }

From 514e4470d58ce2c46dcdb54b94eb031faed43ee6 Mon Sep 17 00:00:00 2001
From: Marcus Pasell <rickyrombo@users.noreply.github.com>
Date: Mon, 9 May 2022 23:13:27 +0000
Subject: [PATCH 10/19] Bad merge

---
 service-commands/src/setup.js | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/service-commands/src/setup.js b/service-commands/src/setup.js
index e6999bf7cfc..17c6560765e 100644
--- a/service-commands/src/setup.js
+++ b/service-commands/src/setup.js
@@ -748,7 +748,7 @@ const allUp = async ({
 
   const prereqs = [
     [Service.INIT_CONTRACTS_INFO, SetupCommand.UP],
-    [Service.INIT_TOKEN_VERSIONS, SetupCommand.UP],
+    [Service.INIT_TOKEN_VERSIONS, SetupCommand.UP]
   ]
   // Add Identity Service commands
   nodeUpCommands.push([[Service.IDENTITY_SERVICE, SetupCommand.UP]])
@@ -762,6 +762,7 @@ const allUp = async ({
     if (buildDataEthContracts) {
       nodeRegisterCommands.push([Service.AAO, SetupCommand.REGISTER])
     }
+  }
 
   const start = Date.now()
 

From aab31a28bc342d2ae8bf6c4825976c17aef7ba1c Mon Sep 17 00:00:00 2001
From: Marcus Pasell <rickyrombo@users.noreply.github.com>
Date: Mon, 9 May 2022 23:18:39 +0000
Subject: [PATCH 11/19] bad merge

---
 service-commands/src/setup.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/service-commands/src/setup.js b/service-commands/src/setup.js
index 17c6560765e..b974b69bc4c 100644
--- a/service-commands/src/setup.js
+++ b/service-commands/src/setup.js
@@ -775,7 +775,7 @@ const allUp = async ({
   if (buildDataEthContracts) {
     await runInSequence(prereqs, options)
   }
-  await runInSequence([Service.USER_REPLICA_SET_MANAGER, SetupCommand.UP])
+  await runInSequence([[Service.USER_REPLICA_SET_MANAGER, SetupCommand.UP]])
 
   if (parallel) {
     const startProv = Date.now()

From 1aed0e5e6768417f42c0aac0c7464b229310652e Mon Sep 17 00:00:00 2001
From: Marcus Pasell <rickyrombo@users.noreply.github.com>
Date: Mon, 9 May 2022 23:54:23 +0000
Subject: [PATCH 12/19] Add wait before healthchecks, group env to command

---
 .../src/commands/service-commands.json        |  4 +---
 service-commands/src/setup.js                 | 20 +++++++++++++++----
 2 files changed, 17 insertions(+), 7 deletions(-)

diff --git a/service-commands/src/commands/service-commands.json b/service-commands/src/commands/service-commands.json
index c14b5d93b4d..3db67bf803d 100644
--- a/service-commands/src/commands/service-commands.json
+++ b/service-commands/src/commands/service-commands.json
@@ -19,9 +19,7 @@
   },
   "network": {
     "up": [
-      "export DOCKER_CLIENT_TIMEOUT=360",
-      "export COMPOSE_HTTP_TIMEOUT=360",
-      "docker network create -d bridge audius_dev || true"
+      "export DOCKER_CLIENT_TIMEOUT=360; export COMPOSE_HTTP_TIMEOUT=360; docker network create -d bridge audius_dev || true"
     ],
     "down": [
       "docker network rm audius_dev || true"
diff --git a/service-commands/src/setup.js b/service-commands/src/setup.js
index b974b69bc4c..606363a229e 100644
--- a/service-commands/src/setup.js
+++ b/service-commands/src/setup.js
@@ -211,7 +211,7 @@ const runSetupCommand = async (
   }
 
   if (setupCommand === SetupCommand.HEALTH_CHECK_RETRY) {
-    await performHealthCheckWithRetry(service, serviceNumber)
+    await performHealthCheckWithRetry(service, serviceNumber, retries, waitSec)
     return
   }
 
@@ -299,9 +299,13 @@ const getServiceURL = (service, serviceNumber) => {
 const performHealthCheckWithRetry = async (
   service,
   serviceNumber,
-  retries = 20
+  retries = 20,
+  waitSec = 0
 ) => {
   let attempts = retries
+  if (waitSec > 0) {
+    await wait(waitSec * 1000)
+  }
   while (attempts > 0) {
     try {
       await performHealthCheck(service, serviceNumber)
@@ -677,8 +681,16 @@ const allUp = async ({
   ]
 
   const contractHealthChecksCommands = [
-    [Service.CONTRACTS_PREDEPLOYED, SetupCommand.HEALTH_CHECK_RETRY],
-    [Service.ETH_CONTRACTS_PREDEPLOYED, SetupCommand.HEALTH_CHECK_RETRY]
+    [
+      Service.CONTRACTS_PREDEPLOYED,
+      SetupCommand.HEALTH_CHECK_RETRY,
+      { waitSec: 3 }
+    ],
+    [
+      Service.ETH_CONTRACTS_PREDEPLOYED,
+      SetupCommand.HEALTH_CHECK_RETRY,
+      { waitSec: 3 }
+    ]
   ]
 
   if (buildSolana) {

From 93d153d5c884e35ab215cebe343ce97ba472ed8e Mon Sep 17 00:00:00 2001
From: Marcus Pasell <rickyrombo@users.noreply.github.com>
Date: Tue, 10 May 2022 19:06:01 +0000
Subject: [PATCH 13/19] Fix libs reselection

---
 .../discoveryProvider/DiscoveryProvider.ts    | 38 ++++++++++++-------
 1 file changed, 25 insertions(+), 13 deletions(-)

diff --git a/libs/src/services/discoveryProvider/DiscoveryProvider.ts b/libs/src/services/discoveryProvider/DiscoveryProvider.ts
index 09ef639fabd..d8939d349c7 100644
--- a/libs/src/services/discoveryProvider/DiscoveryProvider.ts
+++ b/libs/src/services/discoveryProvider/DiscoveryProvider.ts
@@ -956,20 +956,32 @@ export class DiscoveryProvider {
     retry = true,
     attemptedRetries = 0
   ): Promise<Response | undefined | null> {
-    try {
-      const newDiscProvEndpoint =
-        await this.getHealthyDiscoveryProviderEndpoint(attemptedRetries)
-
-      // If new DP endpoint is selected, update disc prov endpoint and reset attemptedRetries count
-      if (this.discoveryProviderEndpoint !== newDiscProvEndpoint) {
-        let updateDiscProvEndpointMsg = `Current Discovery Provider endpoint ${this.discoveryProviderEndpoint} is unhealthy. `
-        updateDiscProvEndpointMsg += `Switching over to the new Discovery Provider endpoint ${newDiscProvEndpoint}!`
-        console.info(updateDiscProvEndpointMsg)
-        this.discoveryProviderEndpoint = newDiscProvEndpoint
-        attemptedRetries = 0
+    let hasEndpoint = false
+    do {
+      try {
+        const newDiscProvEndpoint =
+          await this.getHealthyDiscoveryProviderEndpoint(attemptedRetries)
+
+        // If new DP endpoint is selected, update disc prov endpoint and reset attemptedRetries count
+        if (this.discoveryProviderEndpoint !== newDiscProvEndpoint) {
+          let updateDiscProvEndpointMsg = `Current Discovery Provider endpoint ${this.discoveryProviderEndpoint} is unhealthy. `
+          updateDiscProvEndpointMsg += `Switching over to the new Discovery Provider endpoint ${newDiscProvEndpoint}!`
+          console.info(updateDiscProvEndpointMsg)
+          this.discoveryProviderEndpoint = newDiscProvEndpoint
+          attemptedRetries = 0
+        }
+        hasEndpoint = true
+      } catch (e) {
+        console.error(e)
       }
-    } catch (e) {
-      console.error(e)
+    } while (
+      !hasEndpoint &&
+      // eslint-disable-next-line no-unmodified-loop-condition
+      retry &&
+      attemptedRetries < this.selectionRequestRetries
+    )
+    if (!hasEndpoint || !this.discoveryProviderEndpoint) {
+      console.error('Failed to select Discovery Provider')
       return
     }
     let parsedResponse

From 639e2f7de4217fbfff5c90059d5060d11ce426e1 Mon Sep 17 00:00:00 2001
From: Marcus Pasell <rickyrombo@users.noreply.github.com>
Date: Tue, 10 May 2022 19:08:54 +0000
Subject: [PATCH 14/19] Fix retries

---
 libs/src/services/discoveryProvider/DiscoveryProvider.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libs/src/services/discoveryProvider/DiscoveryProvider.ts b/libs/src/services/discoveryProvider/DiscoveryProvider.ts
index d8939d349c7..23a35055e7e 100644
--- a/libs/src/services/discoveryProvider/DiscoveryProvider.ts
+++ b/libs/src/services/discoveryProvider/DiscoveryProvider.ts
@@ -978,7 +978,7 @@ export class DiscoveryProvider {
       !hasEndpoint &&
       // eslint-disable-next-line no-unmodified-loop-condition
       retry &&
-      attemptedRetries < this.selectionRequestRetries
+      attemptedRetries++ < this.selectionRequestRetries
     )
     if (!hasEndpoint || !this.discoveryProviderEndpoint) {
       console.error('Failed to select Discovery Provider')

From 991de79b978f36bf8eeb46a692da1f32b309b45d Mon Sep 17 00:00:00 2001
From: Marcus Pasell <rickyrombo@users.noreply.github.com>
Date: Tue, 10 May 2022 19:17:33 +0000
Subject: [PATCH 15/19] Revert "Fix retries"

This reverts commit 639e2f7de4217fbfff5c90059d5060d11ce426e1.
---
 libs/src/services/discoveryProvider/DiscoveryProvider.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libs/src/services/discoveryProvider/DiscoveryProvider.ts b/libs/src/services/discoveryProvider/DiscoveryProvider.ts
index 23a35055e7e..d8939d349c7 100644
--- a/libs/src/services/discoveryProvider/DiscoveryProvider.ts
+++ b/libs/src/services/discoveryProvider/DiscoveryProvider.ts
@@ -978,7 +978,7 @@ export class DiscoveryProvider {
       !hasEndpoint &&
       // eslint-disable-next-line no-unmodified-loop-condition
       retry &&
-      attemptedRetries++ < this.selectionRequestRetries
+      attemptedRetries < this.selectionRequestRetries
     )
     if (!hasEndpoint || !this.discoveryProviderEndpoint) {
       console.error('Failed to select Discovery Provider')

From f9ddcf4ddc75d505d6edd0e19666447f9b0622fb Mon Sep 17 00:00:00 2001
From: Marcus Pasell <rickyrombo@users.noreply.github.com>
Date: Tue, 10 May 2022 19:17:42 +0000
Subject: [PATCH 16/19] Revert "Fix libs reselection"

This reverts commit 93d153d5c884e35ab215cebe343ce97ba472ed8e.
---
 .../discoveryProvider/DiscoveryProvider.ts    | 38 +++++++------------
 1 file changed, 13 insertions(+), 25 deletions(-)

diff --git a/libs/src/services/discoveryProvider/DiscoveryProvider.ts b/libs/src/services/discoveryProvider/DiscoveryProvider.ts
index d8939d349c7..09ef639fabd 100644
--- a/libs/src/services/discoveryProvider/DiscoveryProvider.ts
+++ b/libs/src/services/discoveryProvider/DiscoveryProvider.ts
@@ -956,32 +956,20 @@ export class DiscoveryProvider {
     retry = true,
     attemptedRetries = 0
   ): Promise<Response | undefined | null> {
-    let hasEndpoint = false
-    do {
-      try {
-        const newDiscProvEndpoint =
-          await this.getHealthyDiscoveryProviderEndpoint(attemptedRetries)
-
-        // If new DP endpoint is selected, update disc prov endpoint and reset attemptedRetries count
-        if (this.discoveryProviderEndpoint !== newDiscProvEndpoint) {
-          let updateDiscProvEndpointMsg = `Current Discovery Provider endpoint ${this.discoveryProviderEndpoint} is unhealthy. `
-          updateDiscProvEndpointMsg += `Switching over to the new Discovery Provider endpoint ${newDiscProvEndpoint}!`
-          console.info(updateDiscProvEndpointMsg)
-          this.discoveryProviderEndpoint = newDiscProvEndpoint
-          attemptedRetries = 0
-        }
-        hasEndpoint = true
-      } catch (e) {
-        console.error(e)
+    try {
+      const newDiscProvEndpoint =
+        await this.getHealthyDiscoveryProviderEndpoint(attemptedRetries)
+
+      // If new DP endpoint is selected, update disc prov endpoint and reset attemptedRetries count
+      if (this.discoveryProviderEndpoint !== newDiscProvEndpoint) {
+        let updateDiscProvEndpointMsg = `Current Discovery Provider endpoint ${this.discoveryProviderEndpoint} is unhealthy. `
+        updateDiscProvEndpointMsg += `Switching over to the new Discovery Provider endpoint ${newDiscProvEndpoint}!`
+        console.info(updateDiscProvEndpointMsg)
+        this.discoveryProviderEndpoint = newDiscProvEndpoint
+        attemptedRetries = 0
       }
-    } while (
-      !hasEndpoint &&
-      // eslint-disable-next-line no-unmodified-loop-condition
-      retry &&
-      attemptedRetries < this.selectionRequestRetries
-    )
-    if (!hasEndpoint || !this.discoveryProviderEndpoint) {
-      console.error('Failed to select Discovery Provider')
+    } catch (e) {
+      console.error(e)
       return
     }
     let parsedResponse

From 748ffa8fb44a4ce08d92d73c0641d6757b73f0bf Mon Sep 17 00:00:00 2001
From: Marcus Pasell <rickyrombo@users.noreply.github.com>
Date: Tue, 10 May 2022 20:24:57 +0000
Subject: [PATCH 17/19] Bring up DN first

---
 service-commands/src/setup.js | 89 +++++++++++++++++++++--------------
 1 file changed, 54 insertions(+), 35 deletions(-)

diff --git a/service-commands/src/setup.js b/service-commands/src/setup.js
index 606363a229e..14d1603c681 100644
--- a/service-commands/src/setup.js
+++ b/service-commands/src/setup.js
@@ -697,30 +697,30 @@ const allUp = async ({
     ipfsAndContractsCommands.push([Service.SOLANA_PROGRAMS, SetupCommand.UP])
   }
 
-  const nodeUpCommands = []
-  const nodeHealthCheckCommands = []
-  const nodeRegisterCommands = []
+  const discoveryNodeUpCommands = []
+  const discoveryNodeHealthCheckCommands = []
+  const discoveryNodeRegisterCommands = []
   // Add discovery node commands
   for (
     let serviceNumber = 1;
     serviceNumber < numDiscoveryNodes + 1;
     serviceNumber++
   ) {
-    nodeUpCommands.push([
+    discoveryNodeUpCommands.push([
       [
         Service.DISCOVERY_PROVIDER,
         SetupCommand.UP,
         { serviceNumber, ...options }
       ]
     ])
-    nodeHealthCheckCommands.push([
+    discoveryNodeHealthCheckCommands.push([
       [
         Service.DISCOVERY_PROVIDER,
         SetupCommand.HEALTH_CHECK_RETRY,
         { serviceNumber, ...options }
       ]
     ])
-    nodeRegisterCommands.push([
+    discoveryNodeRegisterCommands.push([
       [
         Service.DISCOVERY_PROVIDER,
         SetupCommand.REGISTER,
@@ -728,6 +728,9 @@ const allUp = async ({
       ]
     ])
   }
+  const nodeUpCommands = []
+  const nodeHealthCheckCommands = []
+  const nodeRegisterCommands = []
   // Add creator node commands
   for (
     let serviceNumber = 1;
@@ -790,40 +793,56 @@ const allUp = async ({
   await runInSequence([[Service.USER_REPLICA_SET_MANAGER, SetupCommand.UP]])
 
   if (parallel) {
-    const startProv = Date.now()
-    console.log('Provisioning services in parallel'.info)
-    await Promise.all(
-      nodeUpCommands.map(commandGroup => runInSequence(commandGroup, options))
-    )
-    console.log(
-      `Services provisioned in ${Math.abs((Date.now() - startProv) / 1000.0)}s`
-        .info
-    )
-    const startHealth = Date.now()
-    console.log('Health checking services'.info)
-    await Promise.all(
-      nodeHealthCheckCommands.map(commandGroup =>
-        runInSequence(commandGroup, options)
+    const runParallel = async (up, healthCheck, register) => {
+      const startProv = Date.now()
+      await Promise.all(
+        up.map(commandGroup => runInSequence(commandGroup, options))
       )
-    )
-    console.log(
-      `Services health check complete in ${Math.abs(
-        (Date.now() - startHealth) / 1000.0
-      )}s`.info
-    )
-    const startRegister = Date.now()
-    console.log('Registering services'.info)
-    await Promise.all(
-      nodeRegisterCommands.map(commandGroup =>
-        runInSequence(commandGroup, options)
+      console.log(
+        `Services provisioned in ${Math.abs(
+          (Date.now() - startProv) / 1000.0
+        )}s`.info
+      )
+      const startHealth = Date.now()
+      console.log('Health checking services'.info)
+      await Promise.all(
+        healthCheck.map(commandGroup => runInSequence(commandGroup, options))
       )
+      console.log(
+        `Services health check complete in ${Math.abs(
+          (Date.now() - startHealth) / 1000.0
+        )}s`.info
+      )
+      const startRegister = Date.now()
+      console.log('Registering services'.info)
+      await Promise.all(
+        register.map(commandGroup => runInSequence(commandGroup, options))
+      )
+      console.log(
+        `Services registered in ${Math.abs(
+          (Date.now() - startRegister) / 1000.0
+        )}s`.info
+      )
+    }
+    console.log('Provisioning DNs in parallel'.info)
+    await runParallel(
+      discoveryNodeUpCommands,
+      discoveryNodeHealthCheckCommands,
+      discoveryNodeRegisterCommands
     )
-    console.log(
-      `Services registered in ${Math.abs(
-        (Date.now() - startRegister) / 1000.0
-      )}s`.info
+    console.log('Provisioning CNs, identity, etc in parallel'.info)
+    await runParallel(
+      nodeUpCommands,
+      nodeHealthCheckCommands,
+      nodeRegisterCommands
     )
   } else {
+    console.log('Provisioning DNs in sequence.'.info)
+    await runInSequence(discoveryNodeUpCommands.flat())
+    console.log('Health checking DNs'.info)
+    await runInSequence(discoveryNodeHealthCheckCommands.flat())
+    console.log('Registering DNs'.info)
+    await runInSequence(discoveryNodeRegisterCommands.flat())
     console.log('Provisioning services in sequence.'.info)
     await runInSequence(nodeUpCommands.flat())
     console.log('Health checking services'.info)

From 7c60b9cfef47e1b6d2a688479bcd9883fd88b9eb Mon Sep 17 00:00:00 2001
From: Marcus Pasell <rickyrombo@users.noreply.github.com>
Date: Tue, 10 May 2022 23:31:02 +0000
Subject: [PATCH 18/19] Move URSM to end

---
 service-commands/src/setup.js | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/service-commands/src/setup.js b/service-commands/src/setup.js
index 14d1603c681..f87cb542330 100644
--- a/service-commands/src/setup.js
+++ b/service-commands/src/setup.js
@@ -790,7 +790,6 @@ const allUp = async ({
   if (buildDataEthContracts) {
     await runInSequence(prereqs, options)
   }
-  await runInSequence([[Service.USER_REPLICA_SET_MANAGER, SetupCommand.UP]])
 
   if (parallel) {
     const runParallel = async (up, healthCheck, register) => {
@@ -851,6 +850,9 @@ const allUp = async ({
     await runInSequence(nodeRegisterCommands.flat())
   }
 
+  // URSM has to up after Creator Nodes are registered
+  await runInSequence([[Service.USER_REPLICA_SET_MANAGER, SetupCommand.UP]])
+
   const durationSeconds = Math.abs((Date.now() - start) / 1000)
   console.log(`All services brought up in ${durationSeconds}s`.happy)
 }

From 0e47798daca053136ed275af96bda59ac5b1402c Mon Sep 17 00:00:00 2001
From: Marcus Pasell <rickyrombo@users.noreply.github.com>
Date: Wed, 11 May 2022 01:34:47 +0000
Subject: [PATCH 19/19] Register in sequence

---
 service-commands/src/setup.js | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/service-commands/src/setup.js b/service-commands/src/setup.js
index f87cb542330..1b2de663622 100644
--- a/service-commands/src/setup.js
+++ b/service-commands/src/setup.js
@@ -814,9 +814,7 @@ const allUp = async ({
       )
       const startRegister = Date.now()
       console.log('Registering services'.info)
-      await Promise.all(
-        register.map(commandGroup => runInSequence(commandGroup, options))
-      )
+      await runInSequence(register.flat())
       console.log(
         `Services registered in ${Math.abs(
           (Date.now() - startRegister) / 1000.0