diff --git a/creator-node/src/services/stateMachineManager/stateMachineConstants.ts b/creator-node/src/services/stateMachineManager/stateMachineConstants.ts index 9abb749d3e8..8e7cd6ee099 100644 --- a/creator-node/src/services/stateMachineManager/stateMachineConstants.ts +++ b/creator-node/src/services/stateMachineManager/stateMachineConstants.ts @@ -32,7 +32,7 @@ export const AGGREGATE_RECONFIG_AND_POTENTIAL_SYNC_OPS_BATCH_SIZE = 500 export const SYNC_MONITORING_RETRY_DELAY_MS = 15_000 // Max number of attempts to select new replica set in reconfig -export const MAX_SELECT_NEW_REPLICA_SET_ATTEMPTS = 5 +export const MAX_SELECT_NEW_REPLICA_SET_ATTEMPTS = 20 // Max number of attempts to run a job that attempts to issue a manual sync export const MAX_ISSUE_MANUAL_SYNC_JOB_ATTEMPTS = 2 diff --git a/creator-node/src/services/stateMachineManager/stateReconciliation/updateReplicaSet.jobProcessor.ts b/creator-node/src/services/stateMachineManager/stateReconciliation/updateReplicaSet.jobProcessor.ts index 77ca01a0191..94e3a293878 100644 --- a/creator-node/src/services/stateMachineManager/stateReconciliation/updateReplicaSet.jobProcessor.ts +++ b/creator-node/src/services/stateMachineManager/stateReconciliation/updateReplicaSet.jobProcessor.ts @@ -45,7 +45,7 @@ const reconfigNodeWhitelist = config.get('reconfigNodeWhitelist') * @param {Object} param.replicaToUserInfoMap map(secondary endpoint => { clock, filesHash }) map of user's node endpoint strings to user info on node for user whose replica set should be updated * @param {string[]} param.enabledReconfigModes array of which reconfig modes are enabled */ -module.exports = async function ({ +const updateReplicaSetJobProcessor = async function ({ logger, wallet, userId, @@ -370,19 +370,20 @@ const _selectRandomReplicaSetNodes = async ( ]} ||` const newReplicaNodesSet = new Set() + + const viablePotentialReplicas = healthyNodes.filter( + (node) => !healthyReplicaSet.has(node) + ) + let selectNewReplicaSetAttemptCounter = 0 while ( newReplicaNodesSet.size < numberOfUnhealthyReplicas && selectNewReplicaSetAttemptCounter++ < MAX_SELECT_NEW_REPLICA_SET_ATTEMPTS ) { - const randomHealthyNode = _.sample(healthyNodes) + const randomHealthyNode = _.sample(viablePotentialReplicas) // If node is already present in new replica set or is part of the existing replica set, keep finding a unique healthy node - if ( - newReplicaNodesSet.has(randomHealthyNode) || - healthyReplicaSet.has(randomHealthyNode) - ) - continue + if (newReplicaNodesSet.has(randomHealthyNode)) continue // If the node was marked as healthy before, keep finding a unique healthy node if (unhealthyReplicasSet.has(randomHealthyNode)) { @@ -579,3 +580,5 @@ const _isReconfigEnabled = (enabledReconfigModes: string[], mode: string) => { if (mode === RECONFIG_MODES.RECONFIG_DISABLED.key) return false return enabledReconfigModes.includes(mode) } + +module.exports = updateReplicaSetJobProcessor