diff --git a/packages/discovery-provider/src/utils/rendezvous.py b/packages/discovery-provider/src/utils/rendezvous.py index bc1ca935aed..789da704698 100644 --- a/packages/discovery-provider/src/utils/rendezvous.py +++ b/packages/discovery-provider/src/utils/rendezvous.py @@ -1,35 +1,54 @@ +from hashlib import sha256 from typing import List import crc32c +dead_nodes = ["https://content.grassfed.network/"] + # Python equivalent of https://github.com/tysonmote/rendezvous/blob/be0258dbbd3d/rendezvous.go class RendezvousHash: def __init__(self, *nodes: str): - self.nodes: List[bytes] = [bytes(node, "utf-8") for node in nodes] + self.nodes: List[str] = [n for n in nodes if n not in dead_nodes] def add(self, *nodes: str) -> None: for node in nodes: - self.nodes.append(bytes(node, "utf-8")) + if node not in dead_nodes: + self.nodes.append(node) + + def get_nodes(self) -> List[str]: + return self.nodes def get(self, key: str) -> str: - max_node = max( - (node for node in self.nodes), - default=None, - key=lambda node: self.hash(node, bytes(key, "utf-8")), - ) - return max_node.decode("utf-8") if max_node is not None else "" + ranked = self.rank_hybrid(key) + if ranked: + return ranked[0] + return "" + # HashMigration: use rank_sha256 after migration done def get_n(self, n: int, key: str) -> List[str]: - scores = [(self.hash(node, bytes(key, "utf-8")), node) for node in self.nodes] + return self.rank_hybrid(key)[:n] + + def rank_hybrid(self, key: str) -> List[str]: + legacy = self.rank_crc32(2, key) + modern = [h for h in self.rank_sha256(key) if h not in legacy] + hybrid = legacy + modern + return hybrid + + def rank_crc32(self, n: int, key: str) -> List[str]: + scores = [(self.hash(node, key), node) for node in self.nodes] scores.sort(key=lambda x: (-x[0], x[1])) - return [node.decode("utf-8") for _, node in scores[:n]] + return [t[1] for t in scores[:n]] - def get_nodes(self) -> List[str]: - return [node.decode("utf-8") for node in self.nodes] + def rank_sha256(self, key: str) -> List[str]: + tuples = [ + (h, sha256((h + key).encode("utf-8")).hexdigest()) for h in self.nodes + ] + tuples.sort(key=lambda t: (t[1], t[0])) + return [t[0] for t in tuples] @staticmethod - def hash(node: bytes, key: bytes) -> int: - combined = key + node + def hash(node: str, key: str) -> int: + combined = (key + node).encode("utf-8") # Convert to unsigned 32-bit integer to match golang uint32 here: https://github.com/tysonmote/rendezvous/blob/be0258dbbd3d/rendezvous.go#L92 return crc32c.crc32c(combined) & 0xFFFFFFFF diff --git a/packages/discovery-provider/src/utils/rendezvous_unit_test.py b/packages/discovery-provider/src/utils/rendezvous_unit_test.py index a55617a6fbc..a45b9222a97 100644 --- a/packages/discovery-provider/src/utils/rendezvous_unit_test.py +++ b/packages/discovery-provider/src/utils/rendezvous_unit_test.py @@ -29,7 +29,7 @@ def test_return_top_n_nodes(): nodes = ["node1", "node2", "node3"] hash_obj = RendezvousHash(*nodes) key = "test-key" - top2_nodes = hash_obj.get_n(2, key) + top2_nodes = hash_obj.rank_crc32(2, key) assert len(top2_nodes) == 2 assert all(node in nodes for node in top2_nodes) @@ -65,5 +65,47 @@ def test_hash_get(key, expected): def test_hash_get_n(n, key, expected): hash_obj = RendezvousHash() hash_obj.add("a", "b", "c", "d", "e") - got_nodes = hash_obj.get_n(n, key) + got_nodes = hash_obj.rank_crc32(n, key) assert got_nodes == expected + + +def test_hybrid(): + node_list = "https://creatornode.audius.prod-eks-ap-northeast-1.staked.cloud,https://creatornode.audius1.prod-eks-ap-northeast-1.staked.cloud,https://creatornode.audius2.prod-eks-ap-northeast-1.staked.cloud,https://creatornode.audius3.prod-eks-ap-northeast-1.staked.cloud,https://creatornode.audius8.prod-eks-ap-northeast-1.staked.cloud,https://creatornode.audius.co,https://creatornode2.audius.co,https://creatornode3.audius.co,https://usermetadata.audius.co,https://audius-content-1.cultur3stake.com,https://audius-content-10.cultur3stake.com,https://audius-content-11.cultur3stake.com,https://audius-content-12.cultur3stake.com,https://audius-content-13.cultur3stake.com,https://audius-content-14.cultur3stake.com,https://audius-content-15.cultur3stake.com,https://audius-content-16.cultur3stake.com,https://audius-content-17.cultur3stake.com,https://audius-content-18.cultur3stake.com,https://audius-content-2.cultur3stake.com,https://audius-content-3.cultur3stake.com,https://audius-content-4.cultur3stake.com,https://audius-content-5.cultur3stake.com,https://audius-content-6.cultur3stake.com,https://audius-content-7.cultur3stake.com,https://audius-content-8.cultur3stake.com,https://audius-content-9.cultur3stake.com,https://cn1.stuffisup.com,https://audius-cn1.tikilabs.com,https://audius.prod.capturealpha.io,https://audius-content-1.figment.io,https://audius-content-10.figment.io,https://audius-content-11.figment.io,https://audius-content-12.figment.io,https://audius-content-13.figment.io,https://audius-content-14.figment.io,https://audius-content-2.figment.io,https://audius-content-3.figment.io,https://audius-content-4.figment.io,https://audius-content-5.figment.io,https://audius-content-6.figment.io,https://audius-content-7.figment.io,https://audius-content-8.figment.io,https://audius-content-9.figment.io,https://blockchange-audius-content-01.bdnodes.net,https://blockchange-audius-content-02.bdnodes.net,https://blockchange-audius-content-03.bdnodes.net,https://blockdaemon-audius-content-01.bdnodes.net,https://blockdaemon-audius-content-02.bdnodes.net,https://blockdaemon-audius-content-03.bdnodes.net,https://blockdaemon-audius-content-04.bdnodes.net,https://blockdaemon-audius-content-05.bdnodes.net,https://blockdaemon-audius-content-06.bdnodes.net,https://blockdaemon-audius-content-07.bdnodes.net,https://blockdaemon-audius-content-08.bdnodes.net,https://blockdaemon-audius-content-09.bdnodes.net,https://content.grassfed.network,https://cn0.mainnet.audiusindex.org,https://cn1.mainnet.audiusindex.org,https://cn2.mainnet.audiusindex.org,https://cn3.mainnet.audiusindex.org,https://cn4.mainnet.audiusindex.org,https://audius-content-1.jollyworld.xyz,https://audius-creator-1.theblueprint.xyz,https://audius-creator-2.theblueprint.xyz,https://audius-creator-3.theblueprint.xyz,https://audius-creator-4.theblueprint.xyz,https://audius-creator-5.theblueprint.xyz,https://audius-creator-6.theblueprint.xyz".split( + "," + ) + test_cid = "baeaaaiqsedziwknj44jsl5fak6vcbszzjlnl7pqtw2ipnyg7rsh5a2xnql2p2" + hasher = RendezvousHash(*node_list) + + # sha256 + got = hasher.rank_sha256(test_cid)[:6] + expected = [ + "https://blockdaemon-audius-content-09.bdnodes.net", + "https://cn4.mainnet.audiusindex.org", + "https://audius-content-4.figment.io", + "https://cn0.mainnet.audiusindex.org", + "https://creatornode.audius3.prod-eks-ap-northeast-1.staked.cloud", + "https://blockdaemon-audius-content-07.bdnodes.net", + ] + assert got == expected + + # crc32 top 2 + got = hasher.rank_crc32(2, test_cid) + expected = [ + "https://audius-content-15.cultur3stake.com", + "https://audius-content-4.figment.io", + ] + assert got == expected + + # hybrid + got = hasher.get_n(6, test_cid) + expected = [ + # crc32 (top2) + "https://audius-content-15.cultur3stake.com", + "https://audius-content-4.figment.io", + # sha256 + "https://blockdaemon-audius-content-09.bdnodes.net", + "https://cn4.mainnet.audiusindex.org", + "https://cn0.mainnet.audiusindex.org", + "https://creatornode.audius3.prod-eks-ap-northeast-1.staked.cloud", + ] + assert got == expected