diff --git a/contracts/scripts/migrate-contracts.js b/contracts/scripts/migrate-contracts.js index 3ccee053c48..c085f0ec7fc 100644 --- a/contracts/scripts/migrate-contracts.js +++ b/contracts/scripts/migrate-contracts.js @@ -78,7 +78,7 @@ const outputJsonConfigFile = async (outputPath) => { let addressInfo = require(migrationOutputPath) let outputDictionary = {} outputDictionary['registryAddress'] = addressInfo.registryAddress - outputDictionary['audiusDataAddress'] = addressInfo.audiusDataProxyAddress + outputDictionary['entityManagerProxyAddress'] = addressInfo.entityManagerProxyAddress outputDictionary['ursmAddress'] = addressInfo.ursmAddress outputDictionary['ownerWallet'] = await getDefaultAccount() outputDictionary['allWallets'] = await web3.eth.getAccounts() diff --git a/discovery-provider/alembic/versions/f1e86fba0357_add_playlist_metadata.py b/discovery-provider/alembic/versions/f1e86fba0357_add_playlist_metadata.py new file mode 100644 index 00000000000..c94f7b9e4f9 --- /dev/null +++ b/discovery-provider/alembic/versions/f1e86fba0357_add_playlist_metadata.py @@ -0,0 +1,32 @@ +"""add playlist metadata + +Revision ID: f1e86fba0357 +Revises: 9931f7fd118f +Create Date: 2022-07-29 20:31:41.996125 + +""" +from alembic import op + +# revision identifiers, used by Alembic. +revision = "f1e86fba0357" +down_revision = "9931f7fd118f" +branch_labels = None +depends_on = None + + +def upgrade(): + connection = op.get_bind() + connection.execute( + """ + ALTER TABLE playlists ADD COLUMN IF NOT EXISTS metadata_multihash varchar; + """ + ) + + +def downgrade(): + connection = op.get_bind() + connection.execute( + """ + ALTER TABLE playlists DROP COLUMN IF EXISTS metadata_multihash; + """ + ) diff --git a/discovery-provider/default_config.ini b/discovery-provider/default_config.ini index ade11cf56c8..123e94ed5c9 100644 --- a/discovery-provider/default_config.ini +++ b/discovery-provider/default_config.ini @@ -71,6 +71,7 @@ allow_all = false [contracts] registry = +entity_manager_address = [eth_contracts] registry = diff --git a/discovery-provider/integration_tests/queries/test_get_playlist_id_occupied.py b/discovery-provider/integration_tests/queries/test_get_playlist_id_occupied.py new file mode 100644 index 00000000000..125c467ba8d --- /dev/null +++ b/discovery-provider/integration_tests/queries/test_get_playlist_id_occupied.py @@ -0,0 +1,33 @@ +from integration_tests.utils import populate_mock_db +from src.queries.get_playlist_is_occupied import _get_playlist_is_occupied +from src.utils.db_session import get_db + + +def populate_playlist(db): + test_entities = { + "playlists": [ + {"playlist_id": 1, "is_delete": True, "is_private": True, "is_album": True} + ], + "users": [ + {"user_id": 1, "handle": "user1"}, + ], + } + + populate_mock_db(db, test_entities) + + +def test_get_playlist_is_occupied(app): + """Test getting playlist is occupied""" + + with app.app_context(): + db = get_db() + + populate_playlist(db) + + with db.scoped_session() as session: + is_occupied = _get_playlist_is_occupied(session, 1) + + assert is_occupied == True + + is_occupied = _get_playlist_is_occupied(session, 2) + assert is_occupied == False diff --git a/discovery-provider/integration_tests/tasks/test_entity_manager.py b/discovery-provider/integration_tests/tasks/test_entity_manager.py new file mode 100644 index 00000000000..43725391260 --- /dev/null +++ b/discovery-provider/integration_tests/tasks/test_entity_manager.py @@ -0,0 +1,342 @@ +from typing import List + +from integration_tests.challenges.index_helpers import UpdateTask +from integration_tests.utils import populate_mock_db +from src.models.playlists.playlist import Playlist +from src.tasks.entity_manager.entity_manager import entity_manager_update +from src.tasks.entity_manager.utils import PLAYLIST_ID_OFFSET +from src.utils.db_session import get_db +from web3 import Web3 +from web3.datastructures import AttributeDict + + +def test_index_valid_playlists(app, mocker): + "Tests valid batch of playlists create/update/delete actions" + + # setup db and mocked txs + with app.app_context(): + db = get_db() + web3 = Web3() + update_task = UpdateTask(None, web3, None) + + tx_receipts = { + "CreatePlaylist1Tx": [ + { + "args": AttributeDict( + { + "_entityId": PLAYLIST_ID_OFFSET, + "_entityType": "Playlist", + "_userId": 1, + "_action": "Create", + "_metadata": "QmCreatePlaylist1", + "_signer": "user1wallet", + } + ) + }, + ], + "UpdatePlaylist1Tx": [ + { + "args": AttributeDict( + { + "_entityId": PLAYLIST_ID_OFFSET, + "_entityType": "Playlist", + "_userId": 1, + "_action": "Update", + "_metadata": "QmUpdatePlaylist1", + "_signer": "user1wallet", + } + ) + }, + ], + "DeletePlaylist1Tx": [ + { + "args": AttributeDict( + { + "_entityId": PLAYLIST_ID_OFFSET, + "_entityType": "Playlist", + "_userId": 1, + "_action": "Delete", + "_metadata": "", + "_signer": "user1wallet", + } + ) + }, + ], + "CreatePlaylist2Tx": [ + { + "args": AttributeDict( + { + "_entityId": PLAYLIST_ID_OFFSET + 1, + "_entityType": "Playlist", + "_userId": 1, + "_action": "Create", + "_metadata": "QmCreatePlaylist2", + "_signer": "user1wallet", + } + ) + }, + ], + } + + entity_manager_txs = [ + AttributeDict({"transactionHash": update_task.web3.toBytes(text=tx_receipt)}) + for tx_receipt in tx_receipts + ] + + def get_events_side_effect(_, tx_receipt): + return tx_receipts[tx_receipt.transactionHash.decode("utf-8")] + + mocker.patch( + "src.tasks.entity_manager.entity_manager.get_entity_manager_events_tx", + side_effect=get_events_side_effect, + autospec=True, + ) + test_metadata = { + "QmCreatePlaylist1": { + "playlist_contents": {"track_ids": []}, + "description": "", + "playlist_image_sizes_multihash": "", + "playlist_name": "playlist 1", + }, + "QmCreatePlaylist2": { + "playlist_contents": {"track_ids": []}, + "description": "test description", + "playlist_image_sizes_multihash": "", + "playlist_name": "playlist 2", + }, + "QmUpdatePlaylist1": { + "playlist_contents": {"track_ids": []}, + "description": "", + "playlist_image_sizes_multihash": "", + "playlist_name": "playlist 1 updated", + }, + } + + entities = { + "users": [ + {"user_id": 1, "handle": "user-1", "wallet": "user1wallet"}, + ] + } + populate_mock_db(db, entities) + + with db.scoped_session() as session: + # index transactions + entity_manager_update( + None, + update_task, + session, + entity_manager_txs, + block_number=0, + block_timestamp=1585336422, + block_hash=0, + ipfs_metadata=test_metadata, + ) + + # validate db records + all_playlists: List[Playlist] = session.query(Playlist).all() + assert len(all_playlists) == 4 + + playlist_1: Playlist = ( + session.query(Playlist) + .filter( + Playlist.is_current == True, Playlist.playlist_id == PLAYLIST_ID_OFFSET + ) + .first() + ) + assert playlist_1.playlist_name == "playlist 1 updated" + assert playlist_1.is_delete == True + + playlist_2: Playlist = ( + session.query(Playlist) + .filter( + Playlist.is_current == True, + Playlist.playlist_id == PLAYLIST_ID_OFFSET + 1, + ) + .first() + ) + assert playlist_2.playlist_name == "playlist 2" + assert playlist_2.is_delete == False + + +def test_index_invalid_playlists(app, mocker): + "Tests invalid batch of playlists create/update/delete actions" + + # setup db and mocked txs + with app.app_context(): + db = get_db() + web3 = Web3() + update_task = UpdateTask(None, web3, None) + + tx_receipts = { + # invalid create + "CreatePlaylistBelowOffset": [ + { + "args": AttributeDict( + { + "_entityId": 1, + "_entityType": "Playlist", + "_userId": 1, + "_action": "Create", + "_metadata": "", + "_signer": "user1wallet", + } + ) + }, + ], + "CreatePlaylistUserDoesNotExist": [ + { + "args": AttributeDict( + { + "_entityId": PLAYLIST_ID_OFFSET + 1, + "_entityType": "Playlist", + "_userId": 2, + "_action": "Create", + "_metadata": "", + "_signer": "user1wallet", + } + ) + }, + ], + "CreatePlaylistUserDoesNotMatchSigner": [ + { + "args": AttributeDict( + { + "_entityId": PLAYLIST_ID_OFFSET + 1, + "_entityType": "Playlist", + "_userId": 1, + "_action": "Create", + "_metadata": "", + "_signer": "InvalidWallet", + } + ) + }, + ], + "CreatePlaylistAlreadyExists": [ + { + "args": AttributeDict( + { + "_entityId": PLAYLIST_ID_OFFSET, + "_entityType": "Playlist", + "_userId": 1, + "_action": "Create", + "_metadata": "", + "_signer": "user1wallet", + } + ) + }, + ], + # invalid updates + "UpdatePlaylistInvalidSigner": [ + { + "args": AttributeDict( + { + "_entityId": PLAYLIST_ID_OFFSET, + "_entityType": "Playlist", + "_userId": 1, + "_action": "Update", + "_metadata": "", + "_signer": "InvalidWallet", + } + ) + }, + ], + "UpdatePlaylistInvalidOwner": [ + { + "args": AttributeDict( + { + "_entityId": PLAYLIST_ID_OFFSET, + "_entityType": "Playlist", + "_userId": 2, + "_action": "Update", + "_metadata": "", + "_signer": "User2Wallet", + } + ) + }, + ], + # invalid deletes + "DeletePlaylistInvalidSigner": [ + { + "args": AttributeDict( + { + "_entityId": PLAYLIST_ID_OFFSET, + "_entityType": "Playlist", + "_userId": 1, + "_action": "Delete", + "_metadata": "", + "_signer": "InvalidWallet", + } + ) + }, + ], + "DeletePlaylistDoesNotExist": [ + { + "args": AttributeDict( + { + "_entityId": PLAYLIST_ID_OFFSET + 1, + "_entityType": "Playlist", + "_userId": 1, + "_action": "Update", + "_metadata": "", + "_signer": "user1wallet", + } + ) + }, + ], + "DeletePlaylistInvalidOwner": [ + { + "args": AttributeDict( + { + "_entityId": PLAYLIST_ID_OFFSET + 1, + "_entityType": "Playlist", + "_userId": 2, + "_action": "Update", + "_metadata": "", + "_signer": "User2Wallet", + } + ) + }, + ], + } + + entity_manager_txs = [ + AttributeDict({"transactionHash": update_task.web3.toBytes(text=tx_receipt)}) + for tx_receipt in tx_receipts + ] + + def get_events_side_effect(_, tx_receipt): + return tx_receipts[tx_receipt.transactionHash.decode("utf-8")] + + mocker.patch( + "src.tasks.entity_manager.entity_manager.get_entity_manager_events_tx", + side_effect=get_events_side_effect, + autospec=True, + ) + + entities = { + "users": [ + {"user_id": 1, "handle": "user-1", "wallet": "user1wallet"}, + {"user_id": 2, "handle": "user-1", "wallet": "User2Wallet"}, + ], + "playlists": [ + {"playlist_id": PLAYLIST_ID_OFFSET, "playlist_owner_id": 1}, + ], + } + populate_mock_db(db, entities) + + with db.scoped_session() as session: + # index transactions + entity_manager_update( + None, + update_task, + session, + entity_manager_txs, + block_number=0, + block_timestamp=1585336422, + block_hash=0, + ipfs_metadata={}, + ) + + # validate db records + all_playlists: List[Playlist] = session.query(Playlist).all() + assert len(all_playlists) == 1 # no new playlists indexed diff --git a/discovery-provider/integration_tests/tasks/test_track_entity_manager.py b/discovery-provider/integration_tests/tasks/test_track_entity_manager.py new file mode 100644 index 00000000000..2d05abcc2cf --- /dev/null +++ b/discovery-provider/integration_tests/tasks/test_track_entity_manager.py @@ -0,0 +1,486 @@ +from typing import List + +from integration_tests.challenges.index_helpers import UpdateTask +from integration_tests.utils import populate_mock_db +from src.challenges.challenge_event_bus import ChallengeEventBus, setup_challenge_bus +from src.models.tracks.track import Track +from src.models.tracks.track_route import TrackRoute +from src.tasks.entity_manager.entity_manager import entity_manager_update +from src.tasks.entity_manager.utils import TRACK_ID_OFFSET +from src.utils.db_session import get_db +from web3 import Web3 +from web3.datastructures import AttributeDict + + +def test_index_valid_track(app, mocker): + "Tests valid batch of tracks create/update/delete actions" + + # setup db and mocked txs + with app.app_context(): + db = get_db() + web3 = Web3() + challenge_event_bus: ChallengeEventBus = setup_challenge_bus() + update_task = UpdateTask(None, web3, challenge_event_bus) + + tx_receipts = { + "CreateTrack1Tx": [ + { + "args": AttributeDict( + { + "_entityId": TRACK_ID_OFFSET, + "_entityType": "Track", + "_userId": 1, + "_action": "Create", + "_metadata": "QmCreateTrack1", + "_signer": "user1wallet", + } + ) + }, + ], + "UpdateTrack1Tx": [ + { + "args": AttributeDict( + { + "_entityId": TRACK_ID_OFFSET, + "_entityType": "Track", + "_userId": 1, + "_action": "Update", + "_metadata": "QmUpdateTrack1", + "_signer": "user1wallet", + } + ) + }, + ], + "DeleteTrack1Tx": [ + { + "args": AttributeDict( + { + "_entityId": TRACK_ID_OFFSET, + "_entityType": "Track", + "_userId": 1, + "_action": "Delete", + "_metadata": "", + "_signer": "user1wallet", + } + ) + }, + ], + "CreateTrack2Tx": [ + { + "args": AttributeDict( + { + "_entityId": TRACK_ID_OFFSET + 1, + "_entityType": "Track", + "_userId": 1, + "_action": "Create", + "_metadata": "QmCreateTrack2", + "_signer": "user1wallet", + } + ) + }, + ], + } + + entity_manager_txs = [ + AttributeDict({"transactionHash": update_task.web3.toBytes(text=tx_receipt)}) + for tx_receipt in tx_receipts + ] + + def get_events_side_effect(_, tx_receipt): + return tx_receipts[tx_receipt.transactionHash.decode("utf-8")] + + mocker.patch( + "src.tasks.entity_manager.entity_manager.get_entity_manager_events_tx", + side_effect=get_events_side_effect, + autospec=True, + ) + test_metadata = { + "QmCreateTrack1": { + "owner_id": 1, + "title": "track 1", + "length": None, + "cover_art": None, + "cover_art_sizes": "QmdxhDiRUC3zQEKqwnqksaSsSSeHiRghjwKzwoRvm77yaZ", + "tags": "realmagic,rickyreed,theroom", + "genre": "R&B/Soul", + "mood": "Empowering", + "credits_splits": None, + "created_at": "2020-07-11 08:22:15", + "create_date": None, + "updated_at": "2020-07-11 08:22:15", + "release_date": "Sat Jul 11 2020 01:19:58 GMT-0700", + "file_type": None, + "track_segments": [ + { + "duration": 6.016, + "multihash": "QmabM5svgDgcRdQZaEKSMBCpSZrrYy2y87L8Dx8EQ3T2jp", + } + ], + "has_current_user_reposted": False, + "is_current": True, + "is_unlisted": False, + "field_visibility": { + "mood": True, + "tags": True, + "genre": True, + "share": True, + "play_count": True, + "remixes": True, + }, + "remix_of": {"tracks": [{"parent_track_id": 75808}]}, + "repost_count": 12, + "save_count": 21, + "description": None, + "license": "All rights reserved", + "isrc": None, + "iswc": None, + "download": { + "cid": None, + "is_downloadable": False, + "requires_follow": False, + }, + "track_id": 77955, + "stem_of": None, + }, + "QmCreateTrack2": { + "owner_id": 1, + "title": "track 2", + "length": None, + "cover_art": None, + "cover_art_sizes": "QmQKXkVxGBbCFjcnhgxftzYDhph1CT8PJCuPEsRpffjjGC", + "tags": None, + "genre": "Electronic", + "mood": None, + "credits_splits": None, + "created_at": None, + "create_date": None, + "updated_at": None, + "release_date": None, + "file_type": None, + "track_segments": [], + "has_current_user_reposted": False, + "is_current": True, + "is_unlisted": False, + "field_visibility": { + "genre": True, + "mood": True, + "tags": True, + "share": True, + "play_count": True, + "remixes": True, + }, + "remix_of": None, + "repost_count": 0, + "save_count": 0, + "description": "", + "license": "", + "isrc": "", + "iswc": "", + }, + "QmUpdateTrack1": { + "owner_id": 1, + "title": "track 1 2", + "length": None, + "cover_art": None, + "cover_art_sizes": "QmdxhDiRUC3zQEKqwnqksaSsSSeHiRghjwKzwoRvm77yaZ", + "tags": "realmagic,rickyreed,theroom", + "genre": "R&B/Soul", + "mood": "Empowering", + "credits_splits": None, + "created_at": "2020-07-11 08:22:15", + "create_date": None, + "updated_at": "2020-07-11 08:22:15", + "release_date": "Sat Jul 11 2020 01:19:58 GMT-0700", + "file_type": None, + "track_segments": [ + { + "duration": 6.016, + "multihash": "QmabM5svgDgcRdQZaEKSMBCpSZrrYy2y87L8Dx8EQ3T2jp", + } + ], + "has_current_user_reposted": False, + "is_current": True, + "is_unlisted": False, + "field_visibility": { + "mood": True, + "tags": True, + "genre": True, + "share": True, + "play_count": True, + "remixes": True, + }, + "remix_of": {"tracks": [{"parent_track_id": 75808}]}, + "repost_count": 12, + "save_count": 21, + "description": "updated description", + "license": "All rights reserved", + "isrc": None, + "iswc": None, + "download": { + "cid": None, + "is_downloadable": False, + "requires_follow": False, + }, + "track_id": 77955, + "stem_of": None, + }, + } + + entities = { + "users": [ + {"user_id": 1, "handle": "user-1", "wallet": "user1wallet"}, + ] + } + populate_mock_db(db, entities) + + with db.scoped_session() as session: + # index transactions + entity_manager_update( + None, + update_task, + session, + entity_manager_txs, + block_number=0, + block_timestamp=1585336422, + block_hash=0, + ipfs_metadata=test_metadata, + ) + + # validate db records + all_tracks: List[Track] = session.query(Track).all() + assert len(all_tracks) == 4 + + track_1: Track = ( + session.query(Track) + .filter(Track.is_current == True, Track.track_id == TRACK_ID_OFFSET) + .first() + ) + assert track_1.description == "updated description" + assert track_1.is_delete == True + + track_2: Track = ( + session.query(Track) + .filter( + Track.is_current == True, + Track.track_id == TRACK_ID_OFFSET + 1, + ) + .first() + ) + assert track_2.title == "track 2" + assert track_2.is_delete == False + + # Check that track routes are updated appropriately + track_routes = ( + session.query(TrackRoute) + .filter(TrackRoute.track_id == TRACK_ID_OFFSET) + .all() + ) + # Should have the two routes created on track creation as well as two more for the update + assert len(track_routes) == 2, "Has two total routes after a track name update" + assert ( + len( + [ + route + for route in track_routes + if route.is_current is True and route.slug == "track-1-2" + ] + ) + == 1 + ), "The current route is 'track-1-2'" + assert ( + len([route for route in track_routes if route.is_current is False]) == 1 + ), "One route is marked non-current" + assert ( + len( + [ + route + for route in track_routes + if route.slug in ("track-1-2", "track-1") + ] + ) + == 2 + ), "Has both of the 'new-style' routes" + + +def test_index_invalid_tracks(app, mocker): + "Tests invalid batch of playlists create/update/delete actions" + + # setup db and mocked txs + with app.app_context(): + db = get_db() + web3 = Web3() + update_task = UpdateTask(None, web3, None) + + tx_receipts = { + # invalid create + "CreateTrackBelowOffset": [ + { + "args": AttributeDict( + { + "_entityId": 1, + "_entityType": "Track", + "_userId": 1, + "_action": "Create", + "_metadata": "", + "_signer": "user1wallet", + } + ) + }, + ], + "CreateTrackUserDoesNotExist": [ + { + "args": AttributeDict( + { + "_entityId": TRACK_ID_OFFSET + 1, + "_entityType": "Track", + "_userId": 2, + "_action": "Create", + "_metadata": "", + "_signer": "user1wallet", + } + ) + }, + ], + "CreateTrackUserDoesNotMatchSigner": [ + { + "args": AttributeDict( + { + "_entityId": TRACK_ID_OFFSET + 1, + "_entityType": "Track", + "_userId": 1, + "_action": "Create", + "_metadata": "", + "_signer": "InvalidWallet", + } + ) + }, + ], + "CreateTrackAlreadyExists": [ + { + "args": AttributeDict( + { + "_entityId": TRACK_ID_OFFSET, + "_entityType": "Track", + "_userId": 1, + "_action": "Create", + "_metadata": "", + "_signer": "user1wallet", + } + ) + }, + ], + # invalid updates + "UpdateTrackInvalidSigner": [ + { + "args": AttributeDict( + { + "_entityId": TRACK_ID_OFFSET, + "_entityType": "Track", + "_userId": 1, + "_action": "Update", + "_metadata": "", + "_signer": "InvalidWallet", + } + ) + }, + ], + "UpdateTrackInvalidOwner": [ + { + "args": AttributeDict( + { + "_entityId": TRACK_ID_OFFSET, + "_entityType": "Track", + "_userId": 2, + "_action": "Update", + "_metadata": "", + "_signer": "User2Wallet", + } + ) + }, + ], + # invalid deletes + "DeleteTrackInvalidSigner": [ + { + "args": AttributeDict( + { + "_entityId": TRACK_ID_OFFSET, + "_entityType": "Track", + "_userId": 1, + "_action": "Delete", + "_metadata": "", + "_signer": "InvalidWallet", + } + ) + }, + ], + "DeleteTrackDoesNotExist": [ + { + "args": AttributeDict( + { + "_entityId": TRACK_ID_OFFSET + 1, + "_entityType": "Track", + "_userId": 1, + "_action": "Update", + "_metadata": "", + "_signer": "user1wallet", + } + ) + }, + ], + "DeleteTrackInvalidOwner": [ + { + "args": AttributeDict( + { + "_entityId": TRACK_ID_OFFSET + 1, + "_entityType": "Track", + "_userId": 2, + "_action": "Update", + "_metadata": "", + "_signer": "User2Wallet", + } + ) + }, + ], + } + + entity_manager_txs = [ + AttributeDict({"transactionHash": update_task.web3.toBytes(text=tx_receipt)}) + for tx_receipt in tx_receipts + ] + + def get_events_side_effect(_, tx_receipt): + return tx_receipts[tx_receipt.transactionHash.decode("utf-8")] + + mocker.patch( + "src.tasks.entity_manager.entity_manager.get_entity_manager_events_tx", + side_effect=get_events_side_effect, + autospec=True, + ) + + entities = { + "users": [ + {"user_id": 1, "handle": "user-1", "wallet": "user1wallet"}, + {"user_id": 2, "handle": "user-1", "wallet": "User2Wallet"}, + ], + "tracks": [ + {"track_id": TRACK_ID_OFFSET, "owner_id": 1}, + ], + } + populate_mock_db(db, entities) + + with db.scoped_session() as session: + # index transactions + entity_manager_update( + None, + update_task, + session, + entity_manager_txs, + block_number=0, + block_timestamp=1585336422, + block_hash=0, + ipfs_metadata={}, + ) + + # validate db records + all_tracks: List[Track] = session.query(Track).all() + assert len(all_tracks) == 1 # no new playlists indexed diff --git a/discovery-provider/src/api/v1/helpers.py b/discovery-provider/src/api/v1/helpers.py index 4f7f5c7afa7..118e54b318b 100644 --- a/discovery-provider/src/api/v1/helpers.py +++ b/discovery-provider/src/api/v1/helpers.py @@ -66,7 +66,11 @@ def add_playlist_added_timestamps(playlist): added_timestamps = [] for track in playlist["playlist_contents"]["track_ids"]: added_timestamps.append( - {"track_id": encode_int_id(track["track"]), "timestamp": track["time"]} + { + "track_id": encode_int_id(track["track"]), + "timestamp": track["time"], + "metadata_timestamp": track.get("metadata_time"), + } ) return added_timestamps diff --git a/discovery-provider/src/api/v1/models/common.py b/discovery-provider/src/api/v1/models/common.py index 61d44054f96..1cf2a3da5db 100644 --- a/discovery-provider/src/api/v1/models/common.py +++ b/discovery-provider/src/api/v1/models/common.py @@ -38,3 +38,8 @@ "version": fields.Nested(version_metadata, required=True), }, ) + +id_occupied = ns.model( + "occupied", + {"is_occupied": fields.Boolean(required=True)}, +) diff --git a/discovery-provider/src/api/v1/models/playlists.py b/discovery-provider/src/api/v1/models/playlists.py index 8d7927e02c4..32809a06c69 100644 --- a/discovery-provider/src/api/v1/models/playlists.py +++ b/discovery-provider/src/api/v1/models/playlists.py @@ -16,6 +16,7 @@ playlist_added_timestamp = ns.model( "playlist_added_timestamp", { + "metadata_timestamp": fields.Integer(required=True), "timestamp": fields.Integer(required=True), "track_id": fields.String(required=True), }, diff --git a/discovery-provider/src/api/v1/playlists.py b/discovery-provider/src/api/v1/playlists.py index 6d0f517028f..ed4688b8274 100644 --- a/discovery-provider/src/api/v1/playlists.py +++ b/discovery-provider/src/api/v1/playlists.py @@ -21,8 +21,10 @@ success_response, trending_parser, ) +from src.api.v1.models.common import id_occupied from src.api.v1.models.playlists import full_playlist_model, playlist_model from src.api.v1.models.users import user_model_full +from src.queries.get_playlist_is_occupied import get_playlist_is_occupied from src.queries.get_playlist_tracks import get_playlist_tracks from src.queries.get_playlists import get_playlists from src.queries.get_reposters_for_playlist import get_reposters_for_playlist @@ -140,7 +142,6 @@ def get(self, playlist_id): playlist_id = decode_with_abort(playlist_id, full_ns) args = current_user_parser.parse_args() current_user_id = get_current_user_id(args) - playlist = get_playlist(playlist_id, current_user_id) if playlist: tracks = get_tracks_for_playlist(playlist_id, current_user_id) @@ -408,3 +409,25 @@ def get(self, version): ) playlists = get_full_trending_playlists(request, args, strategy) return success_response(playlists) + + +occupied_response = make_response("occupied_response", ns, fields.Nested(id_occupied)) +PLAYLIST_OCCUPIED_ROUTE = "//occupied" + + +@ns.route(PLAYLIST_OCCUPIED_ROUTE) +class GetPlaylistIsOccupied(Resource): + @record_metrics + @ns.doc( + id="""Check if Playlist ID is occupied""", + description="""Check if Playlist ID is occupied""", + params={"playlist_id": "A Playlist ID"}, + responses={200: "Success", 400: "Bad request", 500: "Server error"}, + ) + @ns.marshal_with(occupied_response) + @cache(ttl_sec=5) + def get(self, playlist_id): + playlist_id = decode_with_abort(playlist_id, ns) + is_occupied = get_playlist_is_occupied(playlist_id) + response = success_response({"is_occupied": is_occupied}) + return response diff --git a/discovery-provider/src/app.py b/discovery-provider/src/app.py index cd34809c037..b1d4ec0d1c9 100644 --- a/discovery-provider/src/app.py +++ b/discovery-provider/src/app.py @@ -63,6 +63,7 @@ playlist_factory = None user_library_factory = None user_replica_set_manager = None +entity_manager = None contract_addresses: Dict[str, Any] = defaultdict() logger = logging.getLogger(__name__) @@ -126,6 +127,16 @@ def init_contracts(): abi=abi_values["UserReplicaSetManager"]["abi"], ) + entity_manager_address = None + entity_manager_inst = None + if shared_config["contracts"]["entity_manager_address"]: + entity_manager_address = web3.toChecksumAddress( + shared_config["contracts"]["entity_manager_address"] + ) + entity_manager_inst = web3.eth.contract( + address=entity_manager_address, abi=abi_values["EntityManager"]["abi"] + ) + contract_address_dict = { "registry": registry_address, "user_factory": user_factory_address, @@ -134,6 +145,7 @@ def init_contracts(): "playlist_factory": playlist_factory_address, "user_library_factory": user_library_factory_address, "user_replica_set_manager": user_replica_set_manager_address, + "entity_manager": entity_manager_address, } return ( @@ -144,6 +156,7 @@ def init_contracts(): playlist_factory_inst, user_library_factory_inst, user_replica_set_manager_inst, + entity_manager_inst, contract_address_dict, ) @@ -176,6 +189,7 @@ def create_celery(test_config=None): global playlist_factory global user_library_factory global user_replica_set_manager + global entity_manager global contract_addresses # pylint: enable=W0603 @@ -187,6 +201,7 @@ def create_celery(test_config=None): playlist_factory, user_library_factory, user_replica_set_manager, + entity_manager, contract_addresses, ) = init_contracts() diff --git a/discovery-provider/src/models/playlists/playlist.py b/discovery-provider/src/models/playlists/playlist.py index 9d06394913f..e105ac411fb 100644 --- a/discovery-provider/src/models/playlists/playlist.py +++ b/discovery-provider/src/models/playlists/playlist.py @@ -33,6 +33,7 @@ class Playlist(Base, RepresentableMixin): ) last_added_to = Column(DateTime) slot = Column(Integer) + metadata_multihash = Column(String) block = relationship( # type: ignore "Block", primaryjoin="Playlist.blockhash == Block.blockhash" diff --git a/discovery-provider/src/queries/get_playlist_is_occupied.py b/discovery-provider/src/queries/get_playlist_is_occupied.py new file mode 100644 index 00000000000..8d7a655a7d8 --- /dev/null +++ b/discovery-provider/src/queries/get_playlist_is_occupied.py @@ -0,0 +1,31 @@ +import logging # pylint: disable=C0302 + +from sqlalchemy.orm import Session +from src.models.playlists.playlist import Playlist +from src.utils.db_session import get_db_read_replica + +logger = logging.getLogger(__name__) + + +def get_playlist_is_occupied(playlist_id: int): + db = get_db_read_replica() + with db.scoped_session() as session: + return _get_playlist_is_occupied(session, playlist_id) + + +def _get_playlist_is_occupied(session: Session, playlist_id: int): + x = session.query( + session.query(Playlist) + .filter(Playlist.is_current == True, Playlist.playlist_id == playlist_id) + .exists() + ) + logger.info(x) + logger.info(x) + logger.info(x) + + playlist_exists = session.query( + session.query(Playlist) + .filter(Playlist.is_current == True, Playlist.playlist_id == playlist_id) + .exists() + ).scalar() + return playlist_exists diff --git a/discovery-provider/src/queries/get_playlists.py b/discovery-provider/src/queries/get_playlists.py index e3fce4a6f0b..c0de38bde9c 100644 --- a/discovery-provider/src/queries/get_playlists.py +++ b/discovery-provider/src/queries/get_playlists.py @@ -1,7 +1,6 @@ import logging # pylint: disable=C0302 import sqlalchemy -from flask.globals import request from sqlalchemy import desc from src import exceptions from src.models.playlists.playlist import Playlist @@ -15,26 +14,12 @@ ) from src.utils import helpers from src.utils.db_session import get_db_read_replica -from src.utils.redis_cache import extract_key, use_redis_cache logger = logging.getLogger(__name__) UNPOPULATED_PLAYLIST_CACHE_DURATION_SEC = 10 -def make_cache_key(args): - cache_keys = {"user_id": args.get("user_id"), "with_users": args.get("with_users")} - - if args.get("playlist_id"): - ids = args.get("playlist_id") - ids = map(str, ids) - ids = ",".join(ids) - cache_keys["playlist_id"] = ids - - key = extract_key(f"unpopulated-playlist:{request.path}", cache_keys.items()) - return key - - def get_playlists(args): playlists = [] current_user_id = args.get("current_user_id") @@ -91,18 +76,10 @@ def get_unpopulated_playlists(): playlist_ids = list( map(lambda playlist: playlist["playlist_id"], playlists) ) - return (playlists, playlist_ids) try: - # Get unpopulated playlists, either via - # redis cache or via get_unpopulated_playlists - key = make_cache_key(args) - - (playlists, playlist_ids) = use_redis_cache( - key, UNPOPULATED_PLAYLIST_CACHE_DURATION_SEC, get_unpopulated_playlists - ) - + (playlists, playlist_ids) = get_unpopulated_playlists() # bundle peripheral info into playlist results playlists = populate_playlist_metadata( session, diff --git a/discovery-provider/src/solana/audius_data_transaction_handlers.py b/discovery-provider/src/solana/audius_data_transaction_handlers.py index 4a74ff05d11..ccbcba083d1 100644 --- a/discovery-provider/src/solana/audius_data_transaction_handlers.py +++ b/discovery-provider/src/solana/audius_data_transaction_handlers.py @@ -511,7 +511,7 @@ def update_track_model_metadata( session: Session, track_record: Track, track_metadata: Dict ): track_record.title = track_metadata["title"] - track_record.length = track_metadata["length"] or 0 + track_record.length = track_metadata.get("length", 0) or 0 track_record.cover_art_sizes = track_metadata["cover_art_sizes"] if track_metadata["cover_art"]: track_record.cover_art_sizes = track_record.cover_art diff --git a/discovery-provider/src/tasks/entity_manager/entity_manager.py b/discovery-provider/src/tasks/entity_manager/entity_manager.py new file mode 100644 index 00000000000..34b9642a91c --- /dev/null +++ b/discovery-provider/src/tasks/entity_manager/entity_manager.py @@ -0,0 +1,202 @@ +import logging +from collections import defaultdict +from typing import Any, Dict, List, Set, Tuple + +from sqlalchemy.orm.session import Session +from src.challenges.challenge_event_bus import ChallengeEventBus +from src.database_task import DatabaseTask +from src.models.playlists.playlist import Playlist +from src.models.tracks.track import Track +from src.models.tracks.track_route import TrackRoute +from src.models.users.user import User +from src.tasks.entity_manager.playlist import ( + create_playlist, + delete_playlist, + update_playlist, +) +from src.tasks.entity_manager.track import create_track, delete_track, update_track +from src.tasks.entity_manager.utils import ( + MANAGE_ENTITY_EVENT_TYPE, + Action, + EntityType, + ExistingRecordDict, + ManageEntityParameters, + RecordDict, +) +from src.utils import helpers + +logger = logging.getLogger(__name__) + + +def entity_manager_update( + _, # main indexing task + update_task: DatabaseTask, + session: Session, + entity_manager_txs: List[Any], + block_number: int, + block_timestamp, + block_hash: str, + ipfs_metadata: Dict, +) -> Tuple[int, Dict[str, Set[(int)]]]: + try: + challenge_bus: ChallengeEventBus = update_task.challenge_event_bus + + num_total_changes = 0 + event_blockhash = update_task.web3.toHex(block_hash) + + changed_entity_ids: Dict[str, Set[(int)]] = defaultdict(set) + + if not entity_manager_txs: + return num_total_changes, changed_entity_ids + + # collect events by entity type and action + entities_to_fetch = collect_entities_to_fetch(update_task, entity_manager_txs) + + # fetch existing playlists + existing_records: ExistingRecordDict = fetch_existing_entities( + session, entities_to_fetch + ) + + new_records: RecordDict = { + "playlists": defaultdict(list), + "tracks": defaultdict(list), + } + + pending_track_routes: List[TrackRoute] = [] + + # process in tx order and populate playlists_to_save + for tx_receipt in entity_manager_txs: + txhash = update_task.web3.toHex(tx_receipt.transactionHash) + entity_manager_event_tx = get_entity_manager_events_tx( + update_task, tx_receipt + ) + for event in entity_manager_event_tx: + params = ManageEntityParameters( + session, + challenge_bus, + event, + new_records, # actions below populate these records + existing_records, + pending_track_routes, + ipfs_metadata, + block_timestamp, + block_number, + event_blockhash, + txhash, + ) + if ( + params.action == Action.CREATE + and params.entity_type == EntityType.PLAYLIST + ): + create_playlist(params) + elif ( + params.action == Action.UPDATE + and params.entity_type == EntityType.PLAYLIST + ): + update_playlist(params) + elif ( + params.action == Action.DELETE + and params.entity_type == EntityType.PLAYLIST + ): + delete_playlist(params) + elif ( + params.action == Action.CREATE + and params.entity_type == EntityType.TRACK + ): + create_track(params) + elif ( + params.action == Action.UPDATE + and params.entity_type == EntityType.TRACK + ): + update_track(params) + + elif ( + params.action == Action.DELETE + and params.entity_type == EntityType.TRACK + ): + delete_track(params) + + logger.info(new_records) + + # compile records_to_save + records_to_save = [] + for playlist_records in new_records["playlists"].values(): + # flip is_current to true for the last tx in each playlist + playlist_records[-1].is_current = True + records_to_save.extend(playlist_records) + + for track_records in new_records["tracks"].values(): + # flip is_current to true for the last tx in each playlist + track_records[-1].is_current = True + records_to_save.extend(track_records) + + # insert/update all tracks, playlist records in this block + session.bulk_save_objects(records_to_save) + num_total_changes += len(records_to_save) + + except Exception as e: + logger.error(f"Exception occurred {e}", exc_info=True) + raise e + return num_total_changes, changed_entity_ids + + +def collect_entities_to_fetch( + update_task, + entity_manager_txs, +): + entities_to_fetch: Dict[EntityType, Set[int]] = defaultdict(set) + for tx_receipt in entity_manager_txs: + entity_manager_event_tx = get_entity_manager_events_tx(update_task, tx_receipt) + for event in entity_manager_event_tx: + entity_id = helpers.get_tx_arg(event, "_entityId") + entity_type = helpers.get_tx_arg(event, "_entityType") + user_id = helpers.get_tx_arg(event, "_userId") + + entities_to_fetch[entity_type].add(entity_id) + entities_to_fetch[EntityType.USER].add(user_id) + return entities_to_fetch + + +def fetch_existing_entities( + session: Session, entities_to_fetch: Dict[EntityType, Set[int]] +): + existing_entities: ExistingRecordDict = {} + playlists: List[Playlist] = ( + session.query(Playlist) + .filter( + Playlist.playlist_id.in_(entities_to_fetch[EntityType.PLAYLIST]), + Playlist.is_current == True, + ) + .all() + ) + existing_entities["playlists"] = { + playlist.playlist_id: playlist for playlist in playlists + } + + tracks: List[Track] = ( + session.query(Track) + .filter( + Track.track_id.in_(entities_to_fetch[EntityType.TRACK]), + Track.is_current == True, + ) + .all() + ) + existing_entities["tracks"] = {track.track_id: track for track in tracks} + + users: List[User] = ( + session.query(User) + .filter( + User.user_id.in_(entities_to_fetch[EntityType.USER]), + User.is_current == True, + ) + .all() + ) + existing_entities["users"] = {user.user_id: user for user in users} + + return existing_entities + + +def get_entity_manager_events_tx(update_task, tx_receipt): + return getattr( + update_task.entity_manager_contract.events, MANAGE_ENTITY_EVENT_TYPE + )().processReceipt(tx_receipt) diff --git a/discovery-provider/src/tasks/entity_manager/playlist.py b/discovery-provider/src/tasks/entity_manager/playlist.py new file mode 100644 index 00000000000..656178466a5 --- /dev/null +++ b/discovery-provider/src/tasks/entity_manager/playlist.py @@ -0,0 +1,247 @@ +import logging +from collections import defaultdict +from typing import Dict, Set + +from src.models.playlists.playlist import Playlist +from src.tasks.entity_manager.utils import ( + PLAYLIST_ID_OFFSET, + Action, + EntityType, + ManageEntityParameters, +) + +logger = logging.getLogger(__name__) + + +def is_valid_playlist_tx(params: ManageEntityParameters): + user_id = params.user_id + playlist_id = params.entity_id + if user_id not in params.existing_records["users"]: + # user does not exist + return False + + wallet = params.existing_records["users"][user_id].wallet + if wallet and wallet.lower() != params.signer.lower(): + # user does not match signer + return False + + if params.entity_type != EntityType.PLAYLIST: + return False + + if params.action == Action.CREATE: + if playlist_id in params.existing_records["playlists"]: + # playlist already exists + return False + if playlist_id < PLAYLIST_ID_OFFSET: + return False + else: + # update / delete specific validations + if playlist_id not in params.existing_records["playlists"]: + # playlist does not exist + return False + existing_playlist: Playlist = params.existing_records["playlists"][playlist_id] + if existing_playlist.playlist_owner_id != user_id: + # existing playlist does not match user + return False + + return True + + +def create_playlist(params: ManageEntityParameters): + if not is_valid_playlist_tx(params): + return + + playlist_id = params.entity_id + metadata = params.ipfs_metadata[params.metadata_cid] + tracks = metadata["playlist_contents"].get("track_ids", []) + tracks_with_index_time = [] + for track in tracks: + tracks_with_index_time.append( + { + "track": track["track"], + "metadata_time": track["time"], + "time": params.block_integer_time, + } + ) + logger.info("making model") + create_playlist_record = Playlist( + playlist_id=playlist_id, + metadata_multihash=params.metadata_cid, + playlist_owner_id=params.user_id, + is_album=metadata.get("is_album", False), + description=metadata["description"], + playlist_image_multihash=metadata["playlist_image_sizes_multihash"], + playlist_image_sizes_multihash=metadata["playlist_image_sizes_multihash"], + playlist_name=metadata["playlist_name"], + is_private=metadata.get("is_private", False), + playlist_contents={"track_ids": tracks_with_index_time}, + created_at=params.block_datetime, + updated_at=params.block_datetime, + blocknumber=params.block_number, + blockhash=params.event_blockhash, + txhash=params.txhash, + is_current=False, + is_delete=False, + ) + logger.info("adding playlist info") + logger.info("adding playlist info") + logger.info("adding playlist info") + params.add_playlist_record(playlist_id, create_playlist_record) + + +def update_playlist(params: ManageEntityParameters): + if not is_valid_playlist_tx(params): + return + # TODO ignore updates on deleted playlists? + + playlist_id = params.entity_id + metadata = params.ipfs_metadata[params.metadata_cid] + existing_playlist = params.existing_records["playlists"][playlist_id] + existing_playlist.is_current = False # invalidate + if ( + playlist_id in params.new_records["playlists"] + ): # override with last updated playlist is in this block + existing_playlist = params.new_records["playlists"][playlist_id][-1] + + updated_playlist = copy_record( + existing_playlist, params.block_number, params.event_blockhash, params.txhash + ) + process_playlist_data_event( + updated_playlist, + metadata, + params.block_integer_time, + params.block_datetime, + params.metadata_cid, + ) + params.add_playlist_record(playlist_id, updated_playlist) + + +def delete_playlist(params: ManageEntityParameters): + if not is_valid_playlist_tx(params): + return + + existing_playlist = params.existing_records["playlists"][params.entity_id] + existing_playlist.is_current = False # invalidate old playlist + if params.entity_id in params.new_records["playlists"]: + # override with last updated playlist is in this block + existing_playlist = params.new_records["playlists"][params.entity_id][-1] + + deleted_playlist = copy_record( + existing_playlist, params.block_number, params.event_blockhash, params.txhash + ) + deleted_playlist.is_delete = True + + params.new_records["playlists"][params.entity_id].append(deleted_playlist) + + +def copy_record(old_playlist: Playlist, block_number, event_blockhash, txhash): + new_playlist = Playlist( + playlist_id=old_playlist.playlist_id, + playlist_owner_id=old_playlist.playlist_owner_id, + is_album=old_playlist.is_album, + description=old_playlist.description, + playlist_image_multihash=old_playlist.playlist_image_multihash, + playlist_image_sizes_multihash=old_playlist.playlist_image_sizes_multihash, + playlist_name=old_playlist.playlist_name, + is_private=old_playlist.is_private, + playlist_contents=old_playlist.playlist_contents, + created_at=old_playlist.created_at, + updated_at=old_playlist.updated_at, + blocknumber=block_number, + blockhash=event_blockhash, + txhash=txhash, + is_current=False, + is_delete=old_playlist.is_delete, + metadata_multihash=old_playlist.metadata_multihash, + ) + return new_playlist + + +def process_playlist_contents(playlist_record, playlist_metadata, block_integer_time): + if playlist_record.metadata_multihash: + # playlist already has metadata + metadata_index_time_dict: Dict[int, Dict[int, int]] = defaultdict(dict) + for track in playlist_record.playlist_contents["track_ids"]: + track_id = track["track"] + metadata_time = track["metadata_time"] + metadata_index_time_dict[track_id][metadata_time] = track["time"] + + updated_tracks = [] + for track in playlist_metadata["playlist_contents"]["track_ids"]: + track_id = track["track"] + metadata_time = track["time"] + index_time = block_integer_time # default to current block for new tracks + + if ( + track_id in metadata_index_time_dict + and metadata_time in metadata_index_time_dict[track_id] + ): + # track exists in prev record (reorder / delete) + index_time = metadata_index_time_dict[track_id][metadata_time] + + updated_tracks.append( + { + "track": track_id, + "time": index_time, + "metadata_time": metadata_time, + } + ) + else: + # upgrade legacy playlist to include metadata + # assume metadata and indexing timestamp is the same + track_id_index_times: Set = set() + for track in playlist_record.playlist_contents["track_ids"]: + track_id = track["track"] + index_time = track["time"] + track_id_index_times.add((track_id, index_time)) + + updated_tracks = [] + for track in playlist_metadata["playlist_contents"]["track_ids"]: + track_id = track["track"] + metadata_time = track["time"] + + # use track["time"] if present in previous record else this is a new track + index_time = ( + track["time"] + if (track_id, metadata_time) in track_id_index_times + else block_integer_time + ) + updated_tracks.append( + { + "track": track_id, + "time": index_time, + "metadata_time": metadata_time, + } + ) + + return {"track_ids": updated_tracks} + + +def process_playlist_data_event( + playlist_record: Playlist, + playlist_metadata, + block_integer_time, + block_datetime, + metadata_cid, +): + playlist_record.is_album = ( + playlist_metadata["is_album"] if "is_album" in playlist_metadata else False + ) + playlist_record.description = playlist_metadata["description"] + playlist_record.playlist_image_multihash = playlist_metadata[ + "playlist_image_sizes_multihash" + ] + playlist_record.playlist_image_sizes_multihash = playlist_metadata[ + "playlist_image_sizes_multihash" + ] + playlist_record.playlist_name = playlist_metadata["playlist_name"] + playlist_record.is_private = ( + playlist_metadata["is_private"] if "is_private" in playlist_metadata else False + ) + playlist_record.playlist_contents = process_playlist_contents( + playlist_record, playlist_metadata, block_integer_time + ) + playlist_record.updated_at = block_datetime + playlist_record.metadata_multihash = metadata_cid + + logger.info(f"index.py | AudiusData | Updated playlist record {playlist_record}") diff --git a/discovery-provider/src/tasks/entity_manager/track.py b/discovery-provider/src/tasks/entity_manager/track.py new file mode 100644 index 00000000000..b44efb2c508 --- /dev/null +++ b/discovery-provider/src/tasks/entity_manager/track.py @@ -0,0 +1,199 @@ +import logging +from typing import Dict + +from src.models.tracks.track import Track +from src.models.users.user import User +from src.tasks.entity_manager.utils import ( + TRACK_ID_OFFSET, + Action, + EntityType, + ManageEntityParameters, +) +from src.tasks.tracks import ( + dispatch_challenge_track_upload, + populate_track_record_metadata, + update_remixes_table, + update_stems_table, + update_track_routes_table, +) + +logger = logging.getLogger(__name__) + + +def is_valid_track_tx(params: ManageEntityParameters): + user_id = params.user_id + track_id = params.entity_id + if user_id not in params.existing_records["users"]: + # user does not exist + return False + + wallet = params.existing_records["users"][user_id].wallet + if wallet and wallet.lower() != params.signer.lower(): + # user does not match signer + return False + + if params.entity_type != EntityType.TRACK: + return False + + if params.action == Action.CREATE: + if track_id in params.existing_records["tracks"]: + # playlist already exists + return False + if track_id < TRACK_ID_OFFSET: + return False + else: + # update / delete specific validations + if track_id not in params.existing_records["tracks"]: + # playlist does not exist + return False + existing_track: Track = params.existing_records["tracks"][track_id] + if existing_track.owner_id != params.user_id: + # existing playlist does not match user + return False + + return True + + +def copy_track_record( + old_track: Track, block_number: int, event_blockhash: str, txhash: str +): + return Track( + track_id=old_track.track_id, + owner_id=old_track.owner_id, + title=old_track.title, + length=old_track.length, + cover_art=old_track.cover_art, + tags=old_track.tags, + genre=old_track.genre, + mood=old_track.mood, + credits_splits=old_track.credits_splits, + create_date=old_track.create_date, + release_date=old_track.release_date, + file_type=old_track.file_type, + metadata_multihash=old_track.metadata_multihash, + track_segments=old_track.track_segments, + description=old_track.description, + isrc=old_track.isrc, + iswc=old_track.iswc, + license=old_track.license, + cover_art_sizes=old_track.cover_art_sizes, + download=old_track.download, + is_unlisted=old_track.is_unlisted, + field_visibility=old_track.field_visibility, + route_id=old_track.route_id, + stem_of=old_track.stem_of, + remix_of=old_track.remix_of, + is_available=old_track.is_available, + is_delete=old_track.is_delete, + created_at=old_track.created_at, + updated_at=old_track.updated_at, + blocknumber=block_number, + blockhash=event_blockhash, + txhash=txhash, + is_current=False, + ) + + +def get_handle(params: ManageEntityParameters): + # TODO: get the track owner user handle + handle = ( + params.session.query(User.handle) + .filter(User.user_id == params.user_id, User.is_current == True) + .first() + )[0] + if not handle: + logger.error("missing track user in entity manager handle track") + return handle + + +def update_track_record(params: ManageEntityParameters, track: Track, metadata: Dict): + handle = get_handle(params) + populate_track_record_metadata(track, metadata, handle) + track.metadata_multihash = params.metadata_cid + # if cover_art CID is of a dir, store under _sizes field instead + if track.cover_art: + logger.info( + f"index.py | tracks.py | Processing track cover art {track.cover_art}" + ) + track.cover_art_sizes = track.cover_art + track.cover_art = None + + +def create_track(params: ManageEntityParameters): + if not is_valid_track_tx(params): + return + + track_id = params.entity_id + owner_id = params.user_id + track_metadata = params.ipfs_metadata[params.metadata_cid] + + track_record = Track( + track_id=track_id, + owner_id=owner_id, + txhash=params.txhash, + blockhash=params.event_blockhash, + blocknumber=params.block_number, + created_at=params.block_datetime, + updated_at=params.block_datetime, + is_delete=False, + ) + + update_track_routes_table( + params.session, track_record, track_metadata, params.pending_track_routes + ) + + update_track_record(params, track_record, track_metadata) + + update_stems_table(params.session, track_record, track_metadata) + update_remixes_table(params.session, track_record, track_metadata) + dispatch_challenge_track_upload( + params.challenge_bus, params.block_number, track_record + ) + + params.add_track_record(track_id, track_record) + + +def update_track(params: ManageEntityParameters): + if not is_valid_track_tx(params): + return + # TODO ignore updates on deleted playlists? + + track_metadata = params.ipfs_metadata[params.metadata_cid] + track_id = params.entity_id + existing_track = params.existing_records["tracks"][track_id] + existing_track.is_current = False # invalidate + if ( + track_id in params.new_records["tracks"] + ): # override with last updated playlist is in this block + existing_track = params.new_records["tracks"][track_id][-1] + + updated_track = copy_track_record( + existing_track, params.block_number, params.event_blockhash, params.txhash + ) + + update_track_routes_table( + params.session, updated_track, track_metadata, params.pending_track_routes + ) + update_track_record(params, updated_track, track_metadata) + update_remixes_table(params.session, updated_track, track_metadata) + + params.add_track_record(track_id, updated_track) + + +def delete_track(params: ManageEntityParameters): + if not is_valid_track_tx(params): + return + + track_id = params.entity_id + existing_track = params.existing_records["tracks"][track_id] + existing_track.is_current = False # invalidate old playlist + if params.entity_id in params.new_records["tracks"]: + # override with last updated playlist is in this block + existing_track = params.new_records["tracks"][params.entity_id][-1] + + deleted_track = copy_track_record( + existing_track, params.block_number, params.event_blockhash, params.txhash + ) + deleted_track.is_delete = True + + params.add_track_record(track_id, deleted_track) diff --git a/discovery-provider/src/tasks/entity_manager/utils.py b/discovery-provider/src/tasks/entity_manager/utils.py new file mode 100644 index 00000000000..410990ef010 --- /dev/null +++ b/discovery-provider/src/tasks/entity_manager/utils.py @@ -0,0 +1,91 @@ +from datetime import datetime +from enum import Enum +from typing import Dict, List, TypedDict + +from src.challenges.challenge_event_bus import ChallengeEventBus +from src.models.playlists.playlist import Playlist +from src.models.tracks.track import Track +from src.models.tracks.track_route import TrackRoute +from src.models.users.user import User +from src.utils import helpers +from web3.datastructures import AttributeDict + +PLAYLIST_ID_OFFSET = 400_000 +TRACK_ID_OFFSET = 1_000_000 + + +class Action(str, Enum): + CREATE = "Create" + UPDATE = "Update" + DELETE = "Delete" + + def __str__(self) -> str: + return str.__str__(self) + + +class EntityType(str, Enum): + PLAYLIST = "Playlist" + TRACK = "Track" + USER = "User" + + def __str__(self) -> str: + return str.__str__(self) + + +class RecordDict(TypedDict): + playlists: Dict[int, List[Playlist]] + tracks: Dict[int, List[Track]] + + +class ExistingRecordDict(TypedDict): + playlists: Dict[int, Playlist] + tracks: Dict[int, Track] + users: Dict[int, User] + + +MANAGE_ENTITY_EVENT_TYPE = "ManageEntity" + + +class ManageEntityParameters: + def __init__( + self, + session, + challenge_bus: ChallengeEventBus, + event: AttributeDict, + new_records: RecordDict, + existing_records: ExistingRecordDict, + pending_track_routes: List[TrackRoute], + ipfs_metadata: Dict[str, Dict[str, Dict]], + block_timestamp: int, + block_number: int, + event_blockhash: str, + txhash: str, + ): + self.user_id = helpers.get_tx_arg(event, "_userId") + self.entity_id = helpers.get_tx_arg(event, "_entityId") + self.entity_type = helpers.get_tx_arg(event, "_entityType") + self.action = helpers.get_tx_arg(event, "_action") + self.metadata_cid = helpers.get_tx_arg(event, "_metadata") + self.signer = helpers.get_tx_arg(event, "_signer") + self.block_datetime = datetime.utcfromtimestamp(block_timestamp) + self.block_integer_time = int(block_timestamp) + + self.session = session + self.challenge_bus = challenge_bus + self.pending_track_routes = pending_track_routes + + self.event = event + self.ipfs_metadata = ipfs_metadata + self.block_number = block_number + self.event_blockhash = event_blockhash + self.txhash = txhash + self.new_records = new_records + self.existing_records = existing_records + + def add_playlist_record(self, playlist_id: int, playlist: Playlist): + self.new_records["playlists"][playlist_id].append(playlist) + self.existing_records["playlists"][playlist_id] = playlist + + def add_track_record(self, track_id: int, track: Track): + self.new_records["tracks"][track_id].append(track) + self.existing_records["tracks"][track_id] = track diff --git a/discovery-provider/src/tasks/index.py b/discovery-provider/src/tasks/index.py index bb90ce369ca..20049c46782 100644 --- a/discovery-provider/src/tasks/index.py +++ b/discovery-provider/src/tasks/index.py @@ -31,6 +31,7 @@ ) from src.queries.skipped_transactions import add_network_level_skipped_transaction from src.tasks.celery_app import celery +from src.tasks.entity_manager.entity_manager import entity_manager_update from src.tasks.playlists import playlist_state_update from src.tasks.social_features import social_feature_state_update from src.tasks.sort_block_transactions import sort_block_transactions @@ -66,6 +67,7 @@ most_recent_indexed_block_redis_key, ) from src.utils.session_manager import SessionManager +from src.utils.user_event_constants import entity_manager_event_types_arr USER_FACTORY = CONTRACT_TYPES.USER_FACTORY.value TRACK_FACTORY = CONTRACT_TYPES.TRACK_FACTORY.value @@ -73,6 +75,7 @@ PLAYLIST_FACTORY = CONTRACT_TYPES.PLAYLIST_FACTORY.value USER_LIBRARY_FACTORY = CONTRACT_TYPES.USER_LIBRARY_FACTORY.value USER_REPLICA_SET_MANAGER = CONTRACT_TYPES.USER_REPLICA_SET_MANAGER.value +ENTITY_MANAGER = CONTRACT_TYPES.ENTITY_MANAGER.value USER_FACTORY_CONTRACT_NAME = CONTRACT_NAMES_ON_CHAIN[CONTRACT_TYPES.USER_FACTORY] TRACK_FACTORY_CONTRACT_NAME = CONTRACT_NAMES_ON_CHAIN[CONTRACT_TYPES.TRACK_FACTORY] @@ -88,6 +91,7 @@ USER_REPLICA_SET_MANAGER_CONTRACT_NAME = CONTRACT_NAMES_ON_CHAIN[ CONTRACT_TYPES.USER_REPLICA_SET_MANAGER ] +ENTITY_MANAGER_CONTRACT_NAME = CONTRACT_NAMES_ON_CHAIN[CONTRACT_TYPES.ENTITY_MANAGER] TX_TYPE_TO_HANDLER_MAP = { USER_FACTORY: user_state_update, @@ -96,6 +100,7 @@ PLAYLIST_FACTORY: playlist_state_update, USER_LIBRARY_FACTORY: user_library_state_update, USER_REPLICA_SET_MANAGER: user_replica_set_state_update, + ENTITY_MANAGER: entity_manager_update, } BLOCKS_PER_DAY = (24 * 60 * 60) / 5 @@ -262,14 +267,11 @@ def fetch_tx_receipts(self, block): return block_tx_with_receipts -def fetch_cid_metadata( - db, - user_factory_txs, - track_factory_txs, -): +def fetch_cid_metadata(db, user_factory_txs, track_factory_txs, entity_manager_txs): start_time = datetime.now() user_contract = update_task.user_contract track_contract = update_task.track_contract + entity_manager_contract = update_task.entity_manager_contract cids_txhash_set: Tuple[str, Any] = set() cid_type: Dict[str, str] = {} # cid -> entity type track / user @@ -315,6 +317,23 @@ def fetch_cid_metadata( cid_type[cid] = "track" cid_to_user_id[cid] = track_owner_id + for tx_receipt in entity_manager_txs: + txhash = update_task.web3.toHex(tx_receipt.transactionHash) + for event_type in entity_manager_event_types_arr: + entity_manager_events_tx = getattr( + entity_manager_contract.events, event_type + )().processReceipt(tx_receipt) + for entry in entity_manager_events_tx: + event_args = entry["args"] + user_id = event_args._userId + cid = event_args._metadata + if not cid: + continue + + cids_txhash_set.add((cid, txhash)) + cid_to_user_id[cid] = user_id + cid_type[cid] = "playlist_data" + # user -> replica set string lookup, used to make user and track cid get_metadata fetches faster user_to_replica_set = dict( session.query(User.user_id, User.creator_node_endpoint) @@ -430,6 +449,7 @@ def get_contract_type_for_tx(tx_type_to_grouped_lists_map, tx, tx_receipt): tx_target_contract_address = tx["to"] contract_type = None for tx_type in tx_type_to_grouped_lists_map.keys(): + logger.info(f"index.py | checking {tx_type} vs {tx_target_contract_address}") tx_is_type = tx_target_contract_address == get_contract_addresses()[tx_type] if tx_is_type: contract_type = tx_type @@ -438,6 +458,10 @@ def get_contract_type_for_tx(tx_type_to_grouped_lists_map, tx, tx_receipt): f" tx from block - {tx}, receipt - {tx_receipt}" ) break + + logger.info( + f"index.py | checking returned {contract_type} vs {tx_target_contract_address}" + ) return contract_type @@ -587,6 +611,7 @@ def index_blocks(self, db, blocks_list): PLAYLIST_FACTORY: [], USER_LIBRARY_FACTORY: [], USER_REPLICA_SET_MANAGER: [], + ENTITY_MANAGER: [], } try: """ @@ -652,6 +677,7 @@ def index_blocks(self, db, blocks_list): db, txs_grouped_by_type[USER_FACTORY], txs_grouped_by_type[TRACK_FACTORY], + txs_grouped_by_type[ENTITY_MANAGER], ) logger.info( f"index.py | index_blocks - fetch_ipfs_metadata in {time.time() - fetch_ipfs_metadata_start_time}s" @@ -1108,12 +1134,21 @@ def update_task(self): abi=user_replica_set_manager_abi, ) + entity_manager_contract_abi = update_task.abi_values[ENTITY_MANAGER_CONTRACT_NAME][ + "abi" + ] + entity_manager_contract = update_task.web3.eth.contract( + address=get_contract_addresses()[ENTITY_MANAGER], + abi=entity_manager_contract_abi, + ) + update_task.track_contract = track_contract update_task.user_contract = user_contract update_task.playlist_contract = playlist_contract update_task.social_feature_contract = social_feature_contract update_task.user_library_contract = user_library_contract update_task.user_replica_set_manager_contract = user_replica_set_manager_contract + update_task.entity_manager_contract = entity_manager_contract # Update redis cache for health check queries update_latest_block_redis() diff --git a/discovery-provider/src/tasks/metadata.py b/discovery-provider/src/tasks/metadata.py index b75a29649a2..2470f7225f8 100644 --- a/discovery-provider/src/tasks/metadata.py +++ b/discovery-provider/src/tasks/metadata.py @@ -45,3 +45,13 @@ "events": None, "is_deactivated": None, } + +playlist_metadata_format = { + "playlist_id": None, + "playlist_contents": None, + "playlist_name": None, + "playlist_image_sizes_multihash": None, + "description": None, + "is_album": None, + "is_private": None, +} diff --git a/discovery-provider/src/tasks/playlists.py b/discovery-provider/src/tasks/playlists.py index 52544cd021a..10b7d4fa7fc 100644 --- a/discovery-provider/src/tasks/playlists.py +++ b/discovery-provider/src/tasks/playlists.py @@ -7,6 +7,7 @@ from src.database_task import DatabaseTask from src.models.playlists.playlist import Playlist from src.queries.skipped_transactions import add_node_level_skipped_transaction +from src.tasks.entity_manager.utils import PLAYLIST_ID_OFFSET from src.utils import helpers from src.utils.indexing_errors import EntityMissingRequiredFieldError, IndexingError from src.utils.model_nullable_validator import all_required_fields_present @@ -63,7 +64,7 @@ def playlist_state_update( ) # parse playlist event to add metadata to record - playlist_record = parse_playlist_event( + playlist_record: Playlist = parse_playlist_event( self, update_task, entry, @@ -72,6 +73,11 @@ def playlist_state_update( block_timestamp, session, ) + if playlist_record.playlist_id >= PLAYLIST_ID_OFFSET: + logger.info( + f"index.py | playlists.py | Playlist {playlist_record.playlist_id} is above the playlist ID offset {PLAYLIST_ID_OFFSET}. Skipping transaction." + ) + continue # process playlist record if playlist_record is not None: diff --git a/discovery-provider/src/tasks/tracks.py b/discovery-provider/src/tasks/tracks.py index 4850583df8a..a6a93b70201 100644 --- a/discovery-provider/src/tasks/tracks.py +++ b/discovery-provider/src/tasks/tracks.py @@ -558,7 +558,7 @@ def is_valid_json_field(metadata, field): def populate_track_record_metadata(track_record, track_metadata, handle): track_record.title = track_metadata["title"] - track_record.length = track_metadata["length"] or 0 + track_record.length = track_metadata.get("length", 0) or 0 track_record.cover_art = track_metadata["cover_art"] if track_metadata["cover_art_sizes"]: track_record.cover_art = track_metadata["cover_art_sizes"] diff --git a/discovery-provider/src/utils/cid_metadata_client.py b/discovery-provider/src/utils/cid_metadata_client.py index 03b5da5f9ed..814ffe5dfd7 100644 --- a/discovery-provider/src/utils/cid_metadata_client.py +++ b/discovery-provider/src/utils/cid_metadata_client.py @@ -5,7 +5,11 @@ from urllib.parse import urlparse import aiohttp -from src.tasks.metadata import track_metadata_format, user_metadata_format +from src.tasks.metadata import ( + playlist_metadata_format, + track_metadata_format, + user_metadata_format, +) from src.utils.eth_contracts_helpers import fetch_all_registered_content_nodes logger = logging.getLogger(__name__) @@ -156,12 +160,15 @@ async def _fetch_metadata_from_gateway_endpoints( cid, metadata_json = future_result - # TODO add playlist type - metadata_format = ( - track_metadata_format - if cid_type[cid] == "track" - else user_metadata_format - ) + metadata_format = None + if cid_type[cid] == "track": + metadata_format = track_metadata_format + elif cid_type[cid] == "user": + metadata_format = user_metadata_format + elif cid_type[cid] == "playlist_data": + metadata_format = playlist_metadata_format + else: + raise Exception(f"Unknown metadata type ${cid_type[cid]}") formatted_json = self._get_metadata_from_json( metadata_format, metadata_json diff --git a/discovery-provider/src/utils/constants.py b/discovery-provider/src/utils/constants.py index cb6860de02b..29e55f45c2b 100644 --- a/discovery-provider/src/utils/constants.py +++ b/discovery-provider/src/utils/constants.py @@ -1034,6 +1034,7 @@ class CONTRACT_TYPES(Enum): PLAYLIST_FACTORY = "playlist_factory" USER_LIBRARY_FACTORY = "user_library_factory" USER_REPLICA_SET_MANAGER = "user_replica_set_manager" + ENTITY_MANAGER = "entity_manager" CONTRACT_NAMES_ON_CHAIN = { @@ -1043,4 +1044,5 @@ class CONTRACT_TYPES(Enum): CONTRACT_TYPES.PLAYLIST_FACTORY: "PlaylistFactory", CONTRACT_TYPES.USER_LIBRARY_FACTORY: "UserLibraryFactory", CONTRACT_TYPES.USER_REPLICA_SET_MANAGER: "UserReplicaSetManager", + CONTRACT_TYPES.ENTITY_MANAGER: "EntityManager", } diff --git a/discovery-provider/src/utils/redis_cache.py b/discovery-provider/src/utils/redis_cache.py index 8735c0ba359..bd52b410ae6 100644 --- a/discovery-provider/src/utils/redis_cache.py +++ b/discovery-provider/src/utils/redis_cache.py @@ -144,6 +144,7 @@ def inner_wrap(*args, **kwargs): if cached_resp: if transform is not None: return transform(cached_resp) + return cached_resp, 200 response = func(*args, **kwargs) @@ -152,6 +153,7 @@ def inner_wrap(*args, **kwargs): resp, status_code = response if status_code < 400: set_json_cached_key(redis, key, resp, ttl_sec) + return resp, status_code set_json_cached_key(redis, key, response, ttl_sec) return transform(response) diff --git a/discovery-provider/src/utils/user_event_constants.py b/discovery-provider/src/utils/user_event_constants.py index eeb5026a6ce..5705bc34727 100644 --- a/discovery-provider/src/utils/user_event_constants.py +++ b/discovery-provider/src/utils/user_event_constants.py @@ -35,3 +35,7 @@ user_replica_set_manager_event_types_lookup["update_replica_set"], user_replica_set_manager_event_types_lookup["add_or_update_content_node"], ] + +entity_manager_event_types_lookup = {"manage_entity": "ManageEntity"} + +entity_manager_event_types_arr = [entity_manager_event_types_lookup["manage_entity"]] diff --git a/libs/data-contracts/ABIs/EntityManager.json b/libs/data-contracts/ABIs/EntityManager.json new file mode 100644 index 00000000000..f7398fb21e8 --- /dev/null +++ b/libs/data-contracts/ABIs/EntityManager.json @@ -0,0 +1,174 @@ +{ + "contractName": "EntityManager", + "abi": [ + { + "constant": true, + "inputs": [ + { + "name": "", + "type": "bytes32" + } + ], + "name": "usedSignatures", + "outputs": [ + { + "name": "", + "type": "bool" + } + ], + "payable": false, + "stateMutability": "view", + "type": "function" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": false, + "name": "_userId", + "type": "uint256" + }, + { + "indexed": false, + "name": "_signer", + "type": "address" + }, + { + "indexed": false, + "name": "_entityType", + "type": "string" + }, + { + "indexed": false, + "name": "_entityId", + "type": "uint256" + }, + { + "indexed": false, + "name": "_metadata", + "type": "string" + }, + { + "indexed": false, + "name": "_action", + "type": "string" + } + ], + "name": "ManageEntity", + "type": "event" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": false, + "name": "_userId", + "type": "uint256" + }, + { + "indexed": false, + "name": "_isVerified", + "type": "bool" + } + ], + "name": "ManageIsVerified", + "type": "event" + }, + { + "constant": false, + "inputs": [ + { + "name": "name", + "type": "string" + }, + { + "name": "version", + "type": "string" + }, + { + "name": "chainId", + "type": "uint256" + } + ], + "name": "initialize", + "outputs": [], + "payable": false, + "stateMutability": "nonpayable", + "type": "function" + }, + { + "constant": false, + "inputs": [ + { + "name": "_verifierAddress", + "type": "address" + }, + { + "name": "_networkId", + "type": "uint256" + } + ], + "name": "initialize", + "outputs": [], + "payable": false, + "stateMutability": "nonpayable", + "type": "function" + }, + { + "constant": false, + "inputs": [ + { + "name": "_userId", + "type": "uint256" + }, + { + "name": "_entityType", + "type": "string" + }, + { + "name": "_entityId", + "type": "uint256" + }, + { + "name": "_action", + "type": "string" + }, + { + "name": "_metadata", + "type": "string" + }, + { + "name": "_nonce", + "type": "bytes32" + }, + { + "name": "_subjectSig", + "type": "bytes" + } + ], + "name": "manageEntity", + "outputs": [], + "payable": false, + "stateMutability": "nonpayable", + "type": "function" + }, + { + "constant": false, + "inputs": [ + { + "name": "_userId", + "type": "uint256" + }, + { + "name": "_isVerified", + "type": "bool" + } + ], + "name": "manageIsVerified", + "outputs": [], + "payable": false, + "stateMutability": "nonpayable", + "type": "function" + } + ] +} \ No newline at end of file diff --git a/libs/initScripts/configureLocalDiscProv.js b/libs/initScripts/configureLocalDiscProv.js index 353a3fb1fc7..d79dffa6fcd 100644 --- a/libs/initScripts/configureLocalDiscProv.js +++ b/libs/initScripts/configureLocalDiscProv.js @@ -2,9 +2,12 @@ const fs = require('fs') const path = require('path') const readline = require('readline') const ethContractsMigrationOutput = require('../../eth-contracts/migrations/migration-output.json') +const dataContractsMigrationOutput = require('../../contracts/migrations/migration-output.json') const solanaConfig = require('../../solana-programs/solana-program-config.json') const ETH_CONTRACTS_REGISTRY = 'audius_eth_contracts_registry' +const CONTRACTS_REGISTRY = 'audius_contracts_registry' +const ENTITY_MANAGER_ADDRESS = 'audius_contracts_entity_manager_address' const SOLANA_TRACK_LISTEN_COUNT_ADDRESS = 'audius_solana_track_listen_count_address' const SOLANA_ENDPOINT = 'audius_solana_endpoint' @@ -23,6 +26,8 @@ const SOLANA_ANCHOR_ADMIN_STORAGE_PUBLIC_KEY = 'audius_solana_anchor_admin_stora // Updates audius_eth_contracts_registry in discovery provider const configureLocalDiscProv = async () => { const ethRegistryAddress = ethContractsMigrationOutput.registryAddress + const dataRegistryAddress = dataContractsMigrationOutput.registryAddress + const entityManagerAddress = dataContractsMigrationOutput.entityManagerProxyAddress const solanaTrackListenCountAddress = solanaConfig.trackListenCountAddress const signerGroup = solanaConfig.signerGroup const solanaEndpoint = solanaConfig.endpoint @@ -48,6 +53,8 @@ const configureLocalDiscProv = async () => { rewardsManagerAccount, anchorProgramId, anchorAdminStoragePublicKey, + dataRegistryAddress, + entityManagerAddress ) } @@ -65,6 +72,8 @@ const _updateDiscoveryProviderEnvFile = async ( rewardsManagerAccount, anchorProgramId, anchorAdminStoragePublicKey, + dataRegistryAddress, + entityManagerAddress ) => { const fileStream = fs.createReadStream(readPath) const rl = readline.createInterface({ @@ -83,6 +92,8 @@ const _updateDiscoveryProviderEnvFile = async ( let rewardsAccountFound = false let anchorProgramIdFound = false let anchorAdminStoragePublicKeyFound = false + let dataRegistryLineFound = false + let dataContractLineFound = false const ethRegistryAddressLine = `${ETH_CONTRACTS_REGISTRY}=${ethRegistryAddress}` const solanaTrackListenCountAddressLine = `${SOLANA_TRACK_LISTEN_COUNT_ADDRESS}=${solanaTrackListenCountAddress}` @@ -94,6 +105,8 @@ const _updateDiscoveryProviderEnvFile = async ( const rewardsManagerAccountLine = `${SOLANA_REWARDS_MANAGER_ACCOUNT}=${rewardsManagerAccount}` const anchorProgramIdLine = `${SOLANA_ANCHOR_PROGRAM_ID}=${anchorProgramId}` const anchorAdminStoragePublicKeyLine = `${SOLANA_ANCHOR_ADMIN_STORAGE_PUBLIC_KEY}=${anchorAdminStoragePublicKey}` + const dataRegistryLine = `${CONTRACTS_REGISTRY}=${dataRegistryAddress}` + const entityManagerContractLine = `${ENTITY_MANAGER_ADDRESS}=${entityManagerAddress}` for await (const line of rl) { if (line.includes(ETH_CONTRACTS_REGISTRY)) { @@ -126,6 +139,12 @@ const _updateDiscoveryProviderEnvFile = async ( } else if (line.includes(SOLANA_ANCHOR_ADMIN_STORAGE_PUBLIC_KEY)) { output.push(anchorAdminStoragePublicKeyLine) anchorAdminStoragePublicKeyFound = true + } else if (line.includes(CONTRACTS_REGISTRY)) { + output.push(dataRegistryLine) + dataRegistryLineFound = true + } else if (line.includes(ENTITY_MANAGER_ADDRESS)) { + output.push(entityManagerContractLine) + dataContractLineFound = true } else { output.push(line) } @@ -160,6 +179,12 @@ const _updateDiscoveryProviderEnvFile = async ( if (!anchorAdminStoragePublicKeyFound) { output.push(anchorAdminStoragePublicKeyLine) } + if (!dataRegistryLineFound) { + output.push(dataRegistryLine) + } + if (!dataContractLineFound) { + output.push(entityManagerContractLine) + } fs.writeFileSync(writePath, output.join('\n')) console.log(`Updated DISCOVERY PROVIDER ${writePath} ${ETH_CONTRACTS_REGISTRY}=${ethRegistryAddress} ${output}`) } diff --git a/libs/src/AudiusLibs.ts b/libs/src/AudiusLibs.ts index aa54b9dd01a..6d55e232268 100644 --- a/libs/src/AudiusLibs.ts +++ b/libs/src/AudiusLibs.ts @@ -45,6 +45,7 @@ import { Keypair, PublicKey } from '@solana/web3.js' import { getPlatformLocalStorage, LocalStorage } from './utils/localStorage' import type { BaseConstructorArgs } from './api/base' import type { MonitoringCallbacks } from './services/types' +import { EntityManager } from './api/entityManager' type LibsIdentityServiceConfig = { url: string @@ -145,7 +146,9 @@ export class AudiusLibs { // network chain id networkId: string, // wallet address to force use instead of the first wallet on the provided web3 - walletOverride: Nullable = null + walletOverride: Nullable = null, + // entity manager address + entityManagerAddress: Nullable = null ) { const web3Instance = await Utils.configureWeb3(web3Provider, networkId) if (!web3Instance) { @@ -154,6 +157,7 @@ export class AudiusLibs { const wallets = await web3Instance.eth.getAccounts() return { registryAddress, + entityManagerAddress, useExternalWeb3: true, externalWeb3Config: { web3: web3Instance, @@ -168,7 +172,8 @@ export class AudiusLibs { static configInternalWeb3( registryAddress: string, providers: provider, - privateKey: string + privateKey: string, + entityManagerAddress?: string ) { let providerList if (typeof providers === 'string') { @@ -185,6 +190,7 @@ export class AudiusLibs { return { registryAddress, + entityManagerAddress, useExternalWeb3: false, internalWeb3Config: { web3ProviderEndpoints: providerList, @@ -197,14 +203,12 @@ export class AudiusLibs { * Configures an eth web3 */ static configEthWeb3( - tokenAddress: string, - registryAddress: string, - // web3 provider endpoint(s) - providers: provider, - // owner wallet to establish who we are sending transactions on behalf of - ownerWallet?: string, - claimDistributionContractAddress?: string, - wormholeContractAddress?: string + tokenAddress, + registryAddress, + providers, + ownerWallet, + claimDistributionContractAddress, + wormholeContractAddress ) { let providerList if (typeof providers === 'string') { @@ -364,6 +368,7 @@ export class AudiusLibs { File: Nullable Rewards: Nullable Reactions: Nullable + EntityManager: Nullable preferHigherPatchForPrimary: boolean preferHigherPatchForSecondaries: boolean @@ -445,6 +450,7 @@ export class AudiusLibs { this.File = null this.Rewards = null this.Reactions = null + this.EntityManager = null this.preferHigherPatchForPrimary = preferHigherPatchForPrimary this.preferHigherPatchForSecondaries = preferHigherPatchForSecondaries @@ -550,7 +556,8 @@ export class AudiusLibs { if (this.web3Manager) { this.contracts = new AudiusContracts( this.web3Manager, - this.web3Config.registryAddress, + this.web3Config ? this.web3Config.registryAddress : null, + this.web3Config ? this.web3Config.entityManagerAddress : null, this.isServer, this.logger ) @@ -652,6 +659,7 @@ export class AudiusLibs { this.File = new File(this.User, ...services) this.Rewards = new Rewards(this.ServiceProvider, ...services) this.Reactions = new Reactions(...services) + this.EntityManager = new EntityManager(...services) } } diff --git a/libs/src/api/Track.ts b/libs/src/api/Track.ts index 4b4c6a58d05..35233284298 100644 --- a/libs/src/api/Track.ts +++ b/libs/src/api/Track.ts @@ -408,7 +408,6 @@ export class Track extends Base { } } ) - phase = phases.ADDING_TRACK // Write metadata to chain diff --git a/libs/src/api/entityManager.ts b/libs/src/api/entityManager.ts new file mode 100644 index 00000000000..c271b990b4c --- /dev/null +++ b/libs/src/api/entityManager.ts @@ -0,0 +1,553 @@ +import { Base, Services } from './base' +import type { PlaylistMetadata } from '../services/creatorNode' +import { Nullable, Utils } from '../utils' + +export enum Action { + CREATE = 'Create', + UPDATE = 'Update', + DELETE = 'Delete' +} + +export enum EntityType { + PLAYLIST = 'Playlist' +} + +export interface PlaylistOperationResponse { + /** + * Blockhash of playlist transaction + */ + blockHash: Nullable + /** + * Block number of playlist transaction + */ + blockNumber: Nullable + /** + * ID of playlist being modified + */ + playlistId: Nullable + /** + * String error message returned + */ + error: Nullable +} + +const { encodeHashId, decodeHashId } = Utils + +/* + API surface for updated data contract interactions. + Provides simplified entity management in a generic fashion + Handles metadata + file upload etc. for entities such as Playlist/Track/User +*/ +export class EntityManager extends Base { + /** + * Generate random integer between two known values + */ + getRandomInt(min: number, max: number): number { + min = Math.ceil(min) + max = Math.floor(max) + return Math.floor(Math.random() * (max - min) + min) + } + + async getFullPlaylist(playlistId: number, userId: number) { + const encodedPlaylistId = encodeHashId(playlistId) as string + const encodedUserId = encodeHashId(userId) as string + + const playlist: any = ( + await this.discoveryProvider.getFullPlaylist( + encodedPlaylistId, + encodedUserId + ) + )[0] + return playlist + } + + mapAddedTimestamps(addedTimestamps: any) { + const trackIds = addedTimestamps.map( + (trackObj: { + track_id: string + metadata_timestamp?: number + timestamp: number + }) => ({ + track: decodeHashId(trackObj.track_id), + time: trackObj.metadata_timestamp ?? trackObj.timestamp + }) + ) + + return trackIds + } + + /** + * Playlist default response values + */ + getDefaultPlaylistReponseValues(): PlaylistOperationResponse { + return { + blockHash: null, + blockNumber: null, + playlistId: null, + error: null + } + } + + /** + * Create a playlist using updated data contracts flow + */ + async createPlaylist({ + playlistId, + playlistName, + trackIds, + description, + isAlbum, + isPrivate, + coverArt, + logger = console + }: { + playlistId: number + playlistName: string + trackIds: number[] + description: string + isAlbum: boolean + isPrivate: boolean + coverArt: string + logger: Console + }): Promise { + const responseValues: PlaylistOperationResponse = + this.getDefaultPlaylistReponseValues() + try { + const currentUserId: string | null = + this.userStateManager.getCurrentUserId() + if (!currentUserId) { + responseValues.error = 'Missing current user ID' + return responseValues + } + const userId: number = parseInt(currentUserId) + const createAction = Action.CREATE + const entityType = EntityType.PLAYLIST + this.REQUIRES(Services.CREATOR_NODE) + const updatedPlaylistImage = await this.creatorNode.uploadImage( + coverArt, + true // square + ) + const web3 = this.web3Manager.getWeb3() + const currentBlockNumber = await web3.eth.getBlockNumber() + const currentBlock = await web3.eth.getBlock(currentBlockNumber) + const tracks = trackIds.map((trackId) => ({ + track: trackId, + time: currentBlock.timestamp as number + })) + const dirCID = updatedPlaylistImage.dirCID + const metadata: PlaylistMetadata = { + playlist_id: playlistId, + playlist_contents: { track_ids: tracks }, + playlist_name: playlistName, + playlist_image_sizes_multihash: dirCID, + description, + is_album: isAlbum, + is_private: isPrivate + } + const { metadataMultihash } = + await this.creatorNode.uploadPlaylistMetadata(metadata) + const manageEntityResponse = await this.manageEntity({ + userId: userId, + entityType, + entityId: playlistId, + action: createAction, + metadataMultihash + }) + const txReceipt = manageEntityResponse.txReceipt + responseValues.blockHash = txReceipt.blockHash + responseValues.blockNumber = txReceipt.blockNumber + responseValues.playlistId = playlistId + return responseValues + } catch (e) { + const error = (e as Error).message + responseValues.error = error + return responseValues + } + } + + /** + * Delete a playlist using updated data contracts flow + */ + async deletePlaylist({ + playlistId, + logger = console + }: { + playlistId: number + logger: any + }): Promise<{ blockHash: any; blockNumber: any }> { + const responseValues: PlaylistOperationResponse = + this.getDefaultPlaylistReponseValues() + const currentUserId: string | null = + this.userStateManager.getCurrentUserId() + if (!currentUserId) { + responseValues.error = 'Missing current user ID' + return responseValues + } + const userId: number = parseInt(currentUserId) + try { + const resp = await this.manageEntity({ + userId, + entityType: EntityType.PLAYLIST, + entityId: playlistId, + action: Action.DELETE, + metadataMultihash: '' + }) + const txReceipt = resp.txReceipt + responseValues.blockHash = txReceipt.blockHash + responseValues.blockNumber = txReceipt.blockNumber + responseValues.playlistId = playlistId + return responseValues + } catch (e) { + const error = (e as Error).message + responseValues.error = error + return responseValues + } + } + + /** + * Update a playlist using updated data contracts flow + **/ + async editPlaylist({ + playlistId, + playlistName, + description, + isAlbum, + isPrivate, + coverArt, + logger = console + }: { + playlistId: number + playlistName: Nullable + description: Nullable + isAlbum: Nullable + isPrivate: Nullable + coverArt: Nullable + logger: Console + }): Promise { + const responseValues: PlaylistOperationResponse = + this.getDefaultPlaylistReponseValues() + + try { + const currentUserId: string | null = + this.userStateManager.getCurrentUserId() + if (!playlistId || playlistId === undefined) { + responseValues.error = 'Missing current playlistId' + return responseValues + } + if (!currentUserId) { + responseValues.error = 'Missing current user ID' + return responseValues + } + const userId: number = parseInt(currentUserId) + const updateAction = Action.UPDATE + const entityType = EntityType.PLAYLIST + this.REQUIRES(Services.CREATOR_NODE) + let dirCID + if (coverArt) { + // @ts-expect-error + const updatedPlaylistImage = await this.creatorNode.uploadImage( + coverArt, + true // square + ) + dirCID = updatedPlaylistImage.dirCID + } + const playlist = await this.getFullPlaylist(playlistId, userId) + const existingPlaylistTracks = this.mapAddedTimestamps( + playlist.added_timestamps + ) + const metadata: PlaylistMetadata = { + playlist_id: playlistId, + playlist_contents: { track_ids: existingPlaylistTracks }, + playlist_name: playlistName ?? playlist.playlist_name, + playlist_image_sizes_multihash: dirCID ?? playlist.cover_art, + description: description ?? playlist.description, + is_album: isAlbum ?? playlist.is_album, + is_private: isPrivate ?? playlist.is_private + } + const { metadataMultihash } = + await this.creatorNode.uploadPlaylistMetadata(metadata) + const resp = await this.manageEntity({ + userId, + entityType, + entityId: playlistId, + action: updateAction, + metadataMultihash + }) + const txReceipt = resp.txReceipt + responseValues.blockHash = txReceipt.blockHash + responseValues.blockNumber = txReceipt.blockNumber + responseValues.playlistId = playlistId + return responseValues + } catch (e) { + const error = (e as Error).message + responseValues.error = error + return responseValues + } + } + + async addPlaylistTrack({ + playlistId, + trackId, + timestamp, + logger = console + }: { + playlistId: number + trackId: number + timestamp: number + logger: Console + }): Promise { + const responseValues: PlaylistOperationResponse = + this.getDefaultPlaylistReponseValues() + + try { + const currentUserId: string | null = + this.userStateManager.getCurrentUserId() + if (!playlistId || playlistId === undefined) { + responseValues.error = 'Missing current playlistId' + return responseValues + } + if (!currentUserId) { + responseValues.error = 'Missing current user ID' + return responseValues + } + const userId: number = parseInt(currentUserId) + const updateAction = Action.UPDATE + const entityType = EntityType.PLAYLIST + this.REQUIRES(Services.CREATOR_NODE) + + const playlist = await this.getFullPlaylist(playlistId, userId) + + const updatedPlaylistTracks = this.mapAddedTimestamps( + playlist.added_timestamps + ) + + updatedPlaylistTracks.push({ + track: trackId, + time: timestamp + }) + + const metadata: PlaylistMetadata = { + playlist_id: playlistId, + playlist_contents: { track_ids: updatedPlaylistTracks }, + playlist_name: playlist.playlist_name, + playlist_image_sizes_multihash: playlist.cover_art, + description: playlist.description, + is_album: playlist.is_album, + is_private: playlist.is_private + } + const { metadataMultihash } = + await this.creatorNode.uploadPlaylistMetadata(metadata) + const resp = await this.manageEntity({ + userId, + entityType, + entityId: playlistId, + action: updateAction, + metadataMultihash + }) + const txReceipt = resp.txReceipt + responseValues.blockHash = txReceipt.blockHash + responseValues.blockNumber = txReceipt.blockNumber + responseValues.playlistId = playlistId + return responseValues + } catch (e) { + const error = (e as Error).message + responseValues.error = error + return responseValues + } + } + + async deletePlaylistTrack({ + playlistId, + trackId, + timestamp, + logger = console + }: { + playlistId: number + trackId: number + timestamp: number + logger: Console + }): Promise { + const responseValues: PlaylistOperationResponse = + this.getDefaultPlaylistReponseValues() + + try { + const currentUserId: string | null = + this.userStateManager.getCurrentUserId() + if (!playlistId || playlistId === undefined) { + responseValues.error = 'Missing current playlistId' + return responseValues + } + if (!currentUserId) { + responseValues.error = 'Missing current user ID' + return responseValues + } + const userId: number = parseInt(currentUserId) + const updateAction = Action.UPDATE + const entityType = EntityType.PLAYLIST + this.REQUIRES(Services.CREATOR_NODE) + const playlist = await this.getFullPlaylist(playlistId, userId) + + const existingPlaylistTracks = this.mapAddedTimestamps( + playlist.added_timestamps + ) + + const updatedTrackIds = existingPlaylistTracks.filter( + (trackObj: { track: number; metadata_time?: number; time: number }) => + (trackObj.track !== trackId && + timestamp !== trackObj.metadata_time) ?? + trackObj.time + ) + + const metadata: PlaylistMetadata = { + playlist_id: playlistId, + playlist_contents: { track_ids: updatedTrackIds }, + playlist_name: playlist.playlist_name, + playlist_image_sizes_multihash: playlist.cover_art, + description: playlist.description, + is_album: playlist.is_album, + is_private: playlist.is_private + } + const { metadataMultihash } = + await this.creatorNode.uploadPlaylistMetadata(metadata) + const resp = await this.manageEntity({ + userId, + entityType, + entityId: playlistId, + action: updateAction, + metadataMultihash + }) + const txReceipt = resp.txReceipt + responseValues.blockHash = txReceipt.blockHash + responseValues.blockNumber = txReceipt.blockNumber + responseValues.playlistId = playlistId + return responseValues + } catch (e) { + const error = (e as Error).message + responseValues.error = error + return responseValues + } + } + + /** + * Update a playlist using updated data contracts flow + **/ + async orderPlaylist({ + playlistId, + trackIds, + logger = console + }: { + playlistId: number + trackIds: number[] + logger: Console + }): Promise { + const responseValues: PlaylistOperationResponse = + this.getDefaultPlaylistReponseValues() + + try { + const currentUserId: string | null = + this.userStateManager.getCurrentUserId() + if (!playlistId || playlistId === undefined) { + responseValues.error = 'Missing current playlistId' + return responseValues + } + if (!currentUserId) { + responseValues.error = 'Missing current user ID' + return responseValues + } + const userId: number = parseInt(currentUserId) + const updateAction = Action.UPDATE + const entityType = EntityType.PLAYLIST + this.REQUIRES(Services.CREATOR_NODE) + const playlist = await this.getFullPlaylist(playlistId, userId) + + const existingPlaylistTracks = this.mapAddedTimestamps( + playlist.added_timestamps + ) + + let trackIdsWithTimes = [] + const trackIdTimes = {} + existingPlaylistTracks.forEach( + (trackObj: { track: number; metadata_time?: number; time: number }) => { + const trackId = trackObj.track + const timestamp = trackObj.metadata_time ?? trackObj.time + if (trackId in trackIdTimes) { + trackIdTimes[trackId].push(timestamp) + } else { + trackIdTimes[trackId] = [timestamp] + } + } + ) + + // new tracks default to currentBlock timestamp + trackIdsWithTimes = trackIds.map((trackId: number) => ({ + track: trackId, + time: trackIdTimes[trackId].pop() + })) + const metadata: PlaylistMetadata = { + playlist_id: playlistId, + playlist_contents: { track_ids: trackIdsWithTimes }, + playlist_name: playlist.playlist_name, + playlist_image_sizes_multihash: playlist.cover_art, + description: playlist.description, + is_album: playlist.is_album, + is_private: playlist.is_private + } + const { metadataMultihash } = + await this.creatorNode.uploadPlaylistMetadata(metadata) + + const resp = await this.manageEntity({ + userId, + entityType, + entityId: playlistId, + action: updateAction, + metadataMultihash + }) + const txReceipt = resp.txReceipt + responseValues.blockHash = txReceipt.blockHash + responseValues.blockNumber = txReceipt.blockNumber + responseValues.playlistId = playlistId + return responseValues + } catch (e) { + const error = (e as Error).message + responseValues.error = error + return responseValues + } + } + + /** + * Manage an entity with the updated data contract flow + * Leveraged to manipulate User/Track/Playlist/+ other entities + */ + async manageEntity({ + userId, + entityType, + entityId, + action, + metadataMultihash + }: { + userId: number + entityType: EntityType + entityId: number + action: Action + metadataMultihash: string + }): Promise< + { txReceipt: any; error: null } | { txReceipt: null; error: string } + > { + let error = null + let resp: any + try { + resp = await this.contracts.EntityManagerClient?.manageEntity( + userId, + entityType, + entityId, + action, + metadataMultihash + ) + + return { txReceipt: resp.txReceipt, error } + } catch (e) { + error = (e as Error).message + return { txReceipt: null, error } + } + } +} diff --git a/libs/src/data-contracts/ABIs/AudiusData.json b/libs/src/data-contracts/ABIs/AudiusData.json new file mode 100644 index 00000000000..02f5b9651d1 --- /dev/null +++ b/libs/src/data-contracts/ABIs/AudiusData.json @@ -0,0 +1,174 @@ +{ + "contractName": "AudiusData", + "abi": [ + { + "constant": true, + "inputs": [ + { + "name": "", + "type": "bytes32" + } + ], + "name": "usedSignatures", + "outputs": [ + { + "name": "", + "type": "bool" + } + ], + "payable": false, + "stateMutability": "view", + "type": "function" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": false, + "name": "_userId", + "type": "uint256" + }, + { + "indexed": false, + "name": "_signer", + "type": "address" + }, + { + "indexed": false, + "name": "_entityType", + "type": "string" + }, + { + "indexed": false, + "name": "_entityId", + "type": "uint256" + }, + { + "indexed": false, + "name": "_metadata", + "type": "string" + }, + { + "indexed": false, + "name": "_action", + "type": "string" + } + ], + "name": "ManageEntity", + "type": "event" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": false, + "name": "_userId", + "type": "uint256" + }, + { + "indexed": false, + "name": "_isVerified", + "type": "bool" + } + ], + "name": "ManageIsVerified", + "type": "event" + }, + { + "constant": false, + "inputs": [ + { + "name": "name", + "type": "string" + }, + { + "name": "version", + "type": "string" + }, + { + "name": "chainId", + "type": "uint256" + } + ], + "name": "initialize", + "outputs": [], + "payable": false, + "stateMutability": "nonpayable", + "type": "function" + }, + { + "constant": false, + "inputs": [ + { + "name": "_verifierAddress", + "type": "address" + }, + { + "name": "_networkId", + "type": "uint256" + } + ], + "name": "initialize", + "outputs": [], + "payable": false, + "stateMutability": "nonpayable", + "type": "function" + }, + { + "constant": false, + "inputs": [ + { + "name": "_userId", + "type": "uint256" + }, + { + "name": "_entityType", + "type": "string" + }, + { + "name": "_entityId", + "type": "uint256" + }, + { + "name": "_action", + "type": "string" + }, + { + "name": "_metadata", + "type": "string" + }, + { + "name": "_nonce", + "type": "bytes32" + }, + { + "name": "_subjectSig", + "type": "bytes" + } + ], + "name": "manageEntity", + "outputs": [], + "payable": false, + "stateMutability": "nonpayable", + "type": "function" + }, + { + "constant": false, + "inputs": [ + { + "name": "_userId", + "type": "uint256" + }, + { + "name": "_isVerified", + "type": "bool" + } + ], + "name": "manageIsVerified", + "outputs": [], + "payable": false, + "stateMutability": "nonpayable", + "type": "function" + } + ] +} \ No newline at end of file diff --git a/libs/src/data-contracts/ABIs/EntityManager.json b/libs/src/data-contracts/ABIs/EntityManager.json new file mode 100644 index 00000000000..f7398fb21e8 --- /dev/null +++ b/libs/src/data-contracts/ABIs/EntityManager.json @@ -0,0 +1,174 @@ +{ + "contractName": "EntityManager", + "abi": [ + { + "constant": true, + "inputs": [ + { + "name": "", + "type": "bytes32" + } + ], + "name": "usedSignatures", + "outputs": [ + { + "name": "", + "type": "bool" + } + ], + "payable": false, + "stateMutability": "view", + "type": "function" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": false, + "name": "_userId", + "type": "uint256" + }, + { + "indexed": false, + "name": "_signer", + "type": "address" + }, + { + "indexed": false, + "name": "_entityType", + "type": "string" + }, + { + "indexed": false, + "name": "_entityId", + "type": "uint256" + }, + { + "indexed": false, + "name": "_metadata", + "type": "string" + }, + { + "indexed": false, + "name": "_action", + "type": "string" + } + ], + "name": "ManageEntity", + "type": "event" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": false, + "name": "_userId", + "type": "uint256" + }, + { + "indexed": false, + "name": "_isVerified", + "type": "bool" + } + ], + "name": "ManageIsVerified", + "type": "event" + }, + { + "constant": false, + "inputs": [ + { + "name": "name", + "type": "string" + }, + { + "name": "version", + "type": "string" + }, + { + "name": "chainId", + "type": "uint256" + } + ], + "name": "initialize", + "outputs": [], + "payable": false, + "stateMutability": "nonpayable", + "type": "function" + }, + { + "constant": false, + "inputs": [ + { + "name": "_verifierAddress", + "type": "address" + }, + { + "name": "_networkId", + "type": "uint256" + } + ], + "name": "initialize", + "outputs": [], + "payable": false, + "stateMutability": "nonpayable", + "type": "function" + }, + { + "constant": false, + "inputs": [ + { + "name": "_userId", + "type": "uint256" + }, + { + "name": "_entityType", + "type": "string" + }, + { + "name": "_entityId", + "type": "uint256" + }, + { + "name": "_action", + "type": "string" + }, + { + "name": "_metadata", + "type": "string" + }, + { + "name": "_nonce", + "type": "bytes32" + }, + { + "name": "_subjectSig", + "type": "bytes" + } + ], + "name": "manageEntity", + "outputs": [], + "payable": false, + "stateMutability": "nonpayable", + "type": "function" + }, + { + "constant": false, + "inputs": [ + { + "name": "_userId", + "type": "uint256" + }, + { + "name": "_isVerified", + "type": "bool" + } + ], + "name": "manageIsVerified", + "outputs": [], + "payable": false, + "stateMutability": "nonpayable", + "type": "function" + } + ] +} \ No newline at end of file diff --git a/libs/src/data-contracts/signatureSchemas.ts b/libs/src/data-contracts/signatureSchemas.ts index a832d455c75..921929523ac 100644 --- a/libs/src/data-contracts/signatureSchemas.ts +++ b/libs/src/data-contracts/signatureSchemas.ts @@ -64,6 +64,10 @@ const getUserReplicaSetManagerDomain: DomainFn = (chainId, contractAddress) => { ) } +const getEntityManagerDomain: DomainFn = (chainId, contractAddress) => { + return getDomainData('Entity Manager', '1', chainId, contractAddress) +} + export const domains = { getSocialFeatureFactoryDomain, getUserFactoryDomain, @@ -71,7 +75,8 @@ export const domains = { getPlaylistFactoryDomain, getUserLibraryFactoryDomain, getIPLDBlacklistFactoryDomain, - getUserReplicaSetManagerDomain + getUserReplicaSetManagerDomain, + getEntityManagerDomain } /* contract signing domain */ @@ -258,6 +263,15 @@ const updateReplicaSet = [ { name: 'nonce', type: 'bytes32' } ] +const manageEntity = [ + { name: 'userId', type: 'uint' }, + { name: 'entityType', type: 'string' }, + { name: 'entityId', type: 'uint' }, + { name: 'action', type: 'string' }, + { name: 'metadata', type: 'string' }, + { name: 'nonce', type: 'bytes32' } +] + export const schemas = { domain, addUserRequest, @@ -289,7 +303,8 @@ export const schemas = { deletePlaylistSaveRequest, addIPLDBlacklist, proposeAddOrUpdateContentNode, - updateReplicaSet + updateReplicaSet, + manageEntity } type MessageSchema = readonly EIP712TypeProperty[] @@ -1145,6 +1160,34 @@ const getUpdateReplicaSetRequestData = ( ) } +const getManageEntityData = ( + chainId: number, + contractAddress: string, + userId: number, + entityType: string, + entityId: number, + action: string, + metadata: string, + nonce: string +) => { + const message = { + userId, + entityType, + entityId, + action, + metadata, + nonce + } + return getRequestData( + domains.getEntityManagerDomain, + chainId, + contractAddress, + 'ManageEntity', + schemas.manageEntity, + message + ) +} + export const generators = { getUpdateUserMultihashRequestData, getAddUserRequestData, @@ -1181,7 +1224,8 @@ export const generators = { getUpdatePlaylistDescriptionRequestData, addIPLDToBlacklistRequestData, getProposeAddOrUpdateContentNodeRequestData, - getUpdateReplicaSetRequestData + getUpdateReplicaSetRequestData, + getManageEntityData } type NodeCrypto = { randomBytes: (size: number) => Buffer } diff --git a/libs/src/eth-contracts/ABIs/Wormhole.json b/libs/src/eth-contracts/ABIs/Wormhole.json index 830697357d2..7dc4a28bee0 100644 --- a/libs/src/eth-contracts/ABIs/Wormhole.json +++ b/libs/src/eth-contracts/ABIs/Wormhole.json @@ -1,12 +1,71 @@ { "contractName": "Wormhole", "abi": [ + { + "constant": true, + "inputs": [], + "name": "DOMAIN_SEPARATOR", + "outputs": [ + { + "internalType": "bytes32", + "name": "", + "type": "bytes32" + } + ], + "payable": false, + "stateMutability": "view", + "type": "function" + }, + { + "constant": true, + "inputs": [], + "name": "LOCK_ASSETS_TYPEHASH", + "outputs": [ + { + "internalType": "bytes32", + "name": "", + "type": "bytes32" + } + ], + "payable": false, + "stateMutability": "view", + "type": "function" + }, + { + "constant": false, + "inputs": [ + { + "internalType": "address", + "name": "_tokenAddress", + "type": "address" + }, + { + "internalType": "address", + "name": "_wormholeAddress", + "type": "address" + } + ], + "name": "initialize", + "outputs": [], + "payable": false, + "stateMutability": "nonpayable", + "type": "function" + }, + { + "constant": false, + "inputs": [], + "name": "initialize", + "outputs": [], + "payable": false, + "stateMutability": "nonpayable", + "type": "function" + }, { "constant": false, "inputs": [ { "internalType": "address", - "name": "token", + "name": "from", "type": "address" }, { @@ -14,32 +73,83 @@ "name": "amount", "type": "uint256" }, - { - "internalType": "uint16", - "name": "recipientChain", - "type": "uint16" - }, { "internalType": "bytes32", "name": "recipient", "type": "bytes32" }, + { + "internalType": "uint8", + "name": "targetChain", + "type": "uint8" + }, + { + "internalType": "bool", + "name": "refundDust", + "type": "bool" + }, { "internalType": "uint256", - "name": "arbiterFee", + "name": "deadline", "type": "uint256" }, { - "internalType": "uint32", - "name": "nonce", - "type": "uint32" + "internalType": "uint8", + "name": "v", + "type": "uint8" + }, + { + "internalType": "bytes32", + "name": "r", + "type": "bytes32" + }, + { + "internalType": "bytes32", + "name": "s", + "type": "bytes32" } ], - "name": "transferTokens", + "name": "lockAssets", "outputs": [], "payable": false, "stateMutability": "nonpayable", "type": "function" + }, + { + "constant": true, + "inputs": [ + { + "internalType": "address", + "name": "", + "type": "address" + } + ], + "name": "nonces", + "outputs": [ + { + "internalType": "uint32", + "name": "", + "type": "uint32" + } + ], + "payable": false, + "stateMutability": "view", + "type": "function" + }, + { + "constant": true, + "inputs": [], + "name": "token", + "outputs": [ + { + "internalType": "address", + "name": "", + "type": "address" + } + ], + "payable": false, + "stateMutability": "view", + "type": "function" } ] } \ No newline at end of file diff --git a/libs/src/services/ABIDecoder/AudiusABIDecoder.ts b/libs/src/services/ABIDecoder/AudiusABIDecoder.ts index f57a8120d21..c3f97baec3c 100644 --- a/libs/src/services/ABIDecoder/AudiusABIDecoder.ts +++ b/libs/src/services/ABIDecoder/AudiusABIDecoder.ts @@ -10,6 +10,7 @@ import SocialFeatureFactoryABI from '../../data-contracts/ABIs/SocialFeatureFact import PlaylistFactoryABI from '../../data-contracts/ABIs/PlaylistFactory.json' import UserLibraryFactoryABI from '../../data-contracts/ABIs/UserLibraryFactory.json' import UserReplicaSetManagerABI from '../../data-contracts/ABIs/UserReplicaSetManager.json' +import EntityManagerABI from '../../data-contracts/ABIs/EntityManager.json' const abiMap: Record = {} @@ -21,7 +22,8 @@ const abiMap: Record = {} SocialFeatureFactoryABI, PlaylistFactoryABI, UserLibraryFactoryABI, - UserReplicaSetManagerABI + UserReplicaSetManagerABI, + EntityManagerABI ].forEach(({ contractName, abi }) => { abiDecoder.addABI(abi as AbiItem[]) abiMap[contractName] = abi as AbiItem[] @@ -39,7 +41,7 @@ export class AudiusABIDecoder { // namespace of functions) const abi = abiMap[contractName] if (!abi) { - throw new Error('Unrecognized contract name') + throw new Error(`Unrecognized contract name ${contractName}`) } let foundFunction: AbiItem | undefined diff --git a/libs/src/services/creatorNode/CreatorNode.ts b/libs/src/services/creatorNode/CreatorNode.ts index b9408b75b42..776eed901c6 100644 --- a/libs/src/services/creatorNode/CreatorNode.ts +++ b/libs/src/services/creatorNode/CreatorNode.ts @@ -5,6 +5,7 @@ import { Nullable, TrackMetadata, Utils, uuid } from '../../utils' import { userSchemaType, trackSchemaType, + playlistSchemaType, Schemas } from '../schemaValidator/SchemaValidator' import type { Web3Manager } from '../web3Manager' @@ -17,6 +18,22 @@ const MAX_TRACK_TRANSCODE_TIMEOUT = 3600000 // 1 hour const POLL_STATUS_INTERVAL = 3000 // 3s const BROWSER_SESSION_REFRESH_TIMEOUT = 604800000 // 1 week +type PlaylistTrackId = { time: number; track: number } + +type PlaylistContents = { + track_ids: PlaylistTrackId[] +} + +export type PlaylistMetadata = { + playlist_contents: PlaylistContents + playlist_id: number + playlist_name: string + playlist_image_sizes_multihash: string + description: string + is_album: boolean + is_private: boolean +} + type ProgressCB = (loaded: number, total: number) => void type ClockValueRequestConfig = { @@ -404,6 +421,58 @@ export class CreatorNode { return body } + /** + * Uploads playlist metadata to a creator node + * source file must be provided (returned from uploading track content). + * @param metadata + */ + async uploadPlaylistMetadata(metadata: PlaylistMetadata) { + // Validate object before sending + try { + this.schemas[playlistSchemaType].validate?.(metadata) + } catch (e) { + console.error('Error validating playlist metadata', e) + } + + const { data: body } = await this._makeRequest( + { + url: '/playlists/metadata', + method: 'post', + data: { + metadata + } + }, + true + ) + return body + } + + /** + * Associate an uploaded playlist metadata file with a blockchainId + * @param blockchainId - Valid ID assigned to playlist + * @param metadataFileUUID unique ID for metadata playlist + * @param blockNumber + */ + async associatePlaylistMetadata( + blockchainId: number, + metadataFileUUID: string, + blockNumber: number + ) { + const { data: body } = await this._makeRequest( + { + url: '/playlists', + method: 'post', + data: { + blockchainId, + metadataFileUUID, + blockNumber + } + }, + true + ) + return body + } + /** * Creates a track on the content node, associating track id with file content * @param audiusTrackId returned by track creation on-blockchain diff --git a/libs/src/services/dataContracts/AudiusContracts.ts b/libs/src/services/dataContracts/AudiusContracts.ts index ead49b1857f..1f2bf84ef43 100644 --- a/libs/src/services/dataContracts/AudiusContracts.ts +++ b/libs/src/services/dataContracts/AudiusContracts.ts @@ -10,6 +10,7 @@ import { PlaylistFactoryClient } from './PlaylistFactoryClient' import { UserLibraryFactoryClient } from './UserLibraryFactoryClient' import { IPLDBlacklistFactoryClient } from './IPLDBlacklistFactoryClient' import { UserReplicaSetManagerClient } from './UserReplicaSetManagerClient' +import { EntityManagerClient } from './EntityManagerClient' import type { Web3Manager } from '../web3Manager' import type { ContractClient } from '../contracts/ContractClient' import { abi as RegistryABI } from '../../data-contracts/ABIs/Registry.json' @@ -20,6 +21,7 @@ import { abi as PlaylistFactoryABI } from '../../data-contracts/ABIs/PlaylistFac import { abi as UserLibraryFactoryABI } from '../../data-contracts/ABIs/UserLibraryFactory.json' import { abi as IPLDBlacklistFactoryABI } from '../../data-contracts/ABIs/IPLDBlacklistFactory.json' import { abi as UserReplicaSetManagerABI } from '../../data-contracts/ABIs/UserReplicaSetManager.json' +import { abi as EntityManagerABI } from '../../data-contracts/ABIs/EntityManager.json' // define contract registry keys const UserFactoryRegistryKey = 'UserFactory' @@ -33,6 +35,7 @@ const UserReplicaSetManagerRegistryKey = 'UserReplicaSetManager' export class AudiusContracts { web3Manager: Web3Manager registryAddress: string + entityManagerAddress: string isServer: boolean logger: Logger RegistryClient: RegistryClient @@ -42,6 +45,7 @@ export class AudiusContracts { PlaylistFactoryClient: PlaylistFactoryClient UserLibraryFactoryClient: UserLibraryFactoryClient IPLDBlacklistFactoryClient: IPLDBlacklistFactoryClient + EntityManagerClient: EntityManagerClient | undefined contractClients: ContractClient[] UserReplicaSetManagerClient: UserReplicaSetManagerClient | undefined | null contracts: Record | undefined @@ -50,11 +54,13 @@ export class AudiusContracts { constructor( web3Manager: Web3Manager, registryAddress: string, + entityManagerAddress: string, isServer: boolean, logger: Logger = console ) { this.web3Manager = web3Manager this.registryAddress = registryAddress + this.entityManagerAddress = entityManagerAddress this.isServer = isServer this.logger = logger @@ -122,6 +128,18 @@ export class AudiusContracts { this.UserLibraryFactoryClient, this.IPLDBlacklistFactoryClient ] + + if (this.entityManagerAddress) { + this.EntityManagerClient = new EntityManagerClient( + this.web3Manager, + EntityManagerABI, + 'EntityManager', + this.getRegistryAddressForContract, + this.logger, + this.entityManagerAddress + ) + this.contractClients.push(this.EntityManagerClient) + } } async init() { diff --git a/libs/src/services/dataContracts/EntityManagerClient.ts b/libs/src/services/dataContracts/EntityManagerClient.ts new file mode 100644 index 00000000000..fdc10f91ca2 --- /dev/null +++ b/libs/src/services/dataContracts/EntityManagerClient.ts @@ -0,0 +1,51 @@ +import { ContractClient } from '../contracts/ContractClient' +import * as signatureSchemas from '../../data-contracts/signatureSchemas' +import type { Web3Manager } from '../web3Manager' + +/** + * Generic management of Audius Data entities + **/ +export class EntityManagerClient extends ContractClient { + override web3Manager!: Web3Manager + + async manageEntity( + userId: number, + entityType: string, + entityId: number, + action: string, + metadata: string + ): Promise<{ txReceipt: any }> { + const nonce = signatureSchemas.getNonce() + const chainId = await this.getEthNetId() + const contractAddress = await this.getAddress() + const signatureData = signatureSchemas.generators.getManageEntityData( + chainId, + contractAddress, + userId, + entityType, + entityId, + action, + metadata, + nonce + ) + const sig = await this.web3Manager.signTypedData(signatureData) + const method = await this.getMethod( + 'manageEntity', + userId, + entityType, + entityId, + action, + metadata, + nonce, + sig + ) + const tx = await this.web3Manager.sendTransaction( + method, + this.contractRegistryKey, + contractAddress + ) + return { + txReceipt: tx + } + } +} diff --git a/libs/src/services/discoveryProvider/DiscoveryProvider.ts b/libs/src/services/discoveryProvider/DiscoveryProvider.ts index bbf102725ff..adad64e8718 100644 --- a/libs/src/services/discoveryProvider/DiscoveryProvider.ts +++ b/libs/src/services/discoveryProvider/DiscoveryProvider.ts @@ -380,7 +380,7 @@ export class DiscoveryProvider { idsArray: Nullable = null, targetUserId: Nullable = null, withUsers = false - ) { + ): Promise { const req = Requests.getPlaylists( limit, offset, @@ -391,6 +391,11 @@ export class DiscoveryProvider { return await this._makeRequest(req) } + async getFullPlaylist(encodedPlaylistId: string, encodedUserId: string) { + const req = Requests.getFullPlaylist(encodedPlaylistId, encodedUserId) + return await this._makeRequest(req) + } + /** * Return social feed for current user * @param filter - filter by "all", "original", or "repost" diff --git a/libs/src/services/discoveryProvider/requests.ts b/libs/src/services/discoveryProvider/requests.ts index 15e0eb3d37f..84e11bdad1b 100644 --- a/libs/src/services/discoveryProvider/requests.ts +++ b/libs/src/services/discoveryProvider/requests.ts @@ -241,6 +241,19 @@ export const getPlaylists = ( } } +export const getFullPlaylist = ( + encodedPlaylistId: string, + encodedUserId: string +) => { + return { + endpoint: 'v1/full/playlists', + urlParams: '/' + encodedPlaylistId, + queryParams: { + user_id: encodedUserId + } + } +} + export const getSocialFeed = ( filter: string, limit = 100, diff --git a/libs/src/services/schemaValidator/SchemaValidator.ts b/libs/src/services/schemaValidator/SchemaValidator.ts index a843db0a67f..a555afcb075 100644 --- a/libs/src/services/schemaValidator/SchemaValidator.ts +++ b/libs/src/services/schemaValidator/SchemaValidator.ts @@ -2,9 +2,11 @@ import { validate } from 'jsonschema' import TrackSchema from './schemas/trackSchema.json' import UserSchema from './schemas/userSchema.json' +import PlaylistSchema from './schemas/playlistSchema.json' export const trackSchemaType = 'TrackSchema' export const userSchemaType = 'UserSchema' +export const playlistSchemaType = 'PlaylistSchema' type SchemaConfig = { schema: { @@ -18,11 +20,15 @@ type SchemaConfig = { validate?: (obj: Record) => void } -type SchemaType = typeof trackSchemaType | typeof userSchemaType +type SchemaType = + | typeof trackSchemaType + | typeof userSchemaType + | typeof playlistSchemaType export type Schemas = { TrackSchema: SchemaConfig UserSchema: SchemaConfig + PlaylistSchema: SchemaConfig } export class SchemaValidator { @@ -50,6 +56,10 @@ export class SchemaValidator { [userSchemaType]: { schema: UserSchema, baseDefinition: 'User' + }, + [playlistSchemaType]: { + schema: PlaylistSchema, + baseDefinition: 'Playlist' } } diff --git a/libs/src/services/schemaValidator/schemas/playlistSchema.json b/libs/src/services/schemaValidator/schemas/playlistSchema.json new file mode 100644 index 00000000000..261828673c8 --- /dev/null +++ b/libs/src/services/schemaValidator/schemas/playlistSchema.json @@ -0,0 +1,70 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$ref": "#/definitions/Playlist", + "definitions": { + "Playlist": { + "type": "object", + "additionalProperties": true, + "properties": { + "playlist_id": { + "type": ["integer", "null"], + "default": null + }, + "playlist_contents": { + "type": "object", + "$ref": "#/definitions/PlaylistContents", + "default": { + "track_ids": [] + } + }, + "playlist_name": { + "type": ["string", "null"], + "default": null + }, + "playlist_image_sizes_multihash": { + "type": ["string", "null"], + "default": null, + "$ref": "#/definitions/CID" + }, + "description": { + "type": ["string", "null"], + "default": null + }, + "is_album": { + "type": "boolean", + "default": null + }, + "is_private": { + "type": "boolean", + "default": null + } + }, + "required": [ + "playlist_name", + "playlist_id", + "description", + "is_album", + "is_private" + ], + "title": "Playlist" + }, + "CID": { + "type": ["string", "null"], + "minLength": 46, + "maxLength": 46, + "pattern": "^Qm[a-zA-Z0-9]{44}$", + "title": "CID" + }, + "PlaylistContents": { + "type": "object", + "additionalProperties": false, + "properties": { + "track_ids": { + "type": "array", + "default": [] + } + }, + "title": "PlaylistContents" + } + } +} diff --git a/libs/src/utils/types.ts b/libs/src/utils/types.ts index fa3645e22de..532f3a11f23 100644 --- a/libs/src/utils/types.ts +++ b/libs/src/utils/types.ts @@ -133,7 +133,12 @@ export type CollectionMetadata = { is_delete: boolean is_private: boolean playlist_contents: { - track_ids: Array<{ time: number; track: ID; uid?: UID }> + track_ids: Array<{ + metadata_time: number + time: number + track: ID + uid?: UID + }> } tracks?: TrackMetadata[] track_count: number diff --git a/service-commands/src/commands/service-commands.json b/service-commands/src/commands/service-commands.json index 5b85df15122..7f29dd5e829 100644 --- a/service-commands/src/commands/service-commands.json +++ b/service-commands/src/commands/service-commands.json @@ -39,7 +39,7 @@ }, "contracts-predeployed": { "up": [ - "docker run --name audius_ganache_cli -d -p 8545:8545 --network=audius_dev audius/ganache:data-contracts-predeployed-a25931b955adce92676cec029661533f80013908", + "docker run --name audius_ganache_cli -d -p 8545:8545 --network=audius_dev audius/ganache:data-contracts-predeployed-f06dbfade0172e99d368882ab192e7377d6d6761", "docker cp audius_ganache_cli:/app/contract-config.json $PROTOCOL_DIR/identity-service/contract-config.json", "docker cp audius_ganache_cli:/app/contract-config.json $PROTOCOL_DIR/creator-node/contract-config.json", "docker cp audius_ganache_cli:/app/contract-config.json ~/.audius/contract-config.json",