From b7db9cd4aea9f7edaecbd030cf307e354e9ed59b Mon Sep 17 00:00:00 2001 From: CS Jiang Date: Wed, 27 Apr 2022 00:57:11 +0000 Subject: [PATCH 001/101] add playlists metadata upload route and test --- creator-node/src/models/playlist.js | 58 +++++++ creator-node/src/routes/playlists.js | 183 +++++++++++++++++++++ creator-node/src/utils/index.js | 7 + creator-node/test/playlists.test.js | 227 +++++++++++++++++++++++++++ 4 files changed, 475 insertions(+) create mode 100644 creator-node/src/models/playlist.js create mode 100644 creator-node/src/routes/playlists.js create mode 100644 creator-node/src/utils/index.js create mode 100644 creator-node/test/playlists.test.js diff --git a/creator-node/src/models/playlist.js b/creator-node/src/models/playlist.js new file mode 100644 index 00000000000..6e4aebaea61 --- /dev/null +++ b/creator-node/src/models/playlist.js @@ -0,0 +1,58 @@ +'use strict' + +module.exports = (sequelize, DataTypes) => { + const Playlist = sequelize.define( + 'Playlist', + { + cnodeUserUUID: { + type: DataTypes.UUID, + primaryKey: true, // composite primary key (cnodeUserUUID, clock) + allowNull: false + }, + playlistId: { + type: DataTypes.BIGINT, + allowNull: false + }, + metadataFileUUID: { + type: DataTypes.UUID, + allowNull: false + }, + metadataJSON: { + type: DataTypes.JSONB, + allowNull: false + }, + coverArtFileUUID: { + type: DataTypes.UUID, + allowNull: true + } + }, + { + indexes: [ + { + unique: true, + fields: ['playlistId'] + } + ] + } + ) + + Playlist.associate = function (models) { + Playlist.belongsTo(models.CNodeUser, { + foreignKey: 'cnodeUserUUID', + targetKey: 'cnodeUserUUID', + onDelete: 'RESTRICT' + }) + Playlist.belongsTo(models.File, { + foreignKey: 'metadataFileUUID', + targetKey: 'fileUUID', + onDelete: 'RESTRICT' + }) + Playlist.belongsTo(models.File, { + foreignKey: 'coverArtFileUUID', + targetKey: 'fileUUID', + onDelete: 'RESTRICT' + }) + } + + return Playlist +} diff --git a/creator-node/src/routes/playlists.js b/creator-node/src/routes/playlists.js new file mode 100644 index 00000000000..c25444f6251 --- /dev/null +++ b/creator-node/src/routes/playlists.js @@ -0,0 +1,183 @@ +const { Buffer } = require('ipfs-http-client') +const fs = require('fs') +const { promisify } = require('util') + +const models = require('../models') +const { saveFileFromBufferToDisk } = require('../fileManager') +const { + handleResponse, + successResponse, + errorResponseBadRequest, + errorResponseServerError +} = require('../apiHelpers') +const { validateStateForImageDirCIDAndReturnFileUUID } = require('../utils') +const { validateMetadata } = require('../utils') +const { + authMiddleware, + syncLockMiddleware, + ensurePrimaryMiddleware, + ensureStorageMiddleware, + issueAndWaitForSecondarySyncRequests +} = require('../middlewares') +const DBManager = require('../dbManager') + +const readFile = promisify(fs.readFile) + +module.exports = function (app) { + /** + * Create Playlist from provided metadata, and make metadata available to network + */ + app.post( + '/playlists/metadata', + authMiddleware, + ensurePrimaryMiddleware, + ensureStorageMiddleware, + syncLockMiddleware, + handleResponse(async (req, res) => { + const metadataJSON = req.body.metadata + const metadataBuffer = Buffer.from(JSON.stringify(metadataJSON)) + const cnodeUserUUID = req.session.cnodeUserUUID + const isValidMetadata = validateMetadata(req, metadataJSON) + if (!isValidMetadata) { + return errorResponseBadRequest('Invalid Playlist Metadata') + } + + // Save file from buffer to disk + let multihash, dstPath + try { + const resp = await saveFileFromBufferToDisk(req, metadataBuffer) + multihash = resp.cid + dstPath = resp.dstPath + } catch (e) { + return errorResponseServerError( + `saveFileFromBufferToDisk op failed: ${e}` + ) + } + + // Record metadata file entry in DB + const transaction = await models.sequelize.transaction() + let fileUUID + try { + const createFileQueryObj = { + multihash, + sourceFile: req.fileName, + storagePath: dstPath, + type: 'metadata' + } + const file = await DBManager.createNewDataRecord( + createFileQueryObj, + cnodeUserUUID, + models.File, + transaction + ) + fileUUID = file.fileUUID + await transaction.commit() + } catch (e) { + await transaction.rollback() + return errorResponseServerError( + `Could not save playlist metadata to db: ${e}` + ) + } + + // This call is not await-ed to avoid delaying or erroring + issueAndWaitForSecondarySyncRequests(req) + + return successResponse({ + metadataMultihash: multihash, + metadataFileUUID: fileUUID + }) + }) + ) + + /** + * Given playlist playlistId, blockNumber, and metadataFileUUID, creates/updates Playlist DB entry + * and associates image file entries with playlist. Ends playlist creation/update process. + */ + app.post( + '/playlists', + authMiddleware, + ensurePrimaryMiddleware, + ensureStorageMiddleware, + syncLockMiddleware, + handleResponse(async (req, res) => { + const { playlistId, blockNumber, metadataFileUUID } = req.body + + if (!playlistId || !blockNumber || !metadataFileUUID) { + return errorResponseBadRequest( + 'Must include playlistId, blockNumber, and metadataFileUUID.' + ) + } + + const cnodeUser = req.session.cnodeUser + if (blockNumber < cnodeUser.latestBlockNumber) { + return errorResponseBadRequest( + `Invalid blockNumber param ${blockNumber}. Must be greater or equal to previously processed blocknumber ${cnodeUser.latestBlockNumber}.` + ) + } + + const cnodeUserUUID = req.session.cnodeUserUUID + + // Fetch metadataJSON for metadataFileUUID. + const file = await models.File.findOne({ + where: { fileUUID: metadataFileUUID, cnodeUserUUID } + }) + if (!file) { + return errorResponseBadRequest( + `No file db record found for provided metadataFileUUID ${metadataFileUUID}.` + ) + } + let metadataJSON + try { + const fileBuffer = await readFile(file.storagePath) + metadataJSON = JSON.parse(fileBuffer) + } catch (e) { + return errorResponseServerError( + `No file stored on disk for metadataFileUUID ${metadataFileUUID} at storagePath ${file.storagePath}: ${e}.` + ) + } + + // Get coverArtFileUUID for multihashes in metadata object, if present. + let coverArtFileUUID + try { + coverArtFileUUID = await validateStateForImageDirCIDAndReturnFileUUID( + req, + metadataJSON.cover_photo_sizes + ) + } catch (e) { + return errorResponseBadRequest(e.message) + } + + // Record Playlist entry + update CNodeUser entry in DB + const transaction = await models.sequelize.transaction() + try { + const createPlaylistQueryObj = { + metadataFileUUID, + metadataJSON, + playlistId, + coverArtFileUUID + } + await DBManager.createNewDataRecord( + createPlaylistQueryObj, + cnodeUserUUID, + models.Playlist, + transaction + ) + + // Update cnodeUser.latestBlockNumber + await cnodeUser.update( + { latestBlockNumber: blockNumber }, + { transaction } + ) + + await transaction.commit() + + await issueAndWaitForSecondarySyncRequests(req) + + return successResponse() + } catch (e) { + await transaction.rollback() + return errorResponseServerError(e.message) + } + }) + ) +} diff --git a/creator-node/src/utils/index.js b/creator-node/src/utils/index.js new file mode 100644 index 00000000000..485b946652b --- /dev/null +++ b/creator-node/src/utils/index.js @@ -0,0 +1,7 @@ +const validateMetadata = require('./validateMetadata') +const { validateAssociatedWallets } = validateMetadata + +module.exports = { + validateMetadata, + validateAssociatedWallets +} diff --git a/creator-node/test/playlists.test.js b/creator-node/test/playlists.test.js new file mode 100644 index 00000000000..910f84aba30 --- /dev/null +++ b/creator-node/test/playlists.test.js @@ -0,0 +1,227 @@ +const request = require('supertest') +const assert = require('assert') +const sinon = require('sinon') +const fs = require('fs') + +const models = require('../src/models') + +const BlacklistManager = require('../src/blacklistManager') +const DiskManager = require('../src/diskManager') + +const { createStarterCNodeUser } = require('./lib/dataSeeds') +const { getLibsMock } = require('./lib/libsMock') +const { sortKeys } = require('../src/apiSigning') + +describe('Test Playlists', function () { + let app, server, session, libsMock + + beforeEach(async () => { + libsMock = getLibsMock() + + const userId = 1 + const spId = 1 + + const { getApp } = require('./lib/app') + + const appInfo = await getApp(libsMock, BlacklistManager, null, spId) + await BlacklistManager.init() + + app = appInfo.app + server = appInfo.server + session = await createStarterCNodeUser(userId) + }) + + afterEach(async () => { + sinon.restore() + await server.close() + }) + + it('should fail if metadata is not found in request body', async function () { + const resp = await request(app) + .post('/playlists/metadata') + .set('X-Session-ID', session.sessionToken) + .set('User-Id', session.userId) + .send({ dummy: 'data' }) + .expect(500) + + // Route will throw error at `Buffer.from(JSON.stringify(metadataJSON))` + assert.deepStrictEqual(resp.body.error, 'Internal server error') + }) + + it('should throw 400 bad request response if metadata validation fails', async function () { + const metadata = { metadata: 'spaghetti' } + const resp = await request(app) + .post('/playlists/metadata') + .set('X-Session-ID', session.sessionToken) + .set('User-Id', session.userId) + .send(metadata) + .expect(400) + + assert.deepStrictEqual(resp.body.error, 'Invalid Playlist Metadata') + }) + + it('successfully creates Audius playlist (POST /playlists/metadata)', async function () { + const metadata = { test: 'field1' } + const resp = await request(app) + .post('/playlists/metadata') + .set('X-Session-ID', session.sessionToken) + .set('User-Id', session.userId) + .send({ metadata }) + .expect(200) + + if (resp.body.data.metadataMultihash !== 'QmQMHXPMuey2AT6fPTKnzKQCrRjPS7AbaQdDTM8VXbHC8W' || !resp.body.data.metadataFileUUID) { + throw new Error('invalid return data') + } + + // check that the metadata file was written to storagePath under its multihash + const metadataPath = DiskManager.computeFilePath(resp.body.data.metadataMultihash) + assert.ok(fs.existsSync(metadataPath)) + + // check that the metadata file contents match the metadata specified + let metadataFileData = fs.readFileSync(metadataPath, 'utf-8') + metadataFileData = sortKeys(JSON.parse(metadataFileData)) + assert.deepStrictEqual(metadataFileData, metadata) + + // check that the correct metadata file properties were written to db + const file = await models.File.findOne({ where: { + multihash: resp.body.data.metadataMultihash, + storagePath: metadataPath, + type: 'metadata' + } }) + assert.ok(file) + }) + + it('successfully completes Audius playlist creation (POST /playlists/metadata -> POST /playlists)', async function () { + const metadata = { test: 'field1' } + const resp = await request(app) + .post('/playlists/metadata') + .set('X-Session-ID', session.sessionToken) + .set('User-Id', session.userId) + .send({ metadata }) + .expect(200) + + // check that the metadata file was written to storagePath under its multihash + const metadataPath = DiskManager.computeFilePath(resp.body.data.metadataMultihash) + assert.ok(fs.existsSync(metadataPath)) + + // check that the metadata file contents match the metadata specified + let metadataFileData = fs.readFileSync(metadataPath, 'utf-8') + metadataFileData = sortKeys(JSON.parse(metadataFileData)) + assert.deepStrictEqual(metadataFileData, metadata) + + // check that the correct metadata file properties were written to db + const file = await models.File.findOne({ where: { + multihash: resp.body.data.metadataMultihash, + storagePath: metadataPath, + type: 'metadata' + } }) + assert.ok(file) + + await request(app) + .post('/playlists') + .set('X-Session-ID', session.sessionToken) + .set('User-Id', session.userId) + .send({ playlistId: 1, blockNumber: 10, metadataFileUUID: resp.body.data.metadataFileUUID }) + .expect(200) + }) + + it('successfully completes Audius playlist creation when retrying an existing block number and original metadata', async function () { + const metadata = { test: 'field1' } + + libsMock.Playlist.getPlaylists.exactly(2) + + const resp = await request(app) + .post('/playlists/metadata') + .set('X-Session-ID', session.sessionToken) + .set('User-Id', session.userId) + .send({ metadata }) + .expect(200) + + if (resp.body.data.metadataMultihash !== 'QmQMHXPMuey2AT6fPTKnzKQCrRjPS7AbaQdDTM8VXbHC8W') { + throw new Error('invalid return data') + } + + await request(app) + .post('/playlists') + .set('X-Session-ID', session.sessionToken) + .set('User-Id', session.userId) + .send({ playlistId: 1, blockNumber: 10, metadataFileUUID: resp.body.data.metadataFileUUID }) + .expect(200) + + await request(app) + .post('/playlists') + .set('X-Session-ID', session.sessionToken) + .set('User-Id', session.userId) + .send({ playlistId: 1, blockNumber: 10, metadataFileUUID: resp.body.data.metadataFileUUID }) + .expect(200) + }) + + it('successfully completes Audius playlist creation when retrying an existing block number and new metadata', async function () { + const metadata1 = { test: 'field1' } + + libsMock.User.getUsers.exactly(2) + + const resp1 = await request(app) + .post('/playlists/metadata') + .set('X-Session-ID', session.sessionToken) + .set('User-Id', session.userId) + .send({ metadata: metadata1 }) + .expect(200) + + if (resp1.body.data.metadataMultihash !== 'QmQMHXPMuey2AT6fPTKnzKQCrRjPS7AbaQdDTM8VXbHC8W') { + throw new Error('invalid return data') + } + + const metadata2 = { test2: 'field2' } + + libsMock.User.getUsers.exactly(2) + + const resp2 = await request(app) + .post('/playlists/metadata') + .set('X-Session-ID', session.sessionToken) + .set('User-Id', session.userId) + .send({ metadata: metadata2 }) + .expect(200) + + if (resp2.body.data.metadataMultihash !== 'QmTiWeEp2PTedHwWbtJNUuZ3deRZgAM5TG2UWrsAN9ik1N') { + throw new Error('invalid return data') + } + + await request(app) + .post('/playlists') + .set('X-Session-ID', session.sessionToken) + .set('User-Id', session.userId) + .send({ playlistId: 1, blockNumber: 10, metadataFileUUID: resp2.body.data.metadataFileUUID }) + .expect(200) + }) + + it('fails Audius user creation when too low of a block number is supplied', async function () { + const metadata = { test: 'field1' } + + libsMock.User.getUsers.exactly(2) + + const resp = await request(app) + .post('/playlists/metadata') + .set('X-Session-ID', session.sessionToken) + .set('User-Id', session.userId) + .send({ metadata }) + .expect(200) + + if (resp.body.data.metadataMultihash !== 'QmQMHXPMuey2AT6fPTKnzKQCrRjPS7AbaQdDTM8VXbHC8W') { + throw new Error('invalid return data') + } + + // Fast forward to block number 100 + const cnodeUser = await models.CNodeUser.findOne({ where: { cnodeUserUUID: session.cnodeUserUUID } }) + await cnodeUser.update({ latestBlockNumber: 100 }) + + await request(app) + .post('/playlists') + .set('X-Session-ID', session.sessionToken) + .set('User-Id', session.userId) + .send({ playlistId: 1, blockNumber: 10, metadataFileUUID: resp.body.data.metadataFileUUID }) + .expect(400, { + error: 'Invalid blockNumber param 10. Must be greater or equal to previously processed blocknumber 100.' + }) + }) +}) From 55d6d272a0fd67677a830b63d7b708c812654234 Mon Sep 17 00:00:00 2001 From: CS Jiang Date: Wed, 27 Apr 2022 02:41:13 +0000 Subject: [PATCH 002/101] wip --- ...14326-add_playlist_to_clockrecords_enum.js | 105 ++++++++++++++++++ creator-node/src/models/clockRecord.js | 3 +- creator-node/src/models/playlist.js | 10 +- creator-node/src/routes/playlists.js | 2 +- creator-node/src/utils/index.js | 2 +- creator-node/test/lib/libsMock.js | 3 + creator-node/test/playlists.test.js | 4 + 7 files changed, 125 insertions(+), 4 deletions(-) create mode 100644 creator-node/sequelize/migrations/20220427014326-add_playlist_to_clockrecords_enum.js diff --git a/creator-node/sequelize/migrations/20220427014326-add_playlist_to_clockrecords_enum.js b/creator-node/sequelize/migrations/20220427014326-add_playlist_to_clockrecords_enum.js new file mode 100644 index 00000000000..2b2b2cdee27 --- /dev/null +++ b/creator-node/sequelize/migrations/20220427014326-add_playlist_to_clockrecords_enum.js @@ -0,0 +1,105 @@ +'use strict' + +/** + * New table - Playlist + * CNodeUsers Table considered a Reference Table only + */ + +module.exports = { + up: async (queryInterface, Sequelize) => { + console.log('STARTING MIGRATION 20220427014326-add_playlist_to_clockrecords_enum') + const transaction = await queryInterface.sequelize.transaction() + + // Create Playlist table + await createPlaylistTable(queryInterface, Sequelize, transaction) + + await addPlaylistToClockRecordSourceTables(queryInterface, Sequelize, transaction) + + // Add composite unique constraint on (playlistId, clock) to Playlist + await addCompositeUniqueConstraints(queryInterface, Sequelize, transaction) + + await transaction.commit() + console.log('FINISHED MIGRATION add_playlist_to_clockrecords_enum') + }, + + down: async (queryInterface, Sequelize) => {/* TODO */ } +} + +async function addPlaylistToClockRecordSourceTables(queryInterface, Sequelize, transaction) { + await queryInterface.sequelize.query(`ALTER TYPE "enum_ClockRecords_sourceTable" ADD VALUE 'Playlist'`); + +} + +async function addCompositeUniqueConstraints (queryInterface, Sequelize, transaction) { + await queryInterface.addConstraint( + 'Playlist', + { + type: 'UNIQUE', + fields: ['playlistId', 'clock'], + name: 'Playlist_unique_(playlistId,clock)', + transaction + } + ) +} + +async function createPlaylistTable (queryInterface, Sequelize, transaction) { + await queryInterface.createTable('Playlist', { + cnodeUserUUID: { + type: Sequelize.UUID, + primaryKey: false, + unique: false, + allowNull: false, + references: { + model: 'CNodeUsers', + key: 'cnodeUserUUID', + as: 'cnodeUserUUID' + }, + onDelete: 'RESTRICT' + }, + clock: { + type: Sequelize.INTEGER, + primaryKey: true, // composite primary key (cnodeUserUUID, clock) + unique: false, + allowNull: false + }, + metadataFileUUID: { + type: Sequelize.UUID, + allowNull: false, + onDelete: 'RESTRICT', + references: { + model: 'Files', + key: 'fileUUID', + as: 'metadataFileUUID' + } + }, + metadataJSON: { + type: Sequelize.JSONB, + allowNull: false + }, + playlistId: { + type: Sequelize.BIGINT, + allowNull: false, + primaryKey: true, // composite primary key (playlistId, clock) + unique: true, + onDelete: 'RESTRICT' + }, + coverArtFileUUID: { + type: Sequelize.UUID, + allowNull: true, + onDelete: 'RESTRICT', + references: { + model: 'Files', + key: 'fileUUID', + as: 'coverArtFileUUID' + } + }, + createdAt: { + allowNull: false, + type: Sequelize.DATE + }, + updatedAt: { + allowNull: false, + type: Sequelize.DATE + } + }, { transaction }) +} diff --git a/creator-node/src/models/clockRecord.js b/creator-node/src/models/clockRecord.js index 5745800e3da..c1aec894498 100644 --- a/creator-node/src/models/clockRecord.js +++ b/creator-node/src/models/clockRecord.js @@ -4,7 +4,8 @@ module.exports = (sequelize, DataTypes) => { const SourceTableTypesObj = { AudiusUser: 'AudiusUser', Track: 'Track', - File: 'File' + File: 'File', + Playlist: 'Playlist' } const ClockRecord = sequelize.define( diff --git a/creator-node/src/models/playlist.js b/creator-node/src/models/playlist.js index 6e4aebaea61..b646946fa98 100644 --- a/creator-node/src/models/playlist.js +++ b/creator-node/src/models/playlist.js @@ -6,11 +6,17 @@ module.exports = (sequelize, DataTypes) => { { cnodeUserUUID: { type: DataTypes.UUID, + primaryKey: false, + allowNull: false + }, + clock: { + type: DataTypes.INTEGER, primaryKey: true, // composite primary key (cnodeUserUUID, clock) allowNull: false }, playlistId: { type: DataTypes.BIGINT, + primaryKey: true, // composite primary key (playlistId, clock) allowNull: false }, metadataFileUUID: { @@ -30,7 +36,7 @@ module.exports = (sequelize, DataTypes) => { indexes: [ { unique: true, - fields: ['playlistId'] + fields: ['playlistId', 'clock'] } ] } @@ -52,6 +58,8 @@ module.exports = (sequelize, DataTypes) => { targetKey: 'fileUUID', onDelete: 'RESTRICT' }) + // Playlist also has a composite foreign key on ClockRecords (cnodeUserUUID, clock) + // sequelize does not support composite foreign keys } return Playlist diff --git a/creator-node/src/routes/playlists.js b/creator-node/src/routes/playlists.js index c25444f6251..f2668d020b5 100644 --- a/creator-node/src/routes/playlists.js +++ b/creator-node/src/routes/playlists.js @@ -11,7 +11,7 @@ const { errorResponseServerError } = require('../apiHelpers') const { validateStateForImageDirCIDAndReturnFileUUID } = require('../utils') -const { validateMetadata } = require('../utils') +const { validateMetadata } = require('../utils/index.js') const { authMiddleware, syncLockMiddleware, diff --git a/creator-node/src/utils/index.js b/creator-node/src/utils/index.js index 485b946652b..71194138fe3 100644 --- a/creator-node/src/utils/index.js +++ b/creator-node/src/utils/index.js @@ -1,4 +1,4 @@ -const validateMetadata = require('./validateMetadata') +const validateMetadata = require('./validateAudiusUserMetadata') const { validateAssociatedWallets } = validateMetadata module.exports = { diff --git a/creator-node/test/lib/libsMock.js b/creator-node/test/lib/libsMock.js index 56eb9927baa..2c4ed1a6664 100644 --- a/creator-node/test/lib/libsMock.js +++ b/creator-node/test/lib/libsMock.js @@ -52,6 +52,9 @@ function getLibsMock () { User: { getUsers: sinon.mock().atLeast(1) }, + Playlist: { + getPlaylists: sinon.mock().atLeast(1) + }, discoveryProvider: { discoveryProviderEndpoint: 'http://docker.for.mac.localhost:5000' } diff --git a/creator-node/test/playlists.test.js b/creator-node/test/playlists.test.js index 910f84aba30..fd613565703 100644 --- a/creator-node/test/playlists.test.js +++ b/creator-node/test/playlists.test.js @@ -122,6 +122,7 @@ describe('Test Playlists', function () { .set('X-Session-ID', session.sessionToken) .set('User-Id', session.userId) .send({ playlistId: 1, blockNumber: 10, metadataFileUUID: resp.body.data.metadataFileUUID }) + .expect(console.log) .expect(200) }) @@ -135,6 +136,7 @@ describe('Test Playlists', function () { .set('X-Session-ID', session.sessionToken) .set('User-Id', session.userId) .send({ metadata }) + .expect(console.log) .expect(200) if (resp.body.data.metadataMultihash !== 'QmQMHXPMuey2AT6fPTKnzKQCrRjPS7AbaQdDTM8VXbHC8W') { @@ -146,6 +148,7 @@ describe('Test Playlists', function () { .set('X-Session-ID', session.sessionToken) .set('User-Id', session.userId) .send({ playlistId: 1, blockNumber: 10, metadataFileUUID: resp.body.data.metadataFileUUID }) + .expect(console.log) .expect(200) await request(app) @@ -153,6 +156,7 @@ describe('Test Playlists', function () { .set('X-Session-ID', session.sessionToken) .set('User-Id', session.userId) .send({ playlistId: 1, blockNumber: 10, metadataFileUUID: resp.body.data.metadataFileUUID }) + .expect(console.log) .expect(200) }) From abc96aa1fd53adce6555c37704bacf02a4d276a9 Mon Sep 17 00:00:00 2001 From: CS Jiang Date: Wed, 27 Apr 2022 20:18:17 +0000 Subject: [PATCH 003/101] working implementation --- ...14326-add_playlist_to_clockrecords_enum.js | 7 +++---- creator-node/test/playlists.test.js | 21 ++++--------------- 2 files changed, 7 insertions(+), 21 deletions(-) diff --git a/creator-node/sequelize/migrations/20220427014326-add_playlist_to_clockrecords_enum.js b/creator-node/sequelize/migrations/20220427014326-add_playlist_to_clockrecords_enum.js index 2b2b2cdee27..3eb11e73c59 100644 --- a/creator-node/sequelize/migrations/20220427014326-add_playlist_to_clockrecords_enum.js +++ b/creator-node/sequelize/migrations/20220427014326-add_playlist_to_clockrecords_enum.js @@ -26,13 +26,12 @@ module.exports = { } async function addPlaylistToClockRecordSourceTables(queryInterface, Sequelize, transaction) { - await queryInterface.sequelize.query(`ALTER TYPE "enum_ClockRecords_sourceTable" ADD VALUE 'Playlist'`); - + await queryInterface.sequelize.query(`ALTER TYPE "enum_ClockRecords_sourceTable" ADD VALUE 'Playlist'`) } async function addCompositeUniqueConstraints (queryInterface, Sequelize, transaction) { await queryInterface.addConstraint( - 'Playlist', + 'Playlists', { type: 'UNIQUE', fields: ['playlistId', 'clock'], @@ -43,7 +42,7 @@ async function addCompositeUniqueConstraints (queryInterface, Sequelize, transac } async function createPlaylistTable (queryInterface, Sequelize, transaction) { - await queryInterface.createTable('Playlist', { + await queryInterface.createTable('Playlists', { cnodeUserUUID: { type: Sequelize.UUID, primaryKey: false, diff --git a/creator-node/test/playlists.test.js b/creator-node/test/playlists.test.js index fd613565703..c1d9a69185e 100644 --- a/creator-node/test/playlists.test.js +++ b/creator-node/test/playlists.test.js @@ -1,6 +1,5 @@ const request = require('supertest') const assert = require('assert') -const sinon = require('sinon') const fs = require('fs') const models = require('../src/models') @@ -32,7 +31,6 @@ describe('Test Playlists', function () { }) afterEach(async () => { - sinon.restore() await server.close() }) @@ -122,7 +120,6 @@ describe('Test Playlists', function () { .set('X-Session-ID', session.sessionToken) .set('User-Id', session.userId) .send({ playlistId: 1, blockNumber: 10, metadataFileUUID: resp.body.data.metadataFileUUID }) - .expect(console.log) .expect(200) }) @@ -136,7 +133,6 @@ describe('Test Playlists', function () { .set('X-Session-ID', session.sessionToken) .set('User-Id', session.userId) .send({ metadata }) - .expect(console.log) .expect(200) if (resp.body.data.metadataMultihash !== 'QmQMHXPMuey2AT6fPTKnzKQCrRjPS7AbaQdDTM8VXbHC8W') { @@ -148,22 +144,13 @@ describe('Test Playlists', function () { .set('X-Session-ID', session.sessionToken) .set('User-Id', session.userId) .send({ playlistId: 1, blockNumber: 10, metadataFileUUID: resp.body.data.metadataFileUUID }) - .expect(console.log) - .expect(200) - - await request(app) - .post('/playlists') - .set('X-Session-ID', session.sessionToken) - .set('User-Id', session.userId) - .send({ playlistId: 1, blockNumber: 10, metadataFileUUID: resp.body.data.metadataFileUUID }) - .expect(console.log) .expect(200) }) it('successfully completes Audius playlist creation when retrying an existing block number and new metadata', async function () { const metadata1 = { test: 'field1' } - libsMock.User.getUsers.exactly(2) + libsMock.Playlist.getPlaylists.exactly(2) const resp1 = await request(app) .post('/playlists/metadata') @@ -178,7 +165,7 @@ describe('Test Playlists', function () { const metadata2 = { test2: 'field2' } - libsMock.User.getUsers.exactly(2) + libsMock.Playlist.getPlaylists.exactly(2) const resp2 = await request(app) .post('/playlists/metadata') @@ -199,10 +186,10 @@ describe('Test Playlists', function () { .expect(200) }) - it('fails Audius user creation when too low of a block number is supplied', async function () { + it('fails Audius playlist creation when too low of a block number is supplied', async function () { const metadata = { test: 'field1' } - libsMock.User.getUsers.exactly(2) + libsMock.Playlist.getPlaylists.exactly(2) const resp = await request(app) .post('/playlists/metadata') From 173142902076ddf565491e4bca13ca7eaabb674a Mon Sep 17 00:00:00 2001 From: CS Jiang Date: Wed, 27 Apr 2022 20:27:52 +0000 Subject: [PATCH 004/101] remove ipfs-http-client dep --- creator-node/src/routes/playlists.js | 1 - 1 file changed, 1 deletion(-) diff --git a/creator-node/src/routes/playlists.js b/creator-node/src/routes/playlists.js index f2668d020b5..fd67a06360f 100644 --- a/creator-node/src/routes/playlists.js +++ b/creator-node/src/routes/playlists.js @@ -1,4 +1,3 @@ -const { Buffer } = require('ipfs-http-client') const fs = require('fs') const { promisify } = require('util') From 1c79cf4bcc906ceeaab4249574156d719f0b45a4 Mon Sep 17 00:00:00 2001 From: CS Jiang Date: Wed, 27 Apr 2022 20:57:40 +0000 Subject: [PATCH 005/101] add down migration --- ...14326-add_playlist_to_clockrecords_enum.js | 31 +++++++++++++++++-- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/creator-node/sequelize/migrations/20220427014326-add_playlist_to_clockrecords_enum.js b/creator-node/sequelize/migrations/20220427014326-add_playlist_to_clockrecords_enum.js index 3eb11e73c59..050c0850faa 100644 --- a/creator-node/sequelize/migrations/20220427014326-add_playlist_to_clockrecords_enum.js +++ b/creator-node/sequelize/migrations/20220427014326-add_playlist_to_clockrecords_enum.js @@ -5,6 +5,8 @@ * CNodeUsers Table considered a Reference Table only */ +const tableName = 'Playlists' + module.exports = { up: async (queryInterface, Sequelize) => { console.log('STARTING MIGRATION 20220427014326-add_playlist_to_clockrecords_enum') @@ -22,16 +24,31 @@ module.exports = { console.log('FINISHED MIGRATION add_playlist_to_clockrecords_enum') }, - down: async (queryInterface, Sequelize) => {/* TODO */ } + down: async (queryInterface, Sequelize) => { + await removeCompositeUniqueConstraints(queryInterface, Sequelize) + + await removePlaylistFromClockRecordSourceTables(queryInterface, Sequelize) + + await dropPlaylistTable(queryInterface, Sequelize) + } } async function addPlaylistToClockRecordSourceTables(queryInterface, Sequelize, transaction) { await queryInterface.sequelize.query(`ALTER TYPE "enum_ClockRecords_sourceTable" ADD VALUE 'Playlist'`) } +async function removePlaylistFromClockRecordSourceTables(queryInterface, Sequelize) { + await queryInterface.sequelize.query(` + ALTER TYPE "enum_ClockRecords_sourceTable" RENAME TO "enum_ClockRecords_sourceTable_old"; + CREATE TYPE "enum_ClockRecords_sourceTable" AS ENUM('AudiusUser', 'Track', 'File'); + ALTER TABLE "ClockRecords" ALTER COLUMN sourceTable TYPE enum_ClockRecords_sourceTable USING sourceTable::text::enum_ClockRecords_sourceTable; + DROP TYPE "enum_ClockRecords_sourceTable_old"; + `) +} + async function addCompositeUniqueConstraints (queryInterface, Sequelize, transaction) { await queryInterface.addConstraint( - 'Playlists', + tableName, { type: 'UNIQUE', fields: ['playlistId', 'clock'], @@ -41,8 +58,12 @@ async function addCompositeUniqueConstraints (queryInterface, Sequelize, transac ) } +async function removeCompositeUniqueConstraints (queryInterface, Sequelize) { + await queryInterface.sequelize.query(`ALTER TABLE ${tableName} DROP CONSTRAINT Playlist_unique_(playlistId,clock); ${tableName}`) +} + async function createPlaylistTable (queryInterface, Sequelize, transaction) { - await queryInterface.createTable('Playlists', { + await queryInterface.createTable(tableName, { cnodeUserUUID: { type: Sequelize.UUID, primaryKey: false, @@ -102,3 +123,7 @@ async function createPlaylistTable (queryInterface, Sequelize, transaction) { } }, { transaction }) } + +async function dropPlaylistTable(queryInterface, Sequelize) { + await queryInterface.dropTable(tableName) +} \ No newline at end of file From 3100858b5706d9054150490e6e43a173075106b0 Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Tue, 14 Jun 2022 19:33:15 +0000 Subject: [PATCH 006/101] Fixing lint changes --- service-commands/scripts/A | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/service-commands/scripts/A b/service-commands/scripts/A index f469362beeb..d88afb29d2c 100755 --- a/service-commands/scripts/A +++ b/service-commands/scripts/A @@ -26,6 +26,7 @@ def create_instance(provider, image, disk_size, machine_type, spot_instance, nam check=False, ) + def create_instance_command_parser(subparser): parser_create_instance = subparser.add_parser( "create-instance", @@ -66,8 +67,17 @@ def create_instance_command_parser(subparser): help="name to assign to the created instance", ) + def setup( - provider, service, user, config, fast, protocol_git_ref, client_git_ref, spot_instance, name + provider, + service, + user, + config, + fast, + protocol_git_ref, + client_git_ref, + spot_instance, + name, ): """Setup a environment suitable for development on a remote machine""" subprocess.run( @@ -170,12 +180,11 @@ def main(): parser = argparse.ArgumentParser( exit_on_error=False, formatter_class=argparse.RawDescriptionHelpFormatter, - epilog= -""" + epilog=""" Other commands: `seed` - seed user data eg `A seed ` or `A seed --help` for options all other commands will pass through to service-commands/scripts/setup.js -""" +""", ) subparser = parser.add_subparsers( From 54be6301353f8b58920d62bb657df76c4a379107 Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Wed, 15 Jun 2022 20:13:17 +0000 Subject: [PATCH 007/101] WORKING MIGRATION AFTER FIX --- ...14326-add_playlist_to_clockrecords_enum.js | 129 ------------------ 1 file changed, 129 deletions(-) delete mode 100644 creator-node/sequelize/migrations/20220427014326-add_playlist_to_clockrecords_enum.js diff --git a/creator-node/sequelize/migrations/20220427014326-add_playlist_to_clockrecords_enum.js b/creator-node/sequelize/migrations/20220427014326-add_playlist_to_clockrecords_enum.js deleted file mode 100644 index 050c0850faa..00000000000 --- a/creator-node/sequelize/migrations/20220427014326-add_playlist_to_clockrecords_enum.js +++ /dev/null @@ -1,129 +0,0 @@ -'use strict' - -/** - * New table - Playlist - * CNodeUsers Table considered a Reference Table only - */ - -const tableName = 'Playlists' - -module.exports = { - up: async (queryInterface, Sequelize) => { - console.log('STARTING MIGRATION 20220427014326-add_playlist_to_clockrecords_enum') - const transaction = await queryInterface.sequelize.transaction() - - // Create Playlist table - await createPlaylistTable(queryInterface, Sequelize, transaction) - - await addPlaylistToClockRecordSourceTables(queryInterface, Sequelize, transaction) - - // Add composite unique constraint on (playlistId, clock) to Playlist - await addCompositeUniqueConstraints(queryInterface, Sequelize, transaction) - - await transaction.commit() - console.log('FINISHED MIGRATION add_playlist_to_clockrecords_enum') - }, - - down: async (queryInterface, Sequelize) => { - await removeCompositeUniqueConstraints(queryInterface, Sequelize) - - await removePlaylistFromClockRecordSourceTables(queryInterface, Sequelize) - - await dropPlaylistTable(queryInterface, Sequelize) - } -} - -async function addPlaylistToClockRecordSourceTables(queryInterface, Sequelize, transaction) { - await queryInterface.sequelize.query(`ALTER TYPE "enum_ClockRecords_sourceTable" ADD VALUE 'Playlist'`) -} - -async function removePlaylistFromClockRecordSourceTables(queryInterface, Sequelize) { - await queryInterface.sequelize.query(` - ALTER TYPE "enum_ClockRecords_sourceTable" RENAME TO "enum_ClockRecords_sourceTable_old"; - CREATE TYPE "enum_ClockRecords_sourceTable" AS ENUM('AudiusUser', 'Track', 'File'); - ALTER TABLE "ClockRecords" ALTER COLUMN sourceTable TYPE enum_ClockRecords_sourceTable USING sourceTable::text::enum_ClockRecords_sourceTable; - DROP TYPE "enum_ClockRecords_sourceTable_old"; - `) -} - -async function addCompositeUniqueConstraints (queryInterface, Sequelize, transaction) { - await queryInterface.addConstraint( - tableName, - { - type: 'UNIQUE', - fields: ['playlistId', 'clock'], - name: 'Playlist_unique_(playlistId,clock)', - transaction - } - ) -} - -async function removeCompositeUniqueConstraints (queryInterface, Sequelize) { - await queryInterface.sequelize.query(`ALTER TABLE ${tableName} DROP CONSTRAINT Playlist_unique_(playlistId,clock); ${tableName}`) -} - -async function createPlaylistTable (queryInterface, Sequelize, transaction) { - await queryInterface.createTable(tableName, { - cnodeUserUUID: { - type: Sequelize.UUID, - primaryKey: false, - unique: false, - allowNull: false, - references: { - model: 'CNodeUsers', - key: 'cnodeUserUUID', - as: 'cnodeUserUUID' - }, - onDelete: 'RESTRICT' - }, - clock: { - type: Sequelize.INTEGER, - primaryKey: true, // composite primary key (cnodeUserUUID, clock) - unique: false, - allowNull: false - }, - metadataFileUUID: { - type: Sequelize.UUID, - allowNull: false, - onDelete: 'RESTRICT', - references: { - model: 'Files', - key: 'fileUUID', - as: 'metadataFileUUID' - } - }, - metadataJSON: { - type: Sequelize.JSONB, - allowNull: false - }, - playlistId: { - type: Sequelize.BIGINT, - allowNull: false, - primaryKey: true, // composite primary key (playlistId, clock) - unique: true, - onDelete: 'RESTRICT' - }, - coverArtFileUUID: { - type: Sequelize.UUID, - allowNull: true, - onDelete: 'RESTRICT', - references: { - model: 'Files', - key: 'fileUUID', - as: 'coverArtFileUUID' - } - }, - createdAt: { - allowNull: false, - type: Sequelize.DATE - }, - updatedAt: { - allowNull: false, - type: Sequelize.DATE - } - }, { transaction }) -} - -async function dropPlaylistTable(queryInterface, Sequelize) { - await queryInterface.dropTable(tableName) -} \ No newline at end of file From 5497968fe51960e83be8c1b25d21d3570295dab6 Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Wed, 15 Jun 2022 21:40:27 +0000 Subject: [PATCH 008/101] Properly functioning up/down migration, optimizing separately --- ...te_playlist_table_and_clock_record_type.js | 156 ++++++++++++++++++ 1 file changed, 156 insertions(+) create mode 100644 creator-node/sequelize/migrations/20220427014326-create_playlist_table_and_clock_record_type.js diff --git a/creator-node/sequelize/migrations/20220427014326-create_playlist_table_and_clock_record_type.js b/creator-node/sequelize/migrations/20220427014326-create_playlist_table_and_clock_record_type.js new file mode 100644 index 00000000000..57736379cca --- /dev/null +++ b/creator-node/sequelize/migrations/20220427014326-create_playlist_table_and_clock_record_type.js @@ -0,0 +1,156 @@ +'use strict' + +/** + * New table - Playlist + * New enum value for clock records sourceTable, Playlist + * CNodeUsers Table considered a Reference Table only + */ + +const tableName = 'Playlists' + +module.exports = { + up: async (queryInterface, Sequelize) => { + const transaction = await queryInterface.sequelize.transaction() + + // Create Playlist table + await createPlaylistTable(queryInterface, Sequelize, transaction) + await addPlaylistToClockRecordSourceTables(queryInterface, transaction) + + // Add composite unique constraint on (blockchainId, clock) to Playlist + await addCompositeUniqueConstraints(queryInterface, transaction) + + await transaction.commit() + }, + + down: async (queryInterface, Sequelize) => { + try { + const transaction = await queryInterface.sequelize.transaction() + await removeCompositeUniqueConstraints(queryInterface, transaction) + await removePlaylistFromClockRecordSourceTables( + queryInterface, + transaction + ) + await dropPlaylistTable(queryInterface, transaction) + await transaction.commit() + } catch (e) { + console.log(e) + } + } +} + +async function addPlaylistToClockRecordSourceTables(queryInterface) { + // Note that this cannot be run inside a transaction block + await queryInterface.sequelize.query( + `ALTER TYPE "enum_ClockRecords_sourceTable" ADD VALUE IF NOT EXISTS 'Playlist'` + ) +} + +async function removePlaylistFromClockRecordSourceTables( + queryInterface, + transaction +) { + await queryInterface.sequelize.query( + ` + ALTER TYPE "enum_ClockRecords_sourceTable" RENAME TO "enum_ClockRecords_sourceTable_old"; + CREATE TYPE "enum_ClockRecords_sourceTable" AS ENUM('AudiusUser', 'Track', 'File'); + ALTER TABLE "ClockRecords" ALTER COLUMN "sourceTable" TYPE "enum_ClockRecords_sourceTable" USING "sourceTable"::text::"enum_ClockRecords_sourceTable"; + DROP TYPE "enum_ClockRecords_sourceTable_old"; + `, + { transaction } + ) +} + +async function addCompositeUniqueConstraints(queryInterface, transaction) { + await queryInterface.addConstraint(tableName, { + type: 'UNIQUE', + fields: ['blockchainId', 'clock'], + name: 'Playlist_unique_(blockchainId,clock)', + transaction + }) +} + +async function removeCompositeUniqueConstraints(queryInterface, transaction) { + try { + await queryInterface.removeConstraint( + tableName, + 'Playlist_unique_(blockchainId,clock)', + { transaction } + ) + } catch (e) { + console.log(e) + } +} + +async function createPlaylistTable(queryInterface, Sequelize, transaction) { + await queryInterface.createTable( + tableName, + { + cnodeUserUUID: { + type: Sequelize.UUID, + primaryKey: true, // composite primary key (cnodeUserUUID, clock) + unique: false, + allowNull: false, + references: { + model: 'CNodeUsers', + key: 'cnodeUserUUID', + as: 'cnodeUserUUID' + }, + onDelete: 'RESTRICT' + }, + clock: { + type: Sequelize.INTEGER, + primaryKey: true, // composite primary key (cnodeUserUUID, clock) + unique: 'compositeIndex', + allowNull: false + }, + metadataFileUUID: { + type: Sequelize.UUID, + allowNull: false, + onDelete: 'RESTRICT', + references: { + model: 'Files', + key: 'fileUUID', + as: 'metadataFileUUID' + } + }, + metadataJSON: { + type: Sequelize.JSONB, + allowNull: false + }, + blockchainId: { + type: Sequelize.BIGINT, + allowNull: false, + primaryKey: false, + unique: 'compositeIndex', + onDelete: 'RESTRICT' + }, + playlistImageFileUUID: { + type: Sequelize.UUID, + allowNull: true, + onDelete: 'RESTRICT', + references: { + model: 'Files', + key: 'fileUUID', + as: 'playlistImageFileUUID' + } + }, + createdAt: { + allowNull: false, + type: Sequelize.DATE + }, + updatedAt: { + allowNull: false, + type: Sequelize.DATE + } + }, + { + transaction + } + ) +} + +async function dropPlaylistTable(queryInterface, transaction) { + await queryInterface.dropTable(tableName, { + transaction + }) +} From 589b40669d0aa632cdb5e80f1910b21de3f2b59e Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Wed, 15 Jun 2022 21:43:07 +0000 Subject: [PATCH 009/101] Corresponding model changes --- creator-node/src/models/playlist.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/creator-node/src/models/playlist.js b/creator-node/src/models/playlist.js index b646946fa98..a7b08483878 100644 --- a/creator-node/src/models/playlist.js +++ b/creator-node/src/models/playlist.js @@ -6,7 +6,7 @@ module.exports = (sequelize, DataTypes) => { { cnodeUserUUID: { type: DataTypes.UUID, - primaryKey: false, + primaryKey: true, // composite primary key (cnodeUserUUID, clock) allowNull: false }, clock: { @@ -14,9 +14,9 @@ module.exports = (sequelize, DataTypes) => { primaryKey: true, // composite primary key (cnodeUserUUID, clock) allowNull: false }, - playlistId: { + blockchainId: { type: DataTypes.BIGINT, - primaryKey: true, // composite primary key (playlistId, clock) + primaryKey: true, allowNull: false }, metadataFileUUID: { @@ -36,7 +36,7 @@ module.exports = (sequelize, DataTypes) => { indexes: [ { unique: true, - fields: ['playlistId', 'clock'] + fields: ['blockchainId', 'clock'] } ] } From 94209627e6e40ee6073793804b285d488649d9bb Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Thu, 16 Jun 2022 14:34:52 +0000 Subject: [PATCH 010/101] Addressing rest of comments --- creator-node/src/models/playlist.js | 4 +- creator-node/src/routes/playlists.js | 25 ++++---- creator-node/test/playlists.test.js | 89 +++++++++++++++++++++------- 3 files changed, 81 insertions(+), 37 deletions(-) diff --git a/creator-node/src/models/playlist.js b/creator-node/src/models/playlist.js index a7b08483878..47d4784efee 100644 --- a/creator-node/src/models/playlist.js +++ b/creator-node/src/models/playlist.js @@ -27,7 +27,7 @@ module.exports = (sequelize, DataTypes) => { type: DataTypes.JSONB, allowNull: false }, - coverArtFileUUID: { + playlistImageFileUUID: { type: DataTypes.UUID, allowNull: true } @@ -54,7 +54,7 @@ module.exports = (sequelize, DataTypes) => { onDelete: 'RESTRICT' }) Playlist.belongsTo(models.File, { - foreignKey: 'coverArtFileUUID', + foreignKey: 'playlistImageFileUUID', targetKey: 'fileUUID', onDelete: 'RESTRICT' }) diff --git a/creator-node/src/routes/playlists.js b/creator-node/src/routes/playlists.js index fd67a06360f..dcb6f0e0137 100644 --- a/creator-node/src/routes/playlists.js +++ b/creator-node/src/routes/playlists.js @@ -89,7 +89,7 @@ module.exports = function (app) { ) /** - * Given playlist playlistId, blockNumber, and metadataFileUUID, creates/updates Playlist DB entry + * Given playlist blockchainId, blockNumber, and metadataFileUUID, creates/updates Playlist DB entry * and associates image file entries with playlist. Ends playlist creation/update process. */ app.post( @@ -99,11 +99,11 @@ module.exports = function (app) { ensureStorageMiddleware, syncLockMiddleware, handleResponse(async (req, res) => { - const { playlistId, blockNumber, metadataFileUUID } = req.body + const { blockchainId, blockNumber, metadataFileUUID } = req.body - if (!playlistId || !blockNumber || !metadataFileUUID) { + if (!blockchainId || !blockNumber || !metadataFileUUID) { return errorResponseBadRequest( - 'Must include playlistId, blockNumber, and metadataFileUUID.' + 'Must include blockchainId, blockNumber, and metadataFileUUID.' ) } @@ -135,13 +135,14 @@ module.exports = function (app) { ) } - // Get coverArtFileUUID for multihashes in metadata object, if present. - let coverArtFileUUID + // Get playlistImageFileUUID for multihashes in metadata object, if present. + let playlistImageFileUUID try { - coverArtFileUUID = await validateStateForImageDirCIDAndReturnFileUUID( - req, - metadataJSON.cover_photo_sizes - ) + playlistImageFileUUID = + await validateStateForImageDirCIDAndReturnFileUUID( + req, + metadataJSON.cover_photo_sizes + ) } catch (e) { return errorResponseBadRequest(e.message) } @@ -152,8 +153,8 @@ module.exports = function (app) { const createPlaylistQueryObj = { metadataFileUUID, metadataJSON, - playlistId, - coverArtFileUUID + blockchainId, + playlistImageFileUUID } await DBManager.createNewDataRecord( createPlaylistQueryObj, diff --git a/creator-node/test/playlists.test.js b/creator-node/test/playlists.test.js index c1d9a69185e..36e48aa8e9b 100644 --- a/creator-node/test/playlists.test.js +++ b/creator-node/test/playlists.test.js @@ -67,12 +67,18 @@ describe('Test Playlists', function () { .send({ metadata }) .expect(200) - if (resp.body.data.metadataMultihash !== 'QmQMHXPMuey2AT6fPTKnzKQCrRjPS7AbaQdDTM8VXbHC8W' || !resp.body.data.metadataFileUUID) { + if ( + resp.body.data.metadataMultihash !== + 'QmQMHXPMuey2AT6fPTKnzKQCrRjPS7AbaQdDTM8VXbHC8W' || + !resp.body.data.metadataFileUUID + ) { throw new Error('invalid return data') } // check that the metadata file was written to storagePath under its multihash - const metadataPath = DiskManager.computeFilePath(resp.body.data.metadataMultihash) + const metadataPath = DiskManager.computeFilePath( + resp.body.data.metadataMultihash + ) assert.ok(fs.existsSync(metadataPath)) // check that the metadata file contents match the metadata specified @@ -81,11 +87,13 @@ describe('Test Playlists', function () { assert.deepStrictEqual(metadataFileData, metadata) // check that the correct metadata file properties were written to db - const file = await models.File.findOne({ where: { - multihash: resp.body.data.metadataMultihash, - storagePath: metadataPath, - type: 'metadata' - } }) + const file = await models.File.findOne({ + where: { + multihash: resp.body.data.metadataMultihash, + storagePath: metadataPath, + type: 'metadata' + } + }) assert.ok(file) }) @@ -99,7 +107,9 @@ describe('Test Playlists', function () { .expect(200) // check that the metadata file was written to storagePath under its multihash - const metadataPath = DiskManager.computeFilePath(resp.body.data.metadataMultihash) + const metadataPath = DiskManager.computeFilePath( + resp.body.data.metadataMultihash + ) assert.ok(fs.existsSync(metadataPath)) // check that the metadata file contents match the metadata specified @@ -108,18 +118,24 @@ describe('Test Playlists', function () { assert.deepStrictEqual(metadataFileData, metadata) // check that the correct metadata file properties were written to db - const file = await models.File.findOne({ where: { - multihash: resp.body.data.metadataMultihash, - storagePath: metadataPath, - type: 'metadata' - } }) + const file = await models.File.findOne({ + where: { + multihash: resp.body.data.metadataMultihash, + storagePath: metadataPath, + type: 'metadata' + } + }) assert.ok(file) await request(app) .post('/playlists') .set('X-Session-ID', session.sessionToken) .set('User-Id', session.userId) - .send({ playlistId: 1, blockNumber: 10, metadataFileUUID: resp.body.data.metadataFileUUID }) + .send({ + blockchainId: 1, + blockNumber: 10, + metadataFileUUID: resp.body.data.metadataFileUUID + }) .expect(200) }) @@ -135,7 +151,10 @@ describe('Test Playlists', function () { .send({ metadata }) .expect(200) - if (resp.body.data.metadataMultihash !== 'QmQMHXPMuey2AT6fPTKnzKQCrRjPS7AbaQdDTM8VXbHC8W') { + if ( + resp.body.data.metadataMultihash !== + 'QmQMHXPMuey2AT6fPTKnzKQCrRjPS7AbaQdDTM8VXbHC8W' + ) { throw new Error('invalid return data') } @@ -143,7 +162,11 @@ describe('Test Playlists', function () { .post('/playlists') .set('X-Session-ID', session.sessionToken) .set('User-Id', session.userId) - .send({ playlistId: 1, blockNumber: 10, metadataFileUUID: resp.body.data.metadataFileUUID }) + .send({ + blockchainId: 1, + blockNumber: 10, + metadataFileUUID: resp.body.data.metadataFileUUID + }) .expect(200) }) @@ -159,7 +182,10 @@ describe('Test Playlists', function () { .send({ metadata: metadata1 }) .expect(200) - if (resp1.body.data.metadataMultihash !== 'QmQMHXPMuey2AT6fPTKnzKQCrRjPS7AbaQdDTM8VXbHC8W') { + if ( + resp1.body.data.metadataMultihash !== + 'QmQMHXPMuey2AT6fPTKnzKQCrRjPS7AbaQdDTM8VXbHC8W' + ) { throw new Error('invalid return data') } @@ -174,7 +200,10 @@ describe('Test Playlists', function () { .send({ metadata: metadata2 }) .expect(200) - if (resp2.body.data.metadataMultihash !== 'QmTiWeEp2PTedHwWbtJNUuZ3deRZgAM5TG2UWrsAN9ik1N') { + if ( + resp2.body.data.metadataMultihash !== + 'QmTiWeEp2PTedHwWbtJNUuZ3deRZgAM5TG2UWrsAN9ik1N' + ) { throw new Error('invalid return data') } @@ -182,7 +211,11 @@ describe('Test Playlists', function () { .post('/playlists') .set('X-Session-ID', session.sessionToken) .set('User-Id', session.userId) - .send({ playlistId: 1, blockNumber: 10, metadataFileUUID: resp2.body.data.metadataFileUUID }) + .send({ + blockchainId: 1, + blockNumber: 10, + metadataFileUUID: resp2.body.data.metadataFileUUID + }) .expect(200) }) @@ -198,21 +231,31 @@ describe('Test Playlists', function () { .send({ metadata }) .expect(200) - if (resp.body.data.metadataMultihash !== 'QmQMHXPMuey2AT6fPTKnzKQCrRjPS7AbaQdDTM8VXbHC8W') { + if ( + resp.body.data.metadataMultihash !== + 'QmQMHXPMuey2AT6fPTKnzKQCrRjPS7AbaQdDTM8VXbHC8W' + ) { throw new Error('invalid return data') } // Fast forward to block number 100 - const cnodeUser = await models.CNodeUser.findOne({ where: { cnodeUserUUID: session.cnodeUserUUID } }) + const cnodeUser = await models.CNodeUser.findOne({ + where: { cnodeUserUUID: session.cnodeUserUUID } + }) await cnodeUser.update({ latestBlockNumber: 100 }) await request(app) .post('/playlists') .set('X-Session-ID', session.sessionToken) .set('User-Id', session.userId) - .send({ playlistId: 1, blockNumber: 10, metadataFileUUID: resp.body.data.metadataFileUUID }) + .send({ + blockchainId: 1, + blockNumber: 10, + metadataFileUUID: resp.body.data.metadataFileUUID + }) .expect(400, { - error: 'Invalid blockNumber param 10. Must be greater or equal to previously processed blocknumber 100.' + error: + 'Invalid blockNumber param 10. Must be greater or equal to previously processed blocknumber 100.' }) }) }) From de41a3cb2c6177e919d8ac7a5fec7b899980ead0 Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Thu, 16 Jun 2022 14:53:09 +0000 Subject: [PATCH 011/101] Update playlist image sizes multihash field --- creator-node/src/routes/playlists.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/creator-node/src/routes/playlists.js b/creator-node/src/routes/playlists.js index dcb6f0e0137..f25e122d331 100644 --- a/creator-node/src/routes/playlists.js +++ b/creator-node/src/routes/playlists.js @@ -141,7 +141,7 @@ module.exports = function (app) { playlistImageFileUUID = await validateStateForImageDirCIDAndReturnFileUUID( req, - metadataJSON.cover_photo_sizes + metadataJSON.playlist_image_sizes_multihash ) } catch (e) { return errorResponseBadRequest(e.message) From 5f7317effdbe0885e8b724c22df2580513de73ba Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Thu, 16 Jun 2022 17:01:22 +0000 Subject: [PATCH 012/101] Fix hyphenated migration --- ...create-playlist-table-and-clock-record-type.js} | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) rename creator-node/sequelize/migrations/{20220427014326-create_playlist_table_and_clock_record_type.js => 20220427014326-create-playlist-table-and-clock-record-type.js} (83%) diff --git a/creator-node/sequelize/migrations/20220427014326-create_playlist_table_and_clock_record_type.js b/creator-node/sequelize/migrations/20220427014326-create-playlist-table-and-clock-record-type.js similarity index 83% rename from creator-node/sequelize/migrations/20220427014326-create_playlist_table_and_clock_record_type.js rename to creator-node/sequelize/migrations/20220427014326-create-playlist-table-and-clock-record-type.js index 57736379cca..eedaeb651e2 100644 --- a/creator-node/sequelize/migrations/20220427014326-create_playlist_table_and_clock_record_type.js +++ b/creator-node/sequelize/migrations/20220427014326-create-playlist-table-and-clock-record-type.js @@ -38,10 +38,20 @@ module.exports = { } } -async function addPlaylistToClockRecordSourceTables(queryInterface) { +async function addPlaylistToClockRecordSourceTables( + queryInterface, + transaction +) { // Note that this cannot be run inside a transaction block + // TODO: Instead of alter type, rename and create new type in order to preserve atomicity in a single tx. reference explaining why we cannot embed alter type in tx: https://stackoverflow.com/questions/53149484/error-alter-type-add-cannot-run-inside-a-transaction-block await queryInterface.sequelize.query( - `ALTER TYPE "enum_ClockRecords_sourceTable" ADD VALUE IF NOT EXISTS 'Playlist'` + ` + ALTER TYPE "enum_ClockRecords_sourceTable" RENAME TO "enum_ClockRecords_sourceTable_old"; + CREATE TYPE "enum_ClockRecords_sourceTable" AS ENUM('AudiusUser', 'Track', 'File', 'Playlist'); + ALTER TABLE "ClockRecords" ALTER COLUMN "sourceTable" TYPE "enum_ClockRecords_sourceTable" USING "sourceTable"::text::"enum_ClockRecords_sourceTable"; + DROP TYPE "enum_ClockRecords_sourceTable_old"; + `, + { transaction } ) } From c67c6df3d20031dc3b436a09f7f833b1be71b511 Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Thu, 16 Jun 2022 17:09:00 +0000 Subject: [PATCH 013/101] Minor update --- ...0220427014326-create-playlist-table-and-clock-record-type.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/creator-node/sequelize/migrations/20220427014326-create-playlist-table-and-clock-record-type.js b/creator-node/sequelize/migrations/20220427014326-create-playlist-table-and-clock-record-type.js index eedaeb651e2..535ce30e7a1 100644 --- a/creator-node/sequelize/migrations/20220427014326-create-playlist-table-and-clock-record-type.js +++ b/creator-node/sequelize/migrations/20220427014326-create-playlist-table-and-clock-record-type.js @@ -42,8 +42,6 @@ async function addPlaylistToClockRecordSourceTables( queryInterface, transaction ) { - // Note that this cannot be run inside a transaction block - // TODO: Instead of alter type, rename and create new type in order to preserve atomicity in a single tx. reference explaining why we cannot embed alter type in tx: https://stackoverflow.com/questions/53149484/error-alter-type-add-cannot-run-inside-a-transaction-block await queryInterface.sequelize.query( ` ALTER TYPE "enum_ClockRecords_sourceTable" RENAME TO "enum_ClockRecords_sourceTable_old"; From 1c1750181a241bf6988fa219aad8109f2b4b98b0 Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Thu, 16 Jun 2022 17:37:05 +0000 Subject: [PATCH 014/101] Adding test case --- creator-node/test/playlists.test.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/creator-node/test/playlists.test.js b/creator-node/test/playlists.test.js index 36e48aa8e9b..8c07798e4ce 100644 --- a/creator-node/test/playlists.test.js +++ b/creator-node/test/playlists.test.js @@ -97,6 +97,8 @@ describe('Test Playlists', function () { assert.ok(file) }) + // TODO TEST CASE - Add image files to playlist metadata and mock flow + confirm behavior (missing image dirCID for example) + it('successfully completes Audius playlist creation (POST /playlists/metadata -> POST /playlists)', async function () { const metadata = { test: 'field1' } const resp = await request(app) From 771f4c7ea2baeb9cd3e6f9eb07ed9441dece74a4 Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Thu, 16 Jun 2022 20:47:03 +0000 Subject: [PATCH 015/101] Added imageDIR test --- creator-node/test/playlists.test.js | 71 ++++++++++++++++++++++++++++- 1 file changed, 69 insertions(+), 2 deletions(-) diff --git a/creator-node/test/playlists.test.js b/creator-node/test/playlists.test.js index 8c07798e4ce..9ea8545f09b 100644 --- a/creator-node/test/playlists.test.js +++ b/creator-node/test/playlists.test.js @@ -1,4 +1,5 @@ const request = require('supertest') +const path = require('path') const assert = require('assert') const fs = require('fs') @@ -97,8 +98,6 @@ describe('Test Playlists', function () { assert.ok(file) }) - // TODO TEST CASE - Add image files to playlist metadata and mock flow + confirm behavior (missing image dirCID for example) - it('successfully completes Audius playlist creation (POST /playlists/metadata -> POST /playlists)', async function () { const metadata = { test: 'field1' } const resp = await request(app) @@ -141,6 +140,74 @@ describe('Test Playlists', function () { .expect(200) }) + it('successfully completes Audius playlist creation with imageDirCID', async function () { + const testPicture = path.resolve(__dirname, 'testTrackWrongFormat.jpg') + const file = fs.readFileSync(testPicture) + // Upload test cover image + const resp = await request(app) + .post('/image_upload') + .attach('file', file, { filename: 'abel.jpg' }) + .field('square', true) + .set('Content-Type', 'multipart/form-data') + .set('X-Session-ID', session.sessionToken) + .set('User-Id', session.userId) + + const imageDirCID = resp.body.data.dirCID + const playlistImagePath = DiskManager.computeFilePath(imageDirCID) + assert.ok(fs.existsSync(playlistImagePath)) + const fileRecord = await models.File.findOne({ + where: { + multihash: imageDirCID, + storagePath: playlistImagePath, + type: 'dir' + } + }) + assert.ok(fileRecord) + const metadata = { + test: 'field1', + playlist_image_sizes_multihash: imageDirCID + } + + // Upload playlist metadata with valid image dirCID + const resp2 = await request(app) + .post('/playlists/metadata') + .set('X-Session-ID', session.sessionToken) + .set('User-Id', session.userId) + .send({ metadata }) + .expect(200) + + // check that the metadata file was written to storagePath under its multihash + const metadataPath = DiskManager.computeFilePath( + resp2.body.data.metadataMultihash + ) + assert.ok(fs.existsSync(metadataPath)) + + // check that the metadata file contents match the metadata specified + let metadataFileData = fs.readFileSync(metadataPath, 'utf-8') + metadataFileData = sortKeys(JSON.parse(metadataFileData)) + assert.deepStrictEqual(metadataFileData, metadata) + const metadataFileRecord = await models.File.findOne({ + where: { + multihash: resp2.body.data.metadataMultihash, + storagePath: metadataPath, + type: 'metadata' + } + }) + assert.ok(metadataFileRecord) + + // Associate playlist object with blockchain ID + await request(app) + .post('/playlists') + .set('X-Session-ID', session.sessionToken) + .set('User-Id', session.userId) + .send({ + blockchainId: 1, + blockNumber: 10, + metadataFileUUID: resp2.body.data.metadataFileUUID + }) + .expect(200) + }) + it('successfully completes Audius playlist creation when retrying an existing block number and original metadata', async function () { const metadata = { test: 'field1' } From b2cbd337da9ccba869732f50193b57aa16d5272c Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Thu, 23 Jun 2022 02:13:21 +0000 Subject: [PATCH 016/101] Removing try/catch --- ...-create-playlist-table-and-clock-record-type.js | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/creator-node/sequelize/migrations/20220427014326-create-playlist-table-and-clock-record-type.js b/creator-node/sequelize/migrations/20220427014326-create-playlist-table-and-clock-record-type.js index 535ce30e7a1..811a4cc3cf3 100644 --- a/creator-node/sequelize/migrations/20220427014326-create-playlist-table-and-clock-record-type.js +++ b/creator-node/sequelize/migrations/20220427014326-create-playlist-table-and-clock-record-type.js @@ -78,15 +78,11 @@ async function addCompositeUniqueConstraints(queryInterface, transaction) { } async function removeCompositeUniqueConstraints(queryInterface, transaction) { - try { - await queryInterface.removeConstraint( - tableName, - 'Playlist_unique_(blockchainId,clock)', - { transaction } - ) - } catch (e) { - console.log(e) - } + await queryInterface.removeConstraint( + tableName, + 'Playlist_unique_(blockchainId,clock)', + { transaction } + ) } async function createPlaylistTable(queryInterface, Sequelize, transaction) { From b02bcca03a54f3419e0e93424a518229d2708ce1 Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Thu, 23 Jun 2022 16:13:18 +0000 Subject: [PATCH 017/101] Updated migration --- ...te-playlist-table-and-clock-record-type.js | 40 ++++++------------- 1 file changed, 13 insertions(+), 27 deletions(-) diff --git a/creator-node/sequelize/migrations/20220427014326-create-playlist-table-and-clock-record-type.js b/creator-node/sequelize/migrations/20220427014326-create-playlist-table-and-clock-record-type.js index 811a4cc3cf3..dea84afcd8f 100644 --- a/creator-node/sequelize/migrations/20220427014326-create-playlist-table-and-clock-record-type.js +++ b/creator-node/sequelize/migrations/20220427014326-create-playlist-table-and-clock-record-type.js @@ -22,22 +22,23 @@ module.exports = { await transaction.commit() }, + /* + In the down migration we remove compose unique constraint and playlist table but do not remove the altered enum. + This is in the case that there are existing values in the ClockRecords table pointing to the playlist enum that would have to be either + modified (removes context) or dropped (adds gaps to ClockRecord history) + */ down: async (queryInterface, Sequelize) => { - try { - const transaction = await queryInterface.sequelize.transaction() - await removeCompositeUniqueConstraints(queryInterface, transaction) - await removePlaylistFromClockRecordSourceTables( - queryInterface, - transaction - ) - await dropPlaylistTable(queryInterface, transaction) - await transaction.commit() - } catch (e) { - console.log(e) - } + const transaction = await queryInterface.sequelize.transaction() + await removeCompositeUniqueConstraints(queryInterface, transaction) + await dropPlaylistTable(queryInterface, transaction) + await transaction.commit() } } +/* +Unorthodox approach to modify enum in a transaction - reference notes here on why this is necessary https://www.postgresql.org/docs/11/sql-altertype.html. +It is not possible to use the ALTER TYPE on enum type within a transaction in postgres 11.1 (current version). +*/ async function addPlaylistToClockRecordSourceTables( queryInterface, transaction @@ -53,21 +54,6 @@ async function addPlaylistToClockRecordSourceTables( ) } -async function removePlaylistFromClockRecordSourceTables( - queryInterface, - transaction -) { - await queryInterface.sequelize.query( - ` - ALTER TYPE "enum_ClockRecords_sourceTable" RENAME TO "enum_ClockRecords_sourceTable_old"; - CREATE TYPE "enum_ClockRecords_sourceTable" AS ENUM('AudiusUser', 'Track', 'File'); - ALTER TABLE "ClockRecords" ALTER COLUMN "sourceTable" TYPE "enum_ClockRecords_sourceTable" USING "sourceTable"::text::"enum_ClockRecords_sourceTable"; - DROP TYPE "enum_ClockRecords_sourceTable_old"; - `, - { transaction } - ) -} - async function addCompositeUniqueConstraints(queryInterface, transaction) { await queryInterface.addConstraint(tableName, { type: 'UNIQUE', From 52568fa6719001f1621cb08bc7b017461b228cc2 Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Thu, 23 Jun 2022 17:07:41 +0000 Subject: [PATCH 018/101] More details added to migration --- ...reate-playlist-table-and-clock-record-type.js | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/creator-node/sequelize/migrations/20220427014326-create-playlist-table-and-clock-record-type.js b/creator-node/sequelize/migrations/20220427014326-create-playlist-table-and-clock-record-type.js index dea84afcd8f..05063b2b1eb 100644 --- a/creator-node/sequelize/migrations/20220427014326-create-playlist-table-and-clock-record-type.js +++ b/creator-node/sequelize/migrations/20220427014326-create-playlist-table-and-clock-record-type.js @@ -27,7 +27,7 @@ module.exports = { This is in the case that there are existing values in the ClockRecords table pointing to the playlist enum that would have to be either modified (removes context) or dropped (adds gaps to ClockRecord history) */ - down: async (queryInterface, Sequelize) => { + down: async (queryInterface) => { const transaction = await queryInterface.sequelize.transaction() await removeCompositeUniqueConstraints(queryInterface, transaction) await dropPlaylistTable(queryInterface, transaction) @@ -38,6 +38,20 @@ module.exports = { /* Unorthodox approach to modify enum in a transaction - reference notes here on why this is necessary https://www.postgresql.org/docs/11/sql-altertype.html. It is not possible to use the ALTER TYPE on enum type within a transaction in postgres 11.1 (current version). + +The name of the enum type (generated in sequelize) was determined with the following query: + +audius_creator_node=# SELECT pg_type.typname AS enum_type, pg_enum.enumlabel AS enum_label FROM pg_type JOIN pg_enum ON pg_enum.enumtypid = pg_type.oid; + enum_type | enum_label +-------------------------------+------------ + enum_ContentBlacklists_type | TRACK + enum_ContentBlacklists_type | USER + enum_ContentBlacklists_type | CID + enum_ClockRecords_sourceTable | AudiusUser + enum_ClockRecords_sourceTable | Track + enum_ClockRecords_sourceTable | File + enum_ClockRecords_sourceTable | Playlist +(7 rows) */ async function addPlaylistToClockRecordSourceTables( queryInterface, From 55179f763ac44e3ca3f38d0e0c161a2c794e09e3 Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Thu, 23 Jun 2022 21:36:41 +0000 Subject: [PATCH 019/101] SENDING TXS NOW FUNCTIONING - cleanup required but all good --- libs/data-contracts/ABIs/AudiusData.json | 174 ++++++++++++++++++ libs/data-contracts/signatureSchemas.ts | 50 ++++- .../services/ABIDecoder/AudiusABIDecoder.ts | 3 +- .../services/dataContracts/AudiusContracts.ts | 19 +- .../dataContracts/AudiusDataClient.ts | 45 +++++ 5 files changed, 286 insertions(+), 5 deletions(-) create mode 100644 libs/data-contracts/ABIs/AudiusData.json create mode 100644 libs/src/services/dataContracts/AudiusDataClient.ts diff --git a/libs/data-contracts/ABIs/AudiusData.json b/libs/data-contracts/ABIs/AudiusData.json new file mode 100644 index 00000000000..212b135f2fc --- /dev/null +++ b/libs/data-contracts/ABIs/AudiusData.json @@ -0,0 +1,174 @@ +{ + "contractName": "AudiusData", + "abi": [ + { + "constant": true, + "inputs": [ + { + "name": "", + "type": "bytes32" + } + ], + "name": "usedSignatures", + "outputs": [ + { + "name": "", + "type": "bool" + } + ], + "payable": false, + "stateMutability": "view", + "type": "function" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": false, + "name": "_userId", + "type": "uint256" + }, + { + "indexed": false, + "name": "_signer", + "type": "address" + }, + { + "indexed": false, + "name": "_entityType", + "type": "string" + }, + { + "indexed": false, + "name": "_entityId", + "type": "uint256" + }, + { + "indexed": false, + "name": "_metadata", + "type": "string" + }, + { + "indexed": false, + "name": "_action", + "type": "string" + } + ], + "name": "ManageEntity", + "type": "event" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": false, + "name": "_userId", + "type": "uint256" + }, + { + "indexed": false, + "name": "_isVerified", + "type": "bool" + } + ], + "name": "ManageIsVerified", + "type": "event" + }, + { + "constant": false, + "inputs": [ + { + "name": "name", + "type": "string" + }, + { + "name": "version", + "type": "string" + }, + { + "name": "chainId", + "type": "uint256" + } + ], + "name": "initialize", + "outputs": [], + "payable": false, + "stateMutability": "nonpayable", + "type": "function" + }, + { + "constant": false, + "inputs": [ + { + "name": "_verifierAddress", + "type": "address" + }, + { + "name": "_networkId", + "type": "uint256" + } + ], + "name": "initialize", + "outputs": [], + "payable": false, + "stateMutability": "nonpayable", + "type": "function" + }, + { + "constant": false, + "inputs": [ + { + "name": "_userId", + "type": "uint256" + }, + { + "name": "_entityType", + "type": "string" + }, + { + "name": "_entityId", + "type": "uint256" + }, + { + "name": "_action", + "type": "string" + }, + { + "name": "_metadata", + "type": "string" + }, + { + "name": "_nonce", + "type": "bytes32" + }, + { + "name": "_subjectSig", + "type": "bytes" + } + ], + "name": "manageEntity", + "outputs": [], + "payable": false, + "stateMutability": "nonpayable", + "type": "function" + }, + { + "constant": false, + "inputs": [ + { + "name": "_userId", + "type": "uint256" + }, + { + "name": "_isVerified", + "type": "bool" + } + ], + "name": "manageIsVerified", + "outputs": [], + "payable": false, + "stateMutability": "nonpayable", + "type": "function" + } + ] + } \ No newline at end of file diff --git a/libs/data-contracts/signatureSchemas.ts b/libs/data-contracts/signatureSchemas.ts index fa6c64c5f7f..9462effc106 100644 --- a/libs/data-contracts/signatureSchemas.ts +++ b/libs/data-contracts/signatureSchemas.ts @@ -64,6 +64,10 @@ const getUserReplicaSetManagerDomain: DomainFn = (chainId, contractAddress) => { ) } +const getAudiusDataDomain: DomainFn = (chainId, contractAddress) => { + return getDomainData("Audius Data", "1", chainId, contractAddress) +} + export const domains = { getSocialFeatureFactoryDomain, getUserFactoryDomain, @@ -71,7 +75,8 @@ export const domains = { getPlaylistFactoryDomain, getUserLibraryFactoryDomain, getIPLDBlacklistFactoryDomain, - getUserReplicaSetManagerDomain + getUserReplicaSetManagerDomain, + getAudiusDataDomain } /* contract signing domain */ @@ -258,6 +263,15 @@ const updateReplicaSet = [ { name: 'nonce', type: 'bytes32' } ] +const manageEntity = [ + { name: 'userId', type: 'uint'}, + { name: 'entityType', type: 'string'}, + { name: 'entityId', type: 'uint'}, + { name: 'action', type: 'string'}, + { name: 'metadata', type: 'string'}, + { name: 'nonce', type: 'bytes32'}, +] + export const schemas = { domain, addUserRequest, @@ -289,7 +303,8 @@ export const schemas = { deletePlaylistSaveRequest, addIPLDBlacklist, proposeAddOrUpdateContentNode, - updateReplicaSet + updateReplicaSet, + manageEntity } type MessageSchema = readonly EIP712TypeProperty[] @@ -1145,6 +1160,34 @@ const getUpdateReplicaSetRequestData = ( ) } +const getManageEntityData = ( + chainId: number, + contractAddress: string, + userId: number, + entityType: string, + entityId: number, + action: string, + metadata: string, + nonce: string +) => { + const message = { + userId, + entityType, + entityId, + action, + metadata, + nonce + } + return getRequestData( + domains.getAudiusDataDomain, + chainId, + contractAddress, + 'ManageEntity', + schemas.manageEntity, + message + ) +} + export const generators = { getUpdateUserMultihashRequestData, getAddUserRequestData, @@ -1181,7 +1224,8 @@ export const generators = { getUpdatePlaylistDescriptionRequestData, addIPLDToBlacklistRequestData, getProposeAddOrUpdateContentNodeRequestData, - getUpdateReplicaSetRequestData + getUpdateReplicaSetRequestData, + getManageEntityData } type NodeCrypto = { randomBytes: (size: number) => Buffer } diff --git a/libs/src/services/ABIDecoder/AudiusABIDecoder.ts b/libs/src/services/ABIDecoder/AudiusABIDecoder.ts index 7e9b95d1543..28e6b831aa5 100644 --- a/libs/src/services/ABIDecoder/AudiusABIDecoder.ts +++ b/libs/src/services/ABIDecoder/AudiusABIDecoder.ts @@ -19,6 +19,7 @@ loadABI('SocialFeatureFactory.json') loadABI('PlaylistFactory.json') loadABI('UserLibraryFactory.json') loadABI('UserReplicaSetManager.json') +loadABI('AudiusData.json') // eslint-disable-next-line @typescript-eslint/no-extraneous-class -- should just use esm export class AudiusABIDecoder { @@ -32,7 +33,7 @@ export class AudiusABIDecoder { // namespace of functions) const abi = abiMap[contractName] if (!abi) { - throw new Error('Unrecognized contract name') + throw new Error(`Unrecognized contract name ${contractName}`) } let foundFunction: AbiItem | undefined diff --git a/libs/src/services/dataContracts/AudiusContracts.ts b/libs/src/services/dataContracts/AudiusContracts.ts index 8c170e02dd6..6b01ad4d604 100644 --- a/libs/src/services/dataContracts/AudiusContracts.ts +++ b/libs/src/services/dataContracts/AudiusContracts.ts @@ -9,6 +9,7 @@ import { PlaylistFactoryClient } from './PlaylistFactoryClient' import { UserLibraryFactoryClient } from './UserLibraryFactoryClient' import { IPLDBlacklistFactoryClient } from './IPLDBlacklistFactoryClient' import { UserReplicaSetManagerClient } from './UserReplicaSetManagerClient' +import { AudiusDataClient } from './AudiusDataClient' import type { Web3Manager } from '../web3Manager' import type { ContractClient } from '../contracts/ContractClient' @@ -32,6 +33,10 @@ const IPLDBlacklistFactoryABI = Utils.importDataContractABI( const UserReplicaSetManagerABI = Utils.importDataContractABI( 'UserReplicaSetManager.json' ).abi +const AudiusDataABI = Utils.importDataContractABI( + 'AudiusData.json' +).abi +console.log(AudiusDataABI) // define contract registry keys const UserFactoryRegistryKey = 'UserFactory' @@ -54,6 +59,7 @@ export class AudiusContracts { PlaylistFactoryClient: PlaylistFactoryClient UserLibraryFactoryClient: UserLibraryFactoryClient IPLDBlacklistFactoryClient: IPLDBlacklistFactoryClient + AudiusDataClient: AudiusDataClient contractClients: ContractClient[] UserReplicaSetManagerClient: UserReplicaSetManagerClient | undefined | null contracts: Record | undefined @@ -126,17 +132,28 @@ export class AudiusContracts { this.logger ) + this.AudiusDataClient = new AudiusDataClient( + this.web3Manager, + AudiusDataABI, + "AudiusData", + this.getRegistryAddressForContract, + this.logger, + "0xDDb64fE46a91D46ee29420539FC25FD07c5FEa3E" // TODO: Config var + ) + this.contractClients = [ this.UserFactoryClient, this.TrackFactoryClient, this.SocialFeatureFactoryClient, this.PlaylistFactoryClient, this.UserLibraryFactoryClient, - this.IPLDBlacklistFactoryClient + this.IPLDBlacklistFactoryClient, + this.AudiusDataClient ] } async init() { + console.log('Initializing..2.') if (this.isServer) { await Promise.all( this.contractClients.map(async (client) => await client.init()) diff --git a/libs/src/services/dataContracts/AudiusDataClient.ts b/libs/src/services/dataContracts/AudiusDataClient.ts new file mode 100644 index 00000000000..7bba342fcd4 --- /dev/null +++ b/libs/src/services/dataContracts/AudiusDataClient.ts @@ -0,0 +1,45 @@ +import { ContractClient } from '../contracts/ContractClient' +import * as signatureSchemas from '../../../data-contracts/signatureSchemas' +import type { Web3Manager } from '../web3Manager' + +export class AudiusDataClient extends ContractClient { + override web3Manager!: Web3Manager + + async manageEntity( + userId: number, + entityType: string, + entityId: number, + action: string, + metadata: string, + ) { + const nonce = signatureSchemas.getNonce() + const chainId = await this.getEthNetId() + const contractAddress = await this.getAddress() + console.log(`contract ${contractAddress} - nonce=${nonce} chainId=${chainId}`) + console.log(`- manageEntity ${userId}, ${entityType}, ${entityId}, ${action}, ${metadata}`) + const signatureData = signatureSchemas.generators.getManageEntityData(chainId, contractAddress, userId, entityType, entityId, action, metadata, nonce) + console.log(`- manageEntity ${JSON.stringify(signatureData)}`) + const sig = await this.web3Manager.signTypedData(signatureData) + console.log(`- manageEntity ${sig}`) + const method = await this.getMethod( + 'manageEntity', + userId, + entityType, + entityId, + action, + metadata, + nonce, + sig + ) + console.log(method) + + const tx = await this.web3Manager.sendTransaction( + method, + this.contractRegistryKey, + contractAddress + ) + return { + txReceipt: tx + } + } +} \ No newline at end of file From e4eee727d2ecf50a2281b7c622d15b3558a3c076 Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Fri, 24 Jun 2022 16:30:49 +0000 Subject: [PATCH 020/101] Service level changes --- libs/src/services/dataContracts/AudiusContracts.ts | 1 - libs/src/services/dataContracts/AudiusDataClient.ts | 4 ---- 2 files changed, 5 deletions(-) diff --git a/libs/src/services/dataContracts/AudiusContracts.ts b/libs/src/services/dataContracts/AudiusContracts.ts index 6b01ad4d604..295356a9f03 100644 --- a/libs/src/services/dataContracts/AudiusContracts.ts +++ b/libs/src/services/dataContracts/AudiusContracts.ts @@ -153,7 +153,6 @@ export class AudiusContracts { } async init() { - console.log('Initializing..2.') if (this.isServer) { await Promise.all( this.contractClients.map(async (client) => await client.init()) diff --git a/libs/src/services/dataContracts/AudiusDataClient.ts b/libs/src/services/dataContracts/AudiusDataClient.ts index 7bba342fcd4..79d1290a6f4 100644 --- a/libs/src/services/dataContracts/AudiusDataClient.ts +++ b/libs/src/services/dataContracts/AudiusDataClient.ts @@ -15,12 +15,8 @@ export class AudiusDataClient extends ContractClient { const nonce = signatureSchemas.getNonce() const chainId = await this.getEthNetId() const contractAddress = await this.getAddress() - console.log(`contract ${contractAddress} - nonce=${nonce} chainId=${chainId}`) - console.log(`- manageEntity ${userId}, ${entityType}, ${entityId}, ${action}, ${metadata}`) const signatureData = signatureSchemas.generators.getManageEntityData(chainId, contractAddress, userId, entityType, entityId, action, metadata, nonce) - console.log(`- manageEntity ${JSON.stringify(signatureData)}`) const sig = await this.web3Manager.signTypedData(signatureData) - console.log(`- manageEntity ${sig}`) const method = await this.getMethod( 'manageEntity', userId, From 3525c96620ebf296cae63afceea1fd873366a204 Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Mon, 27 Jun 2022 16:26:44 +0000 Subject: [PATCH 021/101] Minor update --- libs/src/services/dataContracts/AudiusContracts.ts | 2 +- libs/src/services/dataContracts/AudiusDataClient.ts | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/libs/src/services/dataContracts/AudiusContracts.ts b/libs/src/services/dataContracts/AudiusContracts.ts index 295356a9f03..e8e128cbba3 100644 --- a/libs/src/services/dataContracts/AudiusContracts.ts +++ b/libs/src/services/dataContracts/AudiusContracts.ts @@ -138,7 +138,7 @@ export class AudiusContracts { "AudiusData", this.getRegistryAddressForContract, this.logger, - "0xDDb64fE46a91D46ee29420539FC25FD07c5FEa3E" // TODO: Config var + "0xaf5C4C6C7920B4883bC6252e9d9B8fE27187Cf68" // TODO: Config var ) this.contractClients = [ diff --git a/libs/src/services/dataContracts/AudiusDataClient.ts b/libs/src/services/dataContracts/AudiusDataClient.ts index 79d1290a6f4..f142141b4dc 100644 --- a/libs/src/services/dataContracts/AudiusDataClient.ts +++ b/libs/src/services/dataContracts/AudiusDataClient.ts @@ -27,8 +27,6 @@ export class AudiusDataClient extends ContractClient { nonce, sig ) - console.log(method) - const tx = await this.web3Manager.sendTransaction( method, this.contractRegistryKey, From 4beae402445dfab4c1861b88ec7edc3d1e5cd4cf Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Mon, 27 Jun 2022 19:37:45 +0000 Subject: [PATCH 022/101] CN API for metadata --- libs/src/services/creatorNode/CreatorNode.ts | 24 ++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/libs/src/services/creatorNode/CreatorNode.ts b/libs/src/services/creatorNode/CreatorNode.ts index e28ec6d343d..330e8eefd3d 100644 --- a/libs/src/services/creatorNode/CreatorNode.ts +++ b/libs/src/services/creatorNode/CreatorNode.ts @@ -25,6 +25,10 @@ type Metadata = { cover_art_sizes: string } +type PlaylistMetadata = { + playlist_contents: unknown +} + type ProgressCB = (loaded: number, total: number) => void type MonitoringCallbacks = { @@ -402,6 +406,26 @@ export class CreatorNode { return body } + /** + * Uploads playlist metadata to a creator node + * source file must be provided (returned from uploading track content). + * @param metadata + */ + async uploadPlaylistMetadata(metadata: PlaylistMetadata) { + // TODO: Schema validation flow + const { data: body } = await this._makeRequest( + { + url: '/playlists/metadata', + method: 'post', + data: { + metadata + } + }, + true + ) + return body + } + /** * Creates a track on the content node, associating track id with file content * @param audiusTrackId returned by track creation on-blockchain From 1d190f8bf1f8bcc9b55b479d8f9b4db3ddf39aa5 Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Tue, 28 Jun 2022 20:08:24 +0000 Subject: [PATCH 023/101] API write functioning fully w/libs, now moving to TS etc. --- libs/src/api/playlist.js | 42 +++++++++++++++++++ libs/src/services/creatorNode/CreatorNode.ts | 26 ++++++++++++ .../services/dataContracts/AudiusContracts.ts | 2 +- .../dataContracts/AudiusDataClient.ts | 1 + 4 files changed, 70 insertions(+), 1 deletion(-) diff --git a/libs/src/api/playlist.js b/libs/src/api/playlist.js index f41809ef3c4..3cf33bf14c8 100644 --- a/libs/src/api/playlist.js +++ b/libs/src/api/playlist.js @@ -73,6 +73,48 @@ class Playlists extends Base { /* ------- SETTERS ------- */ + async createPlaylist2 (playlistName, trackIds, coverPhoto, description, isAlbum) { + // TODO: Abstract determining a valid ID + const ownerId = this.userStateManager.getCurrentUserId() + const action = 'Create' + const entityType = 'Playlist' + const entityId = 10 + this.REQUIRES(Services.CREATOR_NODE) + console.log(`Playlist2 | New create playlist function invoked! ${ownerId}`) + console.log(coverPhoto) + const dirCID = await this.uploadPlaylistCoverPhoto(coverPhoto, true) + console.log(`Playlist2 | Uploaded image w/dirCID = ${dirCID}`) + const metadata = { + action, + entity_type: entityType, + playlist_id: entityId, + playlist_contents: trackIds, + playlist_name: playlistName, + playlist_image_sizes_multihash: dirCID, + description, + isAlbum + } + console.log(`Playlist2 | New create playlist function metadata! ${ownerId}, ${JSON.stringify(metadata)}`) + const { metadataMultihash, metadataFileUUID } = await this.creatorNode.uploadPlaylistMetadata(metadata) + console.log(`Playlist2 | New playlist metadata ${JSON.stringify(metadataMultihash)}`) + const resp = await this.contracts.AudiusDataClient.manageEntity( + ownerId, + entityType, + entityId, + action, + metadataMultihash + ) + const receipt = resp.txReceipt + console.log(`Playlist2 | ${JSON.stringify(receipt)} ${metadataFileUUID}, associating next...`) + const blocknumber = receipt.blockNumber + const associateResponse = await this.creatorNode.associatePlaylistMetadata( + entityId, + metadataFileUUID, + blocknumber + ) + console.log(`Playlist2 | Associated! ${JSON.stringify(associateResponse)}`) + } + /** * Creates a new playlist * @param {number} userId diff --git a/libs/src/services/creatorNode/CreatorNode.ts b/libs/src/services/creatorNode/CreatorNode.ts index 330e8eefd3d..3e43c50e604 100644 --- a/libs/src/services/creatorNode/CreatorNode.ts +++ b/libs/src/services/creatorNode/CreatorNode.ts @@ -426,6 +426,32 @@ export class CreatorNode { return body } + /** + * Associate an uploaded playlist metadata file with a blockchainId + * @param blockchainId - Valid ID assigned to playlist + * @param metadataFileUUID unique ID for metadata playlist + * @param blockNumber + */ + async associatePlaylistMetadata( + blockchainId: number, + metadataFileUUID: string, + blockNumber: number + ) { + const { data: body } = await this._makeRequest( + { + url: '/playlists', + method: 'post', + data: { + blockchainId, + metadataFileUUID, + blockNumber + } + }, + true + ) + return body + } + /** * Creates a track on the content node, associating track id with file content * @param audiusTrackId returned by track creation on-blockchain diff --git a/libs/src/services/dataContracts/AudiusContracts.ts b/libs/src/services/dataContracts/AudiusContracts.ts index e8e128cbba3..295356a9f03 100644 --- a/libs/src/services/dataContracts/AudiusContracts.ts +++ b/libs/src/services/dataContracts/AudiusContracts.ts @@ -138,7 +138,7 @@ export class AudiusContracts { "AudiusData", this.getRegistryAddressForContract, this.logger, - "0xaf5C4C6C7920B4883bC6252e9d9B8fE27187Cf68" // TODO: Config var + "0xDDb64fE46a91D46ee29420539FC25FD07c5FEa3E" // TODO: Config var ) this.contractClients = [ diff --git a/libs/src/services/dataContracts/AudiusDataClient.ts b/libs/src/services/dataContracts/AudiusDataClient.ts index f142141b4dc..154c02fe818 100644 --- a/libs/src/services/dataContracts/AudiusDataClient.ts +++ b/libs/src/services/dataContracts/AudiusDataClient.ts @@ -32,6 +32,7 @@ export class AudiusDataClient extends ContractClient { this.contractRegistryKey, contractAddress ) + console.log(tx) return { txReceipt: tx } From d9bc483a49499af02b22ca261bd1c61d36f29465 Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Wed, 29 Jun 2022 20:48:35 +0000 Subject: [PATCH 024/101] ID selection functioning --- libs/src/api/playlist.js | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/libs/src/api/playlist.js b/libs/src/api/playlist.js index 3cf33bf14c8..bed78239bad 100644 --- a/libs/src/api/playlist.js +++ b/libs/src/api/playlist.js @@ -24,6 +24,7 @@ class Playlists extends Base { this.addPlaylistSave = this.addPlaylistSave.bind(this) this.deletePlaylistSave = this.deletePlaylistSave.bind(this) this.deletePlaylist = this.deletePlaylist.bind(this) + this.getValidPlaylistId = this.getValidPlaylistId.bind(this) } /* ------- GETTERS ------- */ @@ -71,16 +72,45 @@ class Playlists extends Base { return this.discoveryProvider.getSavedAlbums(limit, offset, withUsers) } + /** + * Calculate an unoccupied playlist ID + * Maximum value is UINT MAX + */ + async getValidPlaylistId () { + let playlistId = 10 + let validIdFound = false + while (!validIdFound) { + const resp = await this.getPlaylists(1, 0, [playlistId]) + if (resp.length !== 0) { + playlistId = Math.floor( + Math.random() * 9007199254740991 + ) + } else { + validIdFound = true + } + } + console.log(`Playlist2 | Returning playlistID=${playlistId}`) + return playlistId + } + /* ------- SETTERS ------- */ + /** + * Create a playlist using updated data contracts flow + * @param {*} playlistName + * @param {*} trackIds + * @param {*} coverPhoto + * @param {*} description + * @param {*} isAlbum + */ async createPlaylist2 (playlistName, trackIds, coverPhoto, description, isAlbum) { // TODO: Abstract determining a valid ID const ownerId = this.userStateManager.getCurrentUserId() + console.log(`Playlist2 | New create playlist function invoked! ${ownerId}`) const action = 'Create' const entityType = 'Playlist' - const entityId = 10 + const entityId = await this.getValidPlaylistId() this.REQUIRES(Services.CREATOR_NODE) - console.log(`Playlist2 | New create playlist function invoked! ${ownerId}`) console.log(coverPhoto) const dirCID = await this.uploadPlaylistCoverPhoto(coverPhoto, true) console.log(`Playlist2 | Uploaded image w/dirCID = ${dirCID}`) From 6a40e4dabf2595b883a42ea2e5b4cb065601847f Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Thu, 30 Jun 2022 19:05:55 +0000 Subject: [PATCH 025/101] Typescript for new data layer interactions - can be split into multiple files async --- libs/src/api/audiusData.ts | 118 +++++++++++++++++++++++++++++++++++++ libs/src/api/base.d.ts | 4 +- libs/src/api/playlist.js | 71 ---------------------- libs/src/libs.js | 2 + 4 files changed, 122 insertions(+), 73 deletions(-) create mode 100644 libs/src/api/audiusData.ts diff --git a/libs/src/api/audiusData.ts b/libs/src/api/audiusData.ts new file mode 100644 index 00000000000..5f82c0b9acc --- /dev/null +++ b/libs/src/api/audiusData.ts @@ -0,0 +1,118 @@ +import { Base, Services } from './base' + +export class AudiusData extends Base { + constructor(...args: any[]) { + super(...args) + // this.submitReaction = this.submitReaction.bind(this) + } + + /** + * Calculate an unoccupied playlist ID + * Maximum value is UINT MAX + */ + async getValidPlaylistId (): Promise { + let playlistId: number = 10 + let validIdFound: boolean = false + while (!validIdFound) { + const resp = await this.discoveryProvider.getPlaylists(1, 0, [playlistId]) + if (resp.length !== 0) { + playlistId = Math.floor( + Math.random() * 9007199254740991 + ) + } else { + validIdFound = true + } + } + console.log(`Playlist2 | Returning playlistID=${playlistId}`) + return playlistId + } + + /** + * Create a playlist using updated data contracts flow + */ + async createPlaylist({ + playlistName, + trackIds, + description, + isAlbum, + coverArt, + logger = console + }: { + playlistName: string, + trackIds: number[], + description: string, + isAlbum: boolean, + coverArt: any, + logger: any + }): Promise { + try { + const ownerId: number = this.userStateManager.getCurrentUserId() + const action = 'Create' + const entityType = 'Playlist' + const entityId = await this.getValidPlaylistId() + logger.info(`CreatePlaylistData: user=${ownerId}, ${action}, ${entityType} ${playlistName}, album=${isAlbum}, ids=${trackIds}, description=${description} playlistId=${entityId}, coverArt=${coverArt}`) + this.REQUIRES(Services.CREATOR_NODE) + const updatedPlaylistImage = await this.creatorNode.uploadImage( + coverArt, + true // square + ) + const dirCID = updatedPlaylistImage.dirCID + logger.info(`CreatePlaylistData - ${dirCID}`) + const metadata = { + action, + entity_type: entityType, + playlist_id: entityId, + playlist_contents: trackIds, + playlist_name: playlistName, + playlist_image_sizes_multihash: dirCID, + description, + isAlbum + } + const { metadataMultihash, metadataFileUUID } = await this.creatorNode.uploadPlaylistMetadata(metadata) + logger.info(`CreatePlaylistData - metadata: ${metadataMultihash} - ${metadataFileUUID}`) + let resp = await this.manageEntity({ + userId: ownerId, + entityType, + entityId, + action, + metadataMultihash + }) + logger.info(`CreatePlaylistData - ${JSON.stringify(resp)}`) + } catch(e) { + logger.error(`Data create playlist: err ${e}`) + } + } + + /** + * Manage an entity with the updated data contract flow + * Leveraged to manipulate User/Track/Playlist/+ other entities + */ + async manageEntity({ + userId, + entityType, + entityId, + action, + metadataMultihash + }: { + userId: number, + entityType: string, + entityId: number, + action: string, + metadataMultihash: string + }): Promise<{ txReceipt: {}, error: any}> { + let error:string = "" + let resp + try { + resp = await this.contracts.AudiusDataClient.manageEntity( + userId, + entityType, + entityId, + action, + metadataMultihash + ) + } catch(e) { + error = (e as Error).message + } + return { txReceipt: resp.txReceipt, error} + } +} diff --git a/libs/src/api/base.d.ts b/libs/src/api/base.d.ts index e33b1246ee4..67cedbc508a 100644 --- a/libs/src/api/base.d.ts +++ b/libs/src/api/base.d.ts @@ -1,4 +1,4 @@ // Placeholder type for base.js declare const Base: any - -export { Base } +declare const Services: any +export { Base, Services } diff --git a/libs/src/api/playlist.js b/libs/src/api/playlist.js index bed78239bad..40e94a88807 100644 --- a/libs/src/api/playlist.js +++ b/libs/src/api/playlist.js @@ -72,79 +72,8 @@ class Playlists extends Base { return this.discoveryProvider.getSavedAlbums(limit, offset, withUsers) } - /** - * Calculate an unoccupied playlist ID - * Maximum value is UINT MAX - */ - async getValidPlaylistId () { - let playlistId = 10 - let validIdFound = false - while (!validIdFound) { - const resp = await this.getPlaylists(1, 0, [playlistId]) - if (resp.length !== 0) { - playlistId = Math.floor( - Math.random() * 9007199254740991 - ) - } else { - validIdFound = true - } - } - console.log(`Playlist2 | Returning playlistID=${playlistId}`) - return playlistId - } - /* ------- SETTERS ------- */ - /** - * Create a playlist using updated data contracts flow - * @param {*} playlistName - * @param {*} trackIds - * @param {*} coverPhoto - * @param {*} description - * @param {*} isAlbum - */ - async createPlaylist2 (playlistName, trackIds, coverPhoto, description, isAlbum) { - // TODO: Abstract determining a valid ID - const ownerId = this.userStateManager.getCurrentUserId() - console.log(`Playlist2 | New create playlist function invoked! ${ownerId}`) - const action = 'Create' - const entityType = 'Playlist' - const entityId = await this.getValidPlaylistId() - this.REQUIRES(Services.CREATOR_NODE) - console.log(coverPhoto) - const dirCID = await this.uploadPlaylistCoverPhoto(coverPhoto, true) - console.log(`Playlist2 | Uploaded image w/dirCID = ${dirCID}`) - const metadata = { - action, - entity_type: entityType, - playlist_id: entityId, - playlist_contents: trackIds, - playlist_name: playlistName, - playlist_image_sizes_multihash: dirCID, - description, - isAlbum - } - console.log(`Playlist2 | New create playlist function metadata! ${ownerId}, ${JSON.stringify(metadata)}`) - const { metadataMultihash, metadataFileUUID } = await this.creatorNode.uploadPlaylistMetadata(metadata) - console.log(`Playlist2 | New playlist metadata ${JSON.stringify(metadataMultihash)}`) - const resp = await this.contracts.AudiusDataClient.manageEntity( - ownerId, - entityType, - entityId, - action, - metadataMultihash - ) - const receipt = resp.txReceipt - console.log(`Playlist2 | ${JSON.stringify(receipt)} ${metadataFileUUID}, associating next...`) - const blocknumber = receipt.blockNumber - const associateResponse = await this.creatorNode.associatePlaylistMetadata( - entityId, - metadataFileUUID, - blocknumber - ) - console.log(`Playlist2 | Associated! ${JSON.stringify(associateResponse)}`) - } - /** * Creates a new playlist * @param {number} userId diff --git a/libs/src/libs.js b/libs/src/libs.js index 320309691ad..6a0cff850b8 100644 --- a/libs/src/libs.js +++ b/libs/src/libs.js @@ -34,6 +34,7 @@ const { RewardsAttester } = require('./services/solanaWeb3Manager/rewardsAttester') const { Reactions } = require('./api/reactions') +const { AudiusData } = require('./api/audiusData') class AudiusLibs { /** @@ -557,6 +558,7 @@ class AudiusLibs { this.File = new File(this.User, ...services) this.Rewards = new Rewards(this.ServiceProvider, ...services) this.Reactions = new Reactions(...services) + this.AudiusData = new AudiusData(...services) } } From 462c8ede161498b98f822d8c208f9f8a2944ea83 Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Fri, 1 Jul 2022 03:13:47 +0000 Subject: [PATCH 026/101] functioning w/cleanup required - audius data txs ingested sucessfully --- discovery-provider/src/app.py | 12 ++++++++++++ discovery-provider/src/tasks/index.py | 7 +++++++ discovery-provider/src/utils/constants.py | 1 + libs/src/api/audiusData.ts | 3 ++- libs/src/api/playlist.js | 1 - 5 files changed, 22 insertions(+), 2 deletions(-) diff --git a/discovery-provider/src/app.py b/discovery-provider/src/app.py index 7c360db9566..809962b24cf 100644 --- a/discovery-provider/src/app.py +++ b/discovery-provider/src/app.py @@ -64,6 +64,7 @@ user_library_factory = None ipld_blacklist_factory = None user_replica_set_manager = None +audius_data = None contract_addresses: Dict[str, Any] = defaultdict() logger = logging.getLogger(__name__) @@ -135,6 +136,12 @@ def init_contracts(): abi=abi_values["UserReplicaSetManager"]["abi"], ) + audius_data_address = "0xEC5d4F247aF81A843612eb1371CBCfa88b762119" + audius_data_inst = web3.eth.contract( + address=audius_data_address, + abi=abi_values["AudiusData"]["abi"] + ) + contract_address_dict = { "registry": registry_address, "user_factory": user_factory_address, @@ -144,6 +151,7 @@ def init_contracts(): "user_library_factory": user_library_factory_address, "ipld_blacklist_factory": ipld_blacklist_factory_address, "user_replica_set_manager": user_replica_set_manager_address, + "audius_data": audius_data_address } return ( @@ -155,6 +163,7 @@ def init_contracts(): user_library_factory_inst, ipld_blacklist_factory_inst, user_replica_set_manager_inst, + audius_data_inst, contract_address_dict, ) @@ -188,6 +197,7 @@ def create_celery(test_config=None): global user_library_factory global ipld_blacklist_factory global user_replica_set_manager + global audius_data global contract_addresses # pylint: enable=W0603 @@ -200,8 +210,10 @@ def create_celery(test_config=None): user_library_factory, ipld_blacklist_factory, user_replica_set_manager, + audius_data, contract_addresses, ) = init_contracts() + logger.info(f"contract_addresses_dict {contract_addresses}") return create(test_config, mode="celery") diff --git a/discovery-provider/src/tasks/index.py b/discovery-provider/src/tasks/index.py index b66bfae7efb..ac22e7b130c 100644 --- a/discovery-provider/src/tasks/index.py +++ b/discovery-provider/src/tasks/index.py @@ -72,6 +72,7 @@ PLAYLIST_FACTORY = CONTRACT_TYPES.PLAYLIST_FACTORY.value USER_LIBRARY_FACTORY = CONTRACT_TYPES.USER_LIBRARY_FACTORY.value USER_REPLICA_SET_MANAGER = CONTRACT_TYPES.USER_REPLICA_SET_MANAGER.value +AUDIUS_DATA = CONTRACT_TYPES.AUDIUS_DATA.value USER_FACTORY_CONTRACT_NAME = CONTRACT_NAMES_ON_CHAIN[CONTRACT_TYPES.USER_FACTORY] TRACK_FACTORY_CONTRACT_NAME = CONTRACT_NAMES_ON_CHAIN[CONTRACT_TYPES.TRACK_FACTORY] @@ -436,6 +437,7 @@ def get_contract_type_for_tx(tx_type_to_grouped_lists_map, tx, tx_receipt): tx_target_contract_address = tx["to"] contract_type = None for tx_type in tx_type_to_grouped_lists_map.keys(): + logger.info(f"index.py | checking {tx_type} vs {tx_target_contract_address}") tx_is_type = tx_target_contract_address == get_contract_addresses()[tx_type] if tx_is_type: contract_type = tx_type @@ -444,6 +446,8 @@ def get_contract_type_for_tx(tx_type_to_grouped_lists_map, tx, tx_receipt): f" tx from block - {tx}, receipt - {tx_receipt}" ) break + + logger.info(f"index.py | checking returned {contract_type} vs {tx_target_contract_address}") return contract_type @@ -599,6 +603,7 @@ def index_blocks(self, db, blocks_list): PLAYLIST_FACTORY: [], USER_LIBRARY_FACTORY: [], USER_REPLICA_SET_MANAGER: [], + AUDIUS_DATA: [] } try: """ @@ -703,6 +708,7 @@ def index_blocks(self, db, blocks_list): Add state changes in block to db (users, tracks, etc.) """ process_state_changes_start_time = time.time() + logger.info(f"index.py | processing {txs_grouped_by_type}") # bulk process operations once all tx's for block have been parsed # and get changed entity IDs for cache clearing # after session commit @@ -1119,6 +1125,7 @@ def update_task(self): address=get_contract_addresses()[USER_REPLICA_SET_MANAGER], abi=user_replica_set_manager_abi, ) + logger.info(f"index.py | contract_address {get_contract_addresses()}") update_task.track_contract = track_contract update_task.user_contract = user_contract diff --git a/discovery-provider/src/utils/constants.py b/discovery-provider/src/utils/constants.py index cb6860de02b..ebc69a1c122 100644 --- a/discovery-provider/src/utils/constants.py +++ b/discovery-provider/src/utils/constants.py @@ -1034,6 +1034,7 @@ class CONTRACT_TYPES(Enum): PLAYLIST_FACTORY = "playlist_factory" USER_LIBRARY_FACTORY = "user_library_factory" USER_REPLICA_SET_MANAGER = "user_replica_set_manager" + AUDIUS_DATA = "audius_data" CONTRACT_NAMES_ON_CHAIN = { diff --git a/libs/src/api/audiusData.ts b/libs/src/api/audiusData.ts index 5f82c0b9acc..ee7dd4fb5e5 100644 --- a/libs/src/api/audiusData.ts +++ b/libs/src/api/audiusData.ts @@ -3,7 +3,8 @@ import { Base, Services } from './base' export class AudiusData extends Base { constructor(...args: any[]) { super(...args) - // this.submitReaction = this.submitReaction.bind(this) + this.getValidPlaylistId = this.getValidPlaylistId.bind(this) + this.createPlaylist = this.createPlaylist.bind(this) } /** diff --git a/libs/src/api/playlist.js b/libs/src/api/playlist.js index 40e94a88807..f41809ef3c4 100644 --- a/libs/src/api/playlist.js +++ b/libs/src/api/playlist.js @@ -24,7 +24,6 @@ class Playlists extends Base { this.addPlaylistSave = this.addPlaylistSave.bind(this) this.deletePlaylistSave = this.deletePlaylistSave.bind(this) this.deletePlaylist = this.deletePlaylist.bind(this) - this.getValidPlaylistId = this.getValidPlaylistId.bind(this) } /* ------- GETTERS ------- */ From 75e3b81679d2524e777faf8d194c1b13472e8a3a Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Fri, 1 Jul 2022 20:22:59 +0000 Subject: [PATCH 027/101] Fully functioning createPlaylist in manageEntity --- discovery-provider/src/app.py | 2 +- discovery-provider/src/tasks/audius_data.py | 171 ++++++++++++++++++ discovery-provider/src/tasks/index.py | 36 +++- discovery-provider/src/tasks/metadata.py | 9 + .../src/utils/cid_metadata_client.py | 29 ++- discovery-provider/src/utils/constants.py | 1 + .../src/utils/user_event_constants.py | 8 + libs/src/api/audiusData.ts | 30 ++- 8 files changed, 268 insertions(+), 18 deletions(-) create mode 100644 discovery-provider/src/tasks/audius_data.py diff --git a/discovery-provider/src/app.py b/discovery-provider/src/app.py index 809962b24cf..82346044a85 100644 --- a/discovery-provider/src/app.py +++ b/discovery-provider/src/app.py @@ -136,7 +136,7 @@ def init_contracts(): abi=abi_values["UserReplicaSetManager"]["abi"], ) - audius_data_address = "0xEC5d4F247aF81A843612eb1371CBCfa88b762119" + audius_data_address = "0xaf5C4C6C7920B4883bC6252e9d9B8fE27187Cf68" audius_data_inst = web3.eth.contract( address=audius_data_address, abi=abi_values["AudiusData"]["abi"] diff --git a/discovery-provider/src/tasks/audius_data.py b/discovery-provider/src/tasks/audius_data.py new file mode 100644 index 00000000000..f8b54045d43 --- /dev/null +++ b/discovery-provider/src/tasks/audius_data.py @@ -0,0 +1,171 @@ +import logging +from datetime import datetime +from typing import Any, Dict, Set, Tuple + +from sqlalchemy.orm.session import Session, make_transient +from src.database_task import DatabaseTask +from src.models import Playlist +from src.tasks.playlists import invalidate_old_playlist +from src.utils import helpers +from src.utils.user_event_constants import audius_data_event_types_arr + +logger = logging.getLogger(__name__) + +def audius_data_state_update( + self, + update_task: DatabaseTask, + session: Session, + audius_data_txs, + block_number, + block_timestamp, + block_hash, + ipfs_metadata, # prefix unused args with underscore to prevent pylint + _blacklisted_cids, +) -> Tuple[int, Set]: + blockhash = update_task.web3.toHex(block_hash) + num_total_changes = 0 + changed_entity_ids: Set[(int, str)] = set() + if not audius_data_txs: + return num_total_changes, changed_entity_ids + + playlist_events_lookup: Dict[int, Dict[str, Any]] = {} + # This stores the playlist_ids created or updated in the set of transactions + playlist_ids: Set[int] = set() + + for tx_receipt in audius_data_txs: + txhash = update_task.web3.toHex(tx_receipt.transactionHash) + for event_type in audius_data_event_types_arr: + audius_data_event_tx = get_audius_data_events_tx( + update_task, event_type, tx_receipt + ) + + processed_entries = 0 + # TODO: Batch reject operations for mismatched signer/userId + for entry in audius_data_event_tx: + user_id = helpers.get_tx_arg(entry, "_userId") + entity_id = helpers.get_tx_arg(entry, "_entityId") + entity_type = helpers.get_tx_arg(entry, "_entityType") + action = helpers.get_tx_arg(entry, "_action") + metadata_cid = helpers.get_tx_arg(entry, "_metadata") + signer = helpers.get_tx_arg(entry, "_signer") + metadata = ipfs_metadata[metadata_cid] if metadata_cid in ipfs_metadata else None + logger.info(f"index.py | AudiusData state update: {user_id}, entity_id={entity_id}, entity_type={entity_type}, action={action}, metadata_cid={metadata_cid}, metadata={metadata} signer={signer}") + + # Handle playlist creation + if entity_type == "Playlist" and action == "Create": + logger.info(f"index.py | AudiusData - create playlist detected") + playlist_id = entity_id + # look up or populate existing record + if playlist_id in playlist_events_lookup: + existing_playlist_record = playlist_events_lookup[playlist_id][ + "playlist" + ] + else: + existing_playlist_record = lookup_playlist_data_record( + update_task, + session, + playlist_id, + block_number, + block_hash, + txhash + ) + + playlist_record = parse_playlist_create_data_event( + update_task, + entry, + user_id, + existing_playlist_record, + metadata, + block_timestamp, + session + ) + + if playlist_record is not None: + if playlist_id not in playlist_events_lookup: + playlist_events_lookup[playlist_id] = { + "playlist": playlist_record, + "events": [], + } + else: + playlist_events_lookup[playlist_id][ + "playlist" + ] = playlist_record + playlist_events_lookup[playlist_id]["events"].append(event_type) + playlist_ids.add(playlist_id) + processed_entries += 1 + num_total_changes += processed_entries + + for playlist_id, value_obj in playlist_events_lookup.items(): + logger.info(f"index.py | playlists.py | Adding {value_obj['playlist']})") + if value_obj["events"]: + invalidate_old_playlist(session, playlist_id) + session.add(value_obj["playlist"]) + + return num_total_changes, changed_entity_ids + +def get_audius_data_events_tx(update_task, event_type, tx_receipt): + return getattr(update_task.audius_data_contract.events, event_type)().processReceipt( + tx_receipt + ) + +def lookup_playlist_data_record(update_task, session, playlist_id, block_number, block_hash, txhash): + event_blockhash = update_task.web3.toHex(block_hash) + # Check if playlist record is in the DB + playlist_exists = ( + session.query(Playlist).filter_by(playlist_id=playlist_id).count() + > 0 + ) + + playlist_record = None + if playlist_exists: + playlist_record = ( + session.query(Playlist) + .filter(Playlist.playlist_id == playlist_id, Playlist.is_current == True) + .first() + ) + + # expunge the result from sqlalchemy so we can modify it without UPDATE statements being made + # https://stackoverflow.com/questions/28871406/how-to-clone-a-sqlalchemy-db-object-with-new-primary-key + session.expunge(playlist_record) + make_transient(playlist_record) + else: + playlist_record = Playlist( + playlist_id=playlist_id, is_current=True, is_delete=False + ) + + # update these fields regardless of type + playlist_record.blocknumber = block_number + playlist_record.blockhash = event_blockhash + playlist_record.txhash = txhash + + return playlist_record + +# Create playlist specific +def parse_playlist_create_data_event( + update_task, entry, playlist_owner_id, playlist_record, playlist_metadata, block_timestamp, session +): + block_datetime = datetime.utcfromtimestamp(block_timestamp) + block_integer_time = int(block_timestamp) + logger.info(f"index.py | AudiusData | Parsing event...") + playlist_record.playlist_owner_id = playlist_owner_id + # TODO: Fix isAlbum to is_album + playlist_record.is_album = playlist_metadata['isAlbum'] + playlist_record.description = playlist_metadata['description'] + playlist_record.playlist_image_multihash = playlist_metadata['playlist_image_sizes_multihash'] + playlist_record.playlist_image_sizes_multihash = playlist_metadata['playlist_image_sizes_multihash'] + playlist_record.playlist_name = playlist_metadata['playlist_name'] + playlist_record.is_private = playlist_metadata['is_private'] if 'is_private' in playlist_metadata else False + playlist_content_array = [] + track_ids = playlist_metadata['playlist_contents'] + if track_ids: + for track_id in track_ids: + playlist_content_array.append( + {"track": track_id, "time": block_integer_time} + ) + playlist_record.playlist_contents = {"track_ids": playlist_content_array} + playlist_record.created_at = block_datetime + playlist_record.updated_at = block_datetime + + logger.info(f"index.py | AudiusData | Created playlist record {playlist_record}") + # TODO: All required fields validation + return playlist_record diff --git a/discovery-provider/src/tasks/index.py b/discovery-provider/src/tasks/index.py index ac22e7b130c..0bb6dc627bb 100644 --- a/discovery-provider/src/tasks/index.py +++ b/discovery-provider/src/tasks/index.py @@ -32,6 +32,7 @@ set_indexing_error, ) from src.queries.skipped_transactions import add_network_level_skipped_transaction +from src.tasks.audius_data import audius_data_state_update from src.tasks.celery_app import celery from src.tasks.ipld_blacklist import is_blacklisted_ipld from src.tasks.playlists import playlist_state_update @@ -65,6 +66,7 @@ most_recent_indexed_block_redis_key, ) from src.utils.session_manager import SessionManager +from src.utils.user_event_constants import audius_data_event_types_arr USER_FACTORY = CONTRACT_TYPES.USER_FACTORY.value TRACK_FACTORY = CONTRACT_TYPES.TRACK_FACTORY.value @@ -88,6 +90,9 @@ USER_REPLICA_SET_MANAGER_CONTRACT_NAME = CONTRACT_NAMES_ON_CHAIN[ CONTRACT_TYPES.USER_REPLICA_SET_MANAGER ] +AUDIUS_DATA_CONTRACT_NAME = CONTRACT_NAMES_ON_CHAIN[ + CONTRACT_TYPES.AUDIUS_DATA +] TX_TYPE_TO_HANDLER_MAP = { USER_FACTORY: user_state_update, @@ -96,6 +101,7 @@ PLAYLIST_FACTORY: playlist_state_update, USER_LIBRARY_FACTORY: user_library_state_update, USER_REPLICA_SET_MANAGER: user_replica_set_state_update, + AUDIUS_DATA: audius_data_state_update } BLOCKS_PER_DAY = (24 * 60 * 60) / 5 @@ -266,10 +272,12 @@ def fetch_cid_metadata( db, user_factory_txs, track_factory_txs, + audius_data_txs ): start_time = datetime.now() user_contract = update_task.user_contract track_contract = update_task.track_contract + audius_data_contract = update_task.audius_data_contract blacklisted_cids: Set[str] = set() cids_txhash_set: Tuple[str, Any] = set() @@ -322,6 +330,23 @@ def fetch_cid_metadata( blacklisted_cids.add(cid) cid_to_user_id[cid] = track_owner_id + for tx_receipt in audius_data_txs: + txhash = update_task.web3.toHex(tx_receipt.transactionHash) + for event_type in audius_data_event_types_arr: + audius_data_events_tx = getattr( + audius_data_contract.events, event_type + )().processReceipt(tx_receipt) + for entry in audius_data_events_tx: + event_args = entry["args"] + user_id = event_args._userId + entity_type = event_args._entityType + cid = event_args._metadata + # TODO - skip if not a multihash + logger.info(f"index.py | newcontract {txhash}, {event_args}, {entity_type}, {cid}") + cids_txhash_set.add((cid, txhash)) + cid_to_user_id[cid] = user_id + cid_type[cid] = "playlist_data" + # user -> replica set string lookup, used to make user and track cid get_metadata fetches faster user_to_replica_set = dict( session.query(User.user_id, User.creator_node_endpoint) @@ -669,6 +694,7 @@ def index_blocks(self, db, blocks_list): db, txs_grouped_by_type[USER_FACTORY], txs_grouped_by_type[TRACK_FACTORY], + txs_grouped_by_type[AUDIUS_DATA] ) logger.info( f"index.py | index_blocks - fetch_ipfs_metadata in {time.time() - fetch_ipfs_metadata_start_time}s" @@ -1125,7 +1151,14 @@ def update_task(self): address=get_contract_addresses()[USER_REPLICA_SET_MANAGER], abi=user_replica_set_manager_abi, ) - logger.info(f"index.py | contract_address {get_contract_addresses()}") + + audius_data_contract_abi = update_task.abi_values[ + AUDIUS_DATA_CONTRACT_NAME + ]["abi"] + audius_data_contract = update_task.web3.eth.contract( + address=get_contract_addresses()[AUDIUS_DATA], + abi=audius_data_contract_abi + ) update_task.track_contract = track_contract update_task.user_contract = user_contract @@ -1133,6 +1166,7 @@ def update_task(self): update_task.social_feature_contract = social_feature_contract update_task.user_library_contract = user_library_contract update_task.user_replica_set_manager_contract = user_replica_set_manager_contract + update_task.audius_data_contract = audius_data_contract # Update redis cache for health check queries update_latest_block_redis() diff --git a/discovery-provider/src/tasks/metadata.py b/discovery-provider/src/tasks/metadata.py index b75a29649a2..3eac36a3999 100644 --- a/discovery-provider/src/tasks/metadata.py +++ b/discovery-provider/src/tasks/metadata.py @@ -45,3 +45,12 @@ "events": None, "is_deactivated": None, } + +playlist_metadata_format = { + "playlist_id": None, + "playlist_contents": None, + "playlist_name": None, + "playlist_image_sizes_multihash": None, + "description": None, + "isAlbum": None +} \ No newline at end of file diff --git a/discovery-provider/src/utils/cid_metadata_client.py b/discovery-provider/src/utils/cid_metadata_client.py index 03b5da5f9ed..355ae392586 100644 --- a/discovery-provider/src/utils/cid_metadata_client.py +++ b/discovery-provider/src/utils/cid_metadata_client.py @@ -5,7 +5,12 @@ from urllib.parse import urlparse import aiohttp -from src.tasks.metadata import track_metadata_format, user_metadata_format +from pyrsistent import m +from src.tasks.metadata import ( + playlist_metadata_format, + track_metadata_format, + user_metadata_format, +) from src.utils.eth_contracts_helpers import fetch_all_registered_content_nodes logger = logging.getLogger(__name__) @@ -69,6 +74,7 @@ async def _get_metadata_async(self, async_session, multihash, gateway_endpoint): f"CIDMetadataClient | Invalid URL from provided gateway addr - {url}" ) + logger.info(f"newcontract | fetching {multihash} from {url}") async with async_session.get( url, timeout=GET_METADATA_TIMEOUT_SECONDS ) as resp: @@ -156,12 +162,21 @@ async def _fetch_metadata_from_gateway_endpoints( cid, metadata_json = future_result - # TODO add playlist type - metadata_format = ( - track_metadata_format - if cid_type[cid] == "track" - else user_metadata_format - ) + # # TODO add playlist type + # metadata_format = ( + # track_metadata_format + # if cid_type[cid] == "track" + # else user_metadata_format + # ) + metadata_format = None + if cid_type[cid] == "track": + metadata_format = track_metadata_format + elif cid_type[cid] == "user": + metadata_format = user_metadata_format + elif cid_type[cid] == "playlist_data": + metadata_format = playlist_metadata_format + else: + raise Exception(f"Unknown metadata type ${cid_type[cid]}") formatted_json = self._get_metadata_from_json( metadata_format, metadata_json diff --git a/discovery-provider/src/utils/constants.py b/discovery-provider/src/utils/constants.py index ebc69a1c122..9545986d813 100644 --- a/discovery-provider/src/utils/constants.py +++ b/discovery-provider/src/utils/constants.py @@ -1044,4 +1044,5 @@ class CONTRACT_TYPES(Enum): CONTRACT_TYPES.PLAYLIST_FACTORY: "PlaylistFactory", CONTRACT_TYPES.USER_LIBRARY_FACTORY: "UserLibraryFactory", CONTRACT_TYPES.USER_REPLICA_SET_MANAGER: "UserReplicaSetManager", + CONTRACT_TYPES.AUDIUS_DATA: "AudiusData" } diff --git a/discovery-provider/src/utils/user_event_constants.py b/discovery-provider/src/utils/user_event_constants.py index 5aa42270bcc..3b6fec4229a 100644 --- a/discovery-provider/src/utils/user_event_constants.py +++ b/discovery-provider/src/utils/user_event_constants.py @@ -37,3 +37,11 @@ user_replica_set_manager_event_types_lookup["update_replica_set"], user_replica_set_manager_event_types_lookup["add_or_update_content_node"], ] + +audius_data_event_types_lookup = { + "manage_entity": "ManageEntity" +} + +audius_data_event_types_arr = [ + audius_data_event_types_lookup["manage_entity"] +] \ No newline at end of file diff --git a/libs/src/api/audiusData.ts b/libs/src/api/audiusData.ts index ee7dd4fb5e5..474e4ea3be1 100644 --- a/libs/src/api/audiusData.ts +++ b/libs/src/api/audiusData.ts @@ -9,7 +9,7 @@ export class AudiusData extends Base { /** * Calculate an unoccupied playlist ID - * Maximum value is UINT MAX + * Maximum value is postgres integer max (2147483647) */ async getValidPlaylistId (): Promise { let playlistId: number = 10 @@ -17,14 +17,14 @@ export class AudiusData extends Base { while (!validIdFound) { const resp = await this.discoveryProvider.getPlaylists(1, 0, [playlistId]) if (resp.length !== 0) { + // TODO: Playlist ID Min offset playlistId = Math.floor( - Math.random() * 9007199254740991 + Math.random() * 2147483647 ) } else { validIdFound = true } } - console.log(`Playlist2 | Returning playlistID=${playlistId}`) return playlistId } @@ -45,20 +45,23 @@ export class AudiusData extends Base { isAlbum: boolean, coverArt: any, logger: any - }): Promise { + }): Promise<{ blockHash: any; blockNumber: any; playlistId: number; }> { + let respValues = { + blockHash: null, + blockNumber: null, + playlistId: 0 + } try { const ownerId: number = this.userStateManager.getCurrentUserId() const action = 'Create' const entityType = 'Playlist' const entityId = await this.getValidPlaylistId() - logger.info(`CreatePlaylistData: user=${ownerId}, ${action}, ${entityType} ${playlistName}, album=${isAlbum}, ids=${trackIds}, description=${description} playlistId=${entityId}, coverArt=${coverArt}`) this.REQUIRES(Services.CREATOR_NODE) const updatedPlaylistImage = await this.creatorNode.uploadImage( coverArt, true // square ) const dirCID = updatedPlaylistImage.dirCID - logger.info(`CreatePlaylistData - ${dirCID}`) const metadata = { action, entity_type: entityType, @@ -69,8 +72,7 @@ export class AudiusData extends Base { description, isAlbum } - const { metadataMultihash, metadataFileUUID } = await this.creatorNode.uploadPlaylistMetadata(metadata) - logger.info(`CreatePlaylistData - metadata: ${metadataMultihash} - ${metadataFileUUID}`) + const { metadataMultihash } = await this.creatorNode.uploadPlaylistMetadata(metadata) let resp = await this.manageEntity({ userId: ownerId, entityType, @@ -79,10 +81,20 @@ export class AudiusData extends Base { metadataMultihash }) logger.info(`CreatePlaylistData - ${JSON.stringify(resp)}`) + let txReceipt = resp.txReceipt + respValues = { + blockHash: txReceipt.blockHash, + blockNumber: txReceipt.blockNumber, + playlistId: entityId + } } catch(e) { logger.error(`Data create playlist: err ${e}`) } - } + return respValues + } + /* +CreatePlaylistData - {"txReceipt":{"transactionHash":"0x1dd301853668949e86ae658d9f610d2ff89c5112f95f7cfdc1e12ab4c3b74f88","transactionIndex":0,"blockHash":"0xde69399a9aa538a22f8dcf8914b006420b362418b67b386bdaf4389da309361f","blockNumber":1160,"from":"0xbe718f98a5b5a473186eb6e30888f26e72be0b66","to":"0xaf5c4c6c7920b4883bc6252e9d9b8fe27187cf68","gasUsed":62800,"cumulativeGasUsed":62800,"contractAddress":null,"logs":[{"logIndex":0,"transactionIndex":0,"transactionHash":"0x1dd301853668949e86ae658d9f610d2ff89c5112f95f7cfdc1e12ab4c3b74f88","blockHash":"0xde69399a9aa538a22f8dcf8914b006420b362418b67b386bdaf4389da309361f","blockNumber":1160,"address":"0xaf5C4C6C7920B4883bC6252e9d9B8fE27187Cf68","data":"0x0000000000000000000000000000000000000000000000000000000000000001000000000000000000000000356b69c97a589947f8aa4fda6a774007d76aff2500000000000000000000000000000000000000000000000000000000000000c0000000000000000000000000000000000000000000000000000000004b079460000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000001600000000000000000000000000000000000000000000000000000000000000008506c61796c697374000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002e516d5743766274784a43753550787358433334626f65573965324341543542657a7472413252487667356164563600000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000064372656174650000000000000000000000000000000000000000000000000000","topics":["0x772d62d21cc8467a14127f11ab2c094d32e5b521433cefba5a7312fc464d88b4"],"type":"mined","id":"log_9b1b81dc"}],"status":true,"logsBloom":"0x00000000000000000000000000000000000000000000000000000000000000001000000000000000000000000200000000000000000000000008000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000","events":{"ManageEntity":{"returnValues":{"_userId":"1","_signer":"0x356b69c97a589947f8aa4fda6a774007d76aff25","_entityType":"Playlist","_entityId":"1258787936","_metadata":"QmWCvbtxJCu5PxsXC34boeW9e2CAT5BeztrA2RHvg5adV6","_action":"Create"}}}},"error":""} + */ /** * Manage an entity with the updated data contract flow From 77cbabcfbc6d17df3bc2b65dc5066f198371dea4 Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Tue, 5 Jul 2022 15:59:29 +0000 Subject: [PATCH 028/101] Fix return type, minor update --- discovery-provider/src/app.py | 5 +- discovery-provider/src/tasks/audius_data.py | 69 +++++++++++++------ discovery-provider/src/tasks/index.py | 32 ++++----- discovery-provider/src/tasks/metadata.py | 2 +- discovery-provider/src/utils/constants.py | 2 +- .../src/utils/user_event_constants.py | 8 +-- 6 files changed, 67 insertions(+), 51 deletions(-) diff --git a/discovery-provider/src/app.py b/discovery-provider/src/app.py index 82346044a85..4d8bcb42260 100644 --- a/discovery-provider/src/app.py +++ b/discovery-provider/src/app.py @@ -138,8 +138,7 @@ def init_contracts(): audius_data_address = "0xaf5C4C6C7920B4883bC6252e9d9B8fE27187Cf68" audius_data_inst = web3.eth.contract( - address=audius_data_address, - abi=abi_values["AudiusData"]["abi"] + address=audius_data_address, abi=abi_values["AudiusData"]["abi"] ) contract_address_dict = { @@ -151,7 +150,7 @@ def init_contracts(): "user_library_factory": user_library_factory_address, "ipld_blacklist_factory": ipld_blacklist_factory_address, "user_replica_set_manager": user_replica_set_manager_address, - "audius_data": audius_data_address + "audius_data": audius_data_address, } return ( diff --git a/discovery-provider/src/tasks/audius_data.py b/discovery-provider/src/tasks/audius_data.py index f8b54045d43..ce3f69fc96c 100644 --- a/discovery-provider/src/tasks/audius_data.py +++ b/discovery-provider/src/tasks/audius_data.py @@ -11,6 +11,7 @@ logger = logging.getLogger(__name__) + def audius_data_state_update( self, update_task: DatabaseTask, @@ -21,10 +22,11 @@ def audius_data_state_update( block_hash, ipfs_metadata, # prefix unused args with underscore to prevent pylint _blacklisted_cids, -) -> Tuple[int, Set]: - blockhash = update_task.web3.toHex(block_hash) +) -> Tuple[int, Dict[str, Set[(int)]]]: num_total_changes = 0 - changed_entity_ids: Set[(int, str)] = set() + + changed_entity_ids: Dict[str, Set[(int)]] + if not audius_data_txs: return num_total_changes, changed_entity_ids @@ -48,8 +50,14 @@ def audius_data_state_update( action = helpers.get_tx_arg(entry, "_action") metadata_cid = helpers.get_tx_arg(entry, "_metadata") signer = helpers.get_tx_arg(entry, "_signer") - metadata = ipfs_metadata[metadata_cid] if metadata_cid in ipfs_metadata else None - logger.info(f"index.py | AudiusData state update: {user_id}, entity_id={entity_id}, entity_type={entity_type}, action={action}, metadata_cid={metadata_cid}, metadata={metadata} signer={signer}") + metadata = ( + ipfs_metadata[metadata_cid] + if metadata_cid in ipfs_metadata + else None + ) + logger.info( + f"index.py | AudiusData state update: {user_id}, entity_id={entity_id}, entity_type={entity_type}, action={action}, metadata_cid={metadata_cid}, metadata={metadata} signer={signer}" + ) # Handle playlist creation if entity_type == "Playlist" and action == "Create": @@ -67,7 +75,7 @@ def audius_data_state_update( playlist_id, block_number, block_hash, - txhash + txhash, ) playlist_record = parse_playlist_create_data_event( @@ -77,7 +85,7 @@ def audius_data_state_update( existing_playlist_record, metadata, block_timestamp, - session + session, ) if playlist_record is not None: @@ -95,6 +103,9 @@ def audius_data_state_update( processed_entries += 1 num_total_changes += processed_entries + # Update changed entity dictionary + changed_entity_ids["playlist"] = playlist_ids + for playlist_id, value_obj in playlist_events_lookup.items(): logger.info(f"index.py | playlists.py | Adding {value_obj['playlist']})") if value_obj["events"]: @@ -103,17 +114,20 @@ def audius_data_state_update( return num_total_changes, changed_entity_ids + def get_audius_data_events_tx(update_task, event_type, tx_receipt): - return getattr(update_task.audius_data_contract.events, event_type)().processReceipt( - tx_receipt - ) + return getattr( + update_task.audius_data_contract.events, event_type + )().processReceipt(tx_receipt) -def lookup_playlist_data_record(update_task, session, playlist_id, block_number, block_hash, txhash): + +def lookup_playlist_data_record( + update_task, session, playlist_id, block_number, block_hash, txhash +): event_blockhash = update_task.web3.toHex(block_hash) # Check if playlist record is in the DB playlist_exists = ( - session.query(Playlist).filter_by(playlist_id=playlist_id).count() - > 0 + session.query(Playlist).filter_by(playlist_id=playlist_id).count() > 0 ) playlist_record = None @@ -140,23 +154,36 @@ def lookup_playlist_data_record(update_task, session, playlist_id, block_number, return playlist_record + # Create playlist specific def parse_playlist_create_data_event( - update_task, entry, playlist_owner_id, playlist_record, playlist_metadata, block_timestamp, session + update_task, + entry, + playlist_owner_id, + playlist_record, + playlist_metadata, + block_timestamp, + session, ): block_datetime = datetime.utcfromtimestamp(block_timestamp) block_integer_time = int(block_timestamp) logger.info(f"index.py | AudiusData | Parsing event...") playlist_record.playlist_owner_id = playlist_owner_id # TODO: Fix isAlbum to is_album - playlist_record.is_album = playlist_metadata['isAlbum'] - playlist_record.description = playlist_metadata['description'] - playlist_record.playlist_image_multihash = playlist_metadata['playlist_image_sizes_multihash'] - playlist_record.playlist_image_sizes_multihash = playlist_metadata['playlist_image_sizes_multihash'] - playlist_record.playlist_name = playlist_metadata['playlist_name'] - playlist_record.is_private = playlist_metadata['is_private'] if 'is_private' in playlist_metadata else False + playlist_record.is_album = playlist_metadata["isAlbum"] + playlist_record.description = playlist_metadata["description"] + playlist_record.playlist_image_multihash = playlist_metadata[ + "playlist_image_sizes_multihash" + ] + playlist_record.playlist_image_sizes_multihash = playlist_metadata[ + "playlist_image_sizes_multihash" + ] + playlist_record.playlist_name = playlist_metadata["playlist_name"] + playlist_record.is_private = ( + playlist_metadata["is_private"] if "is_private" in playlist_metadata else False + ) playlist_content_array = [] - track_ids = playlist_metadata['playlist_contents'] + track_ids = playlist_metadata["playlist_contents"] if track_ids: for track_id in track_ids: playlist_content_array.append( diff --git a/discovery-provider/src/tasks/index.py b/discovery-provider/src/tasks/index.py index 0bb6dc627bb..05c0c39e646 100644 --- a/discovery-provider/src/tasks/index.py +++ b/discovery-provider/src/tasks/index.py @@ -90,9 +90,7 @@ USER_REPLICA_SET_MANAGER_CONTRACT_NAME = CONTRACT_NAMES_ON_CHAIN[ CONTRACT_TYPES.USER_REPLICA_SET_MANAGER ] -AUDIUS_DATA_CONTRACT_NAME = CONTRACT_NAMES_ON_CHAIN[ - CONTRACT_TYPES.AUDIUS_DATA -] +AUDIUS_DATA_CONTRACT_NAME = CONTRACT_NAMES_ON_CHAIN[CONTRACT_TYPES.AUDIUS_DATA] TX_TYPE_TO_HANDLER_MAP = { USER_FACTORY: user_state_update, @@ -101,7 +99,7 @@ PLAYLIST_FACTORY: playlist_state_update, USER_LIBRARY_FACTORY: user_library_state_update, USER_REPLICA_SET_MANAGER: user_replica_set_state_update, - AUDIUS_DATA: audius_data_state_update + AUDIUS_DATA: audius_data_state_update, } BLOCKS_PER_DAY = (24 * 60 * 60) / 5 @@ -268,12 +266,7 @@ def fetch_tx_receipts(self, block): return block_tx_with_receipts -def fetch_cid_metadata( - db, - user_factory_txs, - track_factory_txs, - audius_data_txs -): +def fetch_cid_metadata(db, user_factory_txs, track_factory_txs, audius_data_txs): start_time = datetime.now() user_contract = update_task.user_contract track_contract = update_task.track_contract @@ -342,7 +335,9 @@ def fetch_cid_metadata( entity_type = event_args._entityType cid = event_args._metadata # TODO - skip if not a multihash - logger.info(f"index.py | newcontract {txhash}, {event_args}, {entity_type}, {cid}") + logger.info( + f"index.py | newcontract {txhash}, {event_args}, {entity_type}, {cid}" + ) cids_txhash_set.add((cid, txhash)) cid_to_user_id[cid] = user_id cid_type[cid] = "playlist_data" @@ -472,7 +467,9 @@ def get_contract_type_for_tx(tx_type_to_grouped_lists_map, tx, tx_receipt): ) break - logger.info(f"index.py | checking returned {contract_type} vs {tx_target_contract_address}") + logger.info( + f"index.py | checking returned {contract_type} vs {tx_target_contract_address}" + ) return contract_type @@ -628,7 +625,7 @@ def index_blocks(self, db, blocks_list): PLAYLIST_FACTORY: [], USER_LIBRARY_FACTORY: [], USER_REPLICA_SET_MANAGER: [], - AUDIUS_DATA: [] + AUDIUS_DATA: [], } try: """ @@ -694,7 +691,7 @@ def index_blocks(self, db, blocks_list): db, txs_grouped_by_type[USER_FACTORY], txs_grouped_by_type[TRACK_FACTORY], - txs_grouped_by_type[AUDIUS_DATA] + txs_grouped_by_type[AUDIUS_DATA], ) logger.info( f"index.py | index_blocks - fetch_ipfs_metadata in {time.time() - fetch_ipfs_metadata_start_time}s" @@ -1152,12 +1149,9 @@ def update_task(self): abi=user_replica_set_manager_abi, ) - audius_data_contract_abi = update_task.abi_values[ - AUDIUS_DATA_CONTRACT_NAME - ]["abi"] + audius_data_contract_abi = update_task.abi_values[AUDIUS_DATA_CONTRACT_NAME]["abi"] audius_data_contract = update_task.web3.eth.contract( - address=get_contract_addresses()[AUDIUS_DATA], - abi=audius_data_contract_abi + address=get_contract_addresses()[AUDIUS_DATA], abi=audius_data_contract_abi ) update_task.track_contract = track_contract diff --git a/discovery-provider/src/tasks/metadata.py b/discovery-provider/src/tasks/metadata.py index 3eac36a3999..15531faba4a 100644 --- a/discovery-provider/src/tasks/metadata.py +++ b/discovery-provider/src/tasks/metadata.py @@ -52,5 +52,5 @@ "playlist_name": None, "playlist_image_sizes_multihash": None, "description": None, - "isAlbum": None + "isAlbum": None, } \ No newline at end of file diff --git a/discovery-provider/src/utils/constants.py b/discovery-provider/src/utils/constants.py index 9545986d813..d7221a593df 100644 --- a/discovery-provider/src/utils/constants.py +++ b/discovery-provider/src/utils/constants.py @@ -1044,5 +1044,5 @@ class CONTRACT_TYPES(Enum): CONTRACT_TYPES.PLAYLIST_FACTORY: "PlaylistFactory", CONTRACT_TYPES.USER_LIBRARY_FACTORY: "UserLibraryFactory", CONTRACT_TYPES.USER_REPLICA_SET_MANAGER: "UserReplicaSetManager", - CONTRACT_TYPES.AUDIUS_DATA: "AudiusData" + CONTRACT_TYPES.AUDIUS_DATA: "AudiusData", } diff --git a/discovery-provider/src/utils/user_event_constants.py b/discovery-provider/src/utils/user_event_constants.py index 3b6fec4229a..bbe1e72310a 100644 --- a/discovery-provider/src/utils/user_event_constants.py +++ b/discovery-provider/src/utils/user_event_constants.py @@ -38,10 +38,6 @@ user_replica_set_manager_event_types_lookup["add_or_update_content_node"], ] -audius_data_event_types_lookup = { - "manage_entity": "ManageEntity" -} +audius_data_event_types_lookup = {"manage_entity": "ManageEntity"} -audius_data_event_types_arr = [ - audius_data_event_types_lookup["manage_entity"] -] \ No newline at end of file +audius_data_event_types_arr = [audius_data_event_types_lookup["manage_entity"]] From 8d81f19ec5278d824ce2c94381e31d011c615ba5 Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Tue, 5 Jul 2022 16:02:01 +0000 Subject: [PATCH 029/101] - --- discovery-provider/src/tasks/audius_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/discovery-provider/src/tasks/audius_data.py b/discovery-provider/src/tasks/audius_data.py index ce3f69fc96c..322b18b2768 100644 --- a/discovery-provider/src/tasks/audius_data.py +++ b/discovery-provider/src/tasks/audius_data.py @@ -25,7 +25,7 @@ def audius_data_state_update( ) -> Tuple[int, Dict[str, Set[(int)]]]: num_total_changes = 0 - changed_entity_ids: Dict[str, Set[(int)]] + changed_entity_ids: Dict[str, Set[(int)]] = {} if not audius_data_txs: return num_total_changes, changed_entity_ids From 944213338ef2bafe6c8429491104b4fc50363186 Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Tue, 5 Jul 2022 16:02:26 +0000 Subject: [PATCH 030/101] Lint errors --- discovery-provider/src/tasks/audius_data.py | 1 - 1 file changed, 1 deletion(-) diff --git a/discovery-provider/src/tasks/audius_data.py b/discovery-provider/src/tasks/audius_data.py index 322b18b2768..876abfeeda8 100644 --- a/discovery-provider/src/tasks/audius_data.py +++ b/discovery-provider/src/tasks/audius_data.py @@ -61,7 +61,6 @@ def audius_data_state_update( # Handle playlist creation if entity_type == "Playlist" and action == "Create": - logger.info(f"index.py | AudiusData - create playlist detected") playlist_id = entity_id # look up or populate existing record if playlist_id in playlist_events_lookup: From 55d3db4dd3fd4174533951008ba166cb41a1c7bc Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Tue, 5 Jul 2022 16:03:37 +0000 Subject: [PATCH 031/101] - --- discovery-provider/src/tasks/audius_data.py | 1 - discovery-provider/src/tasks/metadata.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/discovery-provider/src/tasks/audius_data.py b/discovery-provider/src/tasks/audius_data.py index 876abfeeda8..61f0e05be76 100644 --- a/discovery-provider/src/tasks/audius_data.py +++ b/discovery-provider/src/tasks/audius_data.py @@ -166,7 +166,6 @@ def parse_playlist_create_data_event( ): block_datetime = datetime.utcfromtimestamp(block_timestamp) block_integer_time = int(block_timestamp) - logger.info(f"index.py | AudiusData | Parsing event...") playlist_record.playlist_owner_id = playlist_owner_id # TODO: Fix isAlbum to is_album playlist_record.is_album = playlist_metadata["isAlbum"] diff --git a/discovery-provider/src/tasks/metadata.py b/discovery-provider/src/tasks/metadata.py index 15531faba4a..d346e15a8ac 100644 --- a/discovery-provider/src/tasks/metadata.py +++ b/discovery-provider/src/tasks/metadata.py @@ -53,4 +53,4 @@ "playlist_image_sizes_multihash": None, "description": None, "isAlbum": None, -} \ No newline at end of file +} From a24e51ed01cb4f607fc4e1a674d4afadf4ac886c Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Tue, 5 Jul 2022 16:04:37 +0000 Subject: [PATCH 032/101] NIT --- discovery-provider/src/utils/cid_metadata_client.py | 1 - 1 file changed, 1 deletion(-) diff --git a/discovery-provider/src/utils/cid_metadata_client.py b/discovery-provider/src/utils/cid_metadata_client.py index 355ae392586..e50a238ee0d 100644 --- a/discovery-provider/src/utils/cid_metadata_client.py +++ b/discovery-provider/src/utils/cid_metadata_client.py @@ -5,7 +5,6 @@ from urllib.parse import urlparse import aiohttp -from pyrsistent import m from src.tasks.metadata import ( playlist_metadata_format, track_metadata_format, From af6bfb989ad1d00c5d531aaf9eed36902b43122f Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Tue, 5 Jul 2022 16:13:01 +0000 Subject: [PATCH 033/101] Adding is private + fixing bool format --- discovery-provider/src/tasks/audius_data.py | 3 +-- libs/src/api/audiusData.ts | 8 ++++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/discovery-provider/src/tasks/audius_data.py b/discovery-provider/src/tasks/audius_data.py index 61f0e05be76..fcba2134d0d 100644 --- a/discovery-provider/src/tasks/audius_data.py +++ b/discovery-provider/src/tasks/audius_data.py @@ -167,8 +167,7 @@ def parse_playlist_create_data_event( block_datetime = datetime.utcfromtimestamp(block_timestamp) block_integer_time = int(block_timestamp) playlist_record.playlist_owner_id = playlist_owner_id - # TODO: Fix isAlbum to is_album - playlist_record.is_album = playlist_metadata["isAlbum"] + playlist_record.is_album = playlist_metadata["is_album"] playlist_record.description = playlist_metadata["description"] playlist_record.playlist_image_multihash = playlist_metadata[ "playlist_image_sizes_multihash" diff --git a/libs/src/api/audiusData.ts b/libs/src/api/audiusData.ts index 474e4ea3be1..95d42be5234 100644 --- a/libs/src/api/audiusData.ts +++ b/libs/src/api/audiusData.ts @@ -36,6 +36,7 @@ export class AudiusData extends Base { trackIds, description, isAlbum, + isPrivate, coverArt, logger = console }: { @@ -43,6 +44,7 @@ export class AudiusData extends Base { trackIds: number[], description: string, isAlbum: boolean, + isPrivate: boolean, coverArt: any, logger: any }): Promise<{ blockHash: any; blockNumber: any; playlistId: number; }> { @@ -70,7 +72,8 @@ export class AudiusData extends Base { playlist_name: playlistName, playlist_image_sizes_multihash: dirCID, description, - isAlbum + is_album : isAlbum, + is_private: isPrivate } const { metadataMultihash } = await this.creatorNode.uploadPlaylistMetadata(metadata) let resp = await this.manageEntity({ @@ -92,9 +95,6 @@ export class AudiusData extends Base { } return respValues } - /* -CreatePlaylistData - {"txReceipt":{"transactionHash":"0x1dd301853668949e86ae658d9f610d2ff89c5112f95f7cfdc1e12ab4c3b74f88","transactionIndex":0,"blockHash":"0xde69399a9aa538a22f8dcf8914b006420b362418b67b386bdaf4389da309361f","blockNumber":1160,"from":"0xbe718f98a5b5a473186eb6e30888f26e72be0b66","to":"0xaf5c4c6c7920b4883bc6252e9d9b8fe27187cf68","gasUsed":62800,"cumulativeGasUsed":62800,"contractAddress":null,"logs":[{"logIndex":0,"transactionIndex":0,"transactionHash":"0x1dd301853668949e86ae658d9f610d2ff89c5112f95f7cfdc1e12ab4c3b74f88","blockHash":"0xde69399a9aa538a22f8dcf8914b006420b362418b67b386bdaf4389da309361f","blockNumber":1160,"address":"0xaf5C4C6C7920B4883bC6252e9d9B8fE27187Cf68","data":"0x0000000000000000000000000000000000000000000000000000000000000001000000000000000000000000356b69c97a589947f8aa4fda6a774007d76aff2500000000000000000000000000000000000000000000000000000000000000c0000000000000000000000000000000000000000000000000000000004b079460000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000001600000000000000000000000000000000000000000000000000000000000000008506c61796c697374000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002e516d5743766274784a43753550787358433334626f65573965324341543542657a7472413252487667356164563600000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000064372656174650000000000000000000000000000000000000000000000000000","topics":["0x772d62d21cc8467a14127f11ab2c094d32e5b521433cefba5a7312fc464d88b4"],"type":"mined","id":"log_9b1b81dc"}],"status":true,"logsBloom":"0x00000000000000000000000000000000000000000000000000000000000000001000000000000000000000000200000000000000000000000008000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000","events":{"ManageEntity":{"returnValues":{"_userId":"1","_signer":"0x356b69c97a589947f8aa4fda6a774007d76aff25","_entityType":"Playlist","_entityId":"1258787936","_metadata":"QmWCvbtxJCu5PxsXC34boeW9e2CAT5BeztrA2RHvg5adV6","_action":"Create"}}}},"error":""} - */ /** * Manage an entity with the updated data contract flow From dc4104a40d307e75429cb381c89afac6af2063e7 Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Tue, 5 Jul 2022 17:18:39 +0000 Subject: [PATCH 034/101] Libs lint --- libs/src/api/audiusData.ts | 207 +++++++++--------- .../services/dataContracts/AudiusContracts.ts | 9 +- .../dataContracts/AudiusDataClient.ts | 75 ++++--- 3 files changed, 148 insertions(+), 143 deletions(-) diff --git a/libs/src/api/audiusData.ts b/libs/src/api/audiusData.ts index 95d42be5234..46e99ee541b 100644 --- a/libs/src/api/audiusData.ts +++ b/libs/src/api/audiusData.ts @@ -7,125 +7,124 @@ export class AudiusData extends Base { this.createPlaylist = this.createPlaylist.bind(this) } - /** + /** * Calculate an unoccupied playlist ID * Maximum value is postgres integer max (2147483647) */ - async getValidPlaylistId (): Promise { - let playlistId: number = 10 - let validIdFound: boolean = false - while (!validIdFound) { - const resp = await this.discoveryProvider.getPlaylists(1, 0, [playlistId]) - if (resp.length !== 0) { - // TODO: Playlist ID Min offset - playlistId = Math.floor( - Math.random() * 2147483647 - ) - } else { - validIdFound = true - } - } - return playlistId + async getValidPlaylistId(): Promise { + let playlistId: number = 10 + let validIdFound: boolean = false + while (!validIdFound) { + const resp = await this.discoveryProvider.getPlaylists(1, 0, [playlistId]) + if (resp.length !== 0) { + // TODO: Playlist ID Min offset + playlistId = Math.floor(Math.random() * 2147483647) + } else { + validIdFound = true + } } + return playlistId + } /** * Create a playlist using updated data contracts flow */ - async createPlaylist({ - playlistName, - trackIds, - description, - isAlbum, - isPrivate, + async createPlaylist({ + playlistName, + trackIds, + description, + isAlbum, + isPrivate, + coverArt, + logger = console + }: { + playlistName: string + trackIds: number[] + description: string + isAlbum: boolean + isPrivate: boolean + coverArt: any + logger: any + }): Promise<{ blockHash: any; blockNumber: any; playlistId: number }> { + let respValues = { + blockHash: null, + blockNumber: null, + playlistId: 0 + } + try { + const ownerId: number = this.userStateManager.getCurrentUserId() + const action = 'Create' + const entityType = 'Playlist' + const entityId = await this.getValidPlaylistId() + this.REQUIRES(Services.CREATOR_NODE) + const updatedPlaylistImage = await this.creatorNode.uploadImage( coverArt, - logger = console - }: { - playlistName: string, - trackIds: number[], - description: string, - isAlbum: boolean, - isPrivate: boolean, - coverArt: any, - logger: any - }): Promise<{ blockHash: any; blockNumber: any; playlistId: number; }> { - let respValues = { - blockHash: null, - blockNumber: null, - playlistId: 0 + true // square + ) + const dirCID = updatedPlaylistImage.dirCID + const metadata = { + action, + entity_type: entityType, + playlist_id: entityId, + playlist_contents: trackIds, + playlist_name: playlistName, + playlist_image_sizes_multihash: dirCID, + description, + is_album: isAlbum, + is_private: isPrivate + } + const { metadataMultihash } = + await this.creatorNode.uploadPlaylistMetadata(metadata) + const resp = await this.manageEntity({ + userId: ownerId, + entityType, + entityId, + action, + metadataMultihash + }) + logger.info(`CreatePlaylistData - ${JSON.stringify(resp)}`) + const txReceipt = resp.txReceipt + respValues = { + blockHash: txReceipt.blockHash, + blockNumber: txReceipt.blockNumber, + playlistId: entityId } - try { - const ownerId: number = this.userStateManager.getCurrentUserId() - const action = 'Create' - const entityType = 'Playlist' - const entityId = await this.getValidPlaylistId() - this.REQUIRES(Services.CREATOR_NODE) - const updatedPlaylistImage = await this.creatorNode.uploadImage( - coverArt, - true // square - ) - const dirCID = updatedPlaylistImage.dirCID - const metadata = { - action, - entity_type: entityType, - playlist_id: entityId, - playlist_contents: trackIds, - playlist_name: playlistName, - playlist_image_sizes_multihash: dirCID, - description, - is_album : isAlbum, - is_private: isPrivate - } - const { metadataMultihash } = await this.creatorNode.uploadPlaylistMetadata(metadata) - let resp = await this.manageEntity({ - userId: ownerId, - entityType, - entityId, - action, - metadataMultihash - }) - logger.info(`CreatePlaylistData - ${JSON.stringify(resp)}`) - let txReceipt = resp.txReceipt - respValues = { - blockHash: txReceipt.blockHash, - blockNumber: txReceipt.blockNumber, - playlistId: entityId - } - } catch(e) { - logger.error(`Data create playlist: err ${e}`) - } - return respValues + } catch (e) { + logger.error(`Data create playlist: err ${e}`) + } + return respValues } /** * Manage an entity with the updated data contract flow * Leveraged to manipulate User/Track/Playlist/+ other entities */ - async manageEntity({ - userId, - entityType, - entityId, - action, - metadataMultihash - }: { - userId: number, - entityType: string, - entityId: number, - action: string, - metadataMultihash: string - }): Promise<{ txReceipt: {}, error: any}> { - let error:string = "" - let resp - try { - resp = await this.contracts.AudiusDataClient.manageEntity( - userId, - entityType, - entityId, - action, - metadataMultihash - ) - } catch(e) { - error = (e as Error).message - } - return { txReceipt: resp.txReceipt, error} + async manageEntity({ + userId, + entityType, + entityId, + action, + metadataMultihash + }: { + userId: number + entityType: string + entityId: number + action: string + metadataMultihash: string + }): Promise<{ txReceipt: {}; error: any }> { + let error: string = '' + let resp + try { + resp = await this.contracts.AudiusDataClient.manageEntity( + userId, + entityType, + entityId, + action, + metadataMultihash + ) + } catch (e) { + error = (e as Error).message } + return { txReceipt: resp.txReceipt, error } + } } diff --git a/libs/src/services/dataContracts/AudiusContracts.ts b/libs/src/services/dataContracts/AudiusContracts.ts index 295356a9f03..d9561a56d81 100644 --- a/libs/src/services/dataContracts/AudiusContracts.ts +++ b/libs/src/services/dataContracts/AudiusContracts.ts @@ -33,10 +33,7 @@ const IPLDBlacklistFactoryABI = Utils.importDataContractABI( const UserReplicaSetManagerABI = Utils.importDataContractABI( 'UserReplicaSetManager.json' ).abi -const AudiusDataABI = Utils.importDataContractABI( - 'AudiusData.json' -).abi -console.log(AudiusDataABI) +const AudiusDataABI = Utils.importDataContractABI('AudiusData.json').abi // define contract registry keys const UserFactoryRegistryKey = 'UserFactory' @@ -135,10 +132,10 @@ export class AudiusContracts { this.AudiusDataClient = new AudiusDataClient( this.web3Manager, AudiusDataABI, - "AudiusData", + 'AudiusData', this.getRegistryAddressForContract, this.logger, - "0xDDb64fE46a91D46ee29420539FC25FD07c5FEa3E" // TODO: Config var + '0xaf5C4C6C7920B4883bC6252e9d9B8fE27187Cf68' // TODO: Config var ) this.contractClients = [ diff --git a/libs/src/services/dataContracts/AudiusDataClient.ts b/libs/src/services/dataContracts/AudiusDataClient.ts index 154c02fe818..d3ca3f6a8f8 100644 --- a/libs/src/services/dataContracts/AudiusDataClient.ts +++ b/libs/src/services/dataContracts/AudiusDataClient.ts @@ -3,38 +3,47 @@ import * as signatureSchemas from '../../../data-contracts/signatureSchemas' import type { Web3Manager } from '../web3Manager' export class AudiusDataClient extends ContractClient { - override web3Manager!: Web3Manager + override web3Manager!: Web3Manager - async manageEntity( - userId: number, - entityType: string, - entityId: number, - action: string, - metadata: string, - ) { - const nonce = signatureSchemas.getNonce() - const chainId = await this.getEthNetId() - const contractAddress = await this.getAddress() - const signatureData = signatureSchemas.generators.getManageEntityData(chainId, contractAddress, userId, entityType, entityId, action, metadata, nonce) - const sig = await this.web3Manager.signTypedData(signatureData) - const method = await this.getMethod( - 'manageEntity', - userId, - entityType, - entityId, - action, - metadata, - nonce, - sig - ) - const tx = await this.web3Manager.sendTransaction( - method, - this.contractRegistryKey, - contractAddress - ) - console.log(tx) - return { - txReceipt: tx - } + async manageEntity( + userId: number, + entityType: string, + entityId: number, + action: string, + metadata: string + ) { + const nonce = signatureSchemas.getNonce() + const chainId = await this.getEthNetId() + const contractAddress = await this.getAddress() + const signatureData = signatureSchemas.generators.getManageEntityData( + chainId, + contractAddress, + userId, + entityType, + entityId, + action, + metadata, + nonce + ) + const sig = await this.web3Manager.signTypedData(signatureData) + const method = await this.getMethod( + 'manageEntity', + userId, + entityType, + entityId, + action, + metadata, + nonce, + sig + ) + const tx = await this.web3Manager.sendTransaction( + method, + this.contractRegistryKey, + contractAddress + ) + console.log(tx) + return { + txReceipt: tx } -} \ No newline at end of file + } +} From 7a65bc3a54d5f84e1be805892935e121574b0772 Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Tue, 5 Jul 2022 19:14:47 +0000 Subject: [PATCH 035/101] Hardcoded local dev values --- contracts/contract-config.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/contracts/contract-config.js b/contracts/contract-config.js index 5fb41f7dda3..1b25b97c959 100644 --- a/contracts/contract-config.js +++ b/contracts/contract-config.js @@ -4,11 +4,11 @@ module.exports = { development: { verifierAddress: '0xbbbb93A6B3A1D6fDd27909729b95CCB0cc9002C0', blacklisterAddress: null, - bootstrapSPIds: [], - bootstrapSPDelegateWallets: [], - bootstrapSPOwnerWallets: [], + bootstrapSPIds: [1,2,3], + bootstrapSPDelegateWallets: ['0xFFcf8FDEE72ac11b5c542428B35EEF5769C409f0', '0x22d491Bde2303f2f43325b2108D26f1eAbA1e32b', '0xE11BA2b4D45Eaed5996Cd0823791E0C93114882d'], + bootstrapSPOwnerWallets: ['0xFFcf8FDEE72ac11b5c542428B35EEF5769C409f0', '0x22d491Bde2303f2f43325b2108D26f1eAbA1e32b', '0xE11BA2b4D45Eaed5996Cd0823791E0C93114882d'], userReplicaSetBootstrapAddress: null, - registryAddress: null + registryAddress: '0xCfEB869F69431e42cdB54A4F4f105C19C080A601' }, test_local: { verifierAddress: null, From fcacc7499c6f1b631afbbba681475bc641193968 Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Wed, 6 Jul 2022 00:16:39 +0000 Subject: [PATCH 036/101] Caching local setup --- contracts/.gitignore | 1 + contracts/migrations/2_registry_migration.js | 4 +++- .../migrations/4_user_replica_set_migration.js | 1 + .../migrations/5_audius_data_migration.js | 18 ++++++++++++++++++ contracts/scripts/migrate-contracts.js | 18 ++++++++++-------- .../src/commands/service-commands.json | 4 ++-- service-commands/src/setup.js | 3 --- 7 files changed, 35 insertions(+), 14 deletions(-) diff --git a/contracts/.gitignore b/contracts/.gitignore index bfff7097c44..c9cde33d645 100644 --- a/contracts/.gitignore +++ b/contracts/.gitignore @@ -4,5 +4,6 @@ .DS_Store .npmrc ganache_log.txt +migrations/migration-output.json node_modules/ build/ diff --git a/contracts/migrations/2_registry_migration.js b/contracts/migrations/2_registry_migration.js index f57ae04d8ae..f288596cc65 100644 --- a/contracts/migrations/2_registry_migration.js +++ b/contracts/migrations/2_registry_migration.js @@ -2,6 +2,8 @@ const Registry = artifacts.require('Registry') module.exports = (deployer) => { deployer.then(async () => { - await deployer.deploy(Registry) + let registryDeployTx = await deployer.deploy(Registry) + // Set environment variable + process.env.dataContractsRegistryAddress = registryDeployTx.address }) } diff --git a/contracts/migrations/4_user_replica_set_migration.js b/contracts/migrations/4_user_replica_set_migration.js index 35da1d93aaa..30c941fa995 100644 --- a/contracts/migrations/4_user_replica_set_migration.js +++ b/contracts/migrations/4_user_replica_set_migration.js @@ -107,5 +107,6 @@ module.exports = (deployer, network, accounts) => { ) seedComplete = await userReplicaSetManagerInst.getSeedComplete({ from: userReplicaSetBootstrapAddress }) console.log(`Seed complete: ${seedComplete}`) + process.env.dataContractsUrsmAddress = deployedProxyTx.address }) } diff --git a/contracts/migrations/5_audius_data_migration.js b/contracts/migrations/5_audius_data_migration.js index 97a822f88a8..6b54b52ce47 100644 --- a/contracts/migrations/5_audius_data_migration.js +++ b/contracts/migrations/5_audius_data_migration.js @@ -1,3 +1,5 @@ +const fs = require('fs-extra') +const path = require('path') const contractConfig = require('../contract-config.js') const abi = require('ethereumjs-abi') const AudiusData = artifacts.require('AudiusData') @@ -40,5 +42,21 @@ module.exports = (deployer, network, accounts) => { ) let audiusDataProxyAddress = deployedProxyTx.address console.log(`AudiusData Proxy Contract deployed at ${audiusDataProxyAddress}`) + process.env.dataContractsAudiusDataProxyAddress = audiusDataProxyAddress + const outputFilePath = path.join(__dirname, 'migration-output.json') + fs.removeSync(outputFilePath) + const registryAddress = process.env.dataContractsRegistryAddress + const ursmAddress = process.env.dataContractsUrsmAddress + const outputValues = { + audiusDataProxyAddress, + registryAddress, + ursmAddress + } + console.log(`Migration output values: ${JSON.stringify(outputValues)}`) + fs.writeFile(outputFilePath, JSON.stringify(outputValues), (err) => { + if (err != null) { + console.log(err) + } + }) }) } diff --git a/contracts/scripts/migrate-contracts.js b/contracts/scripts/migrate-contracts.js index 612d4c32552..2e75a365f1e 100644 --- a/contracts/scripts/migrate-contracts.js +++ b/contracts/scripts/migrate-contracts.js @@ -13,16 +13,13 @@ const fs = require('fs-extra') const path = require('path') const os = require('os'); -const Registry = artifacts.require('Registry') const truffle_dev_config = artifacts.options['_values']['networks']['development'] const AudiusLibs = 'libs' const AudiusDiscoveryNode = 'discovery-provider' const AudiusIdentityService = 'identity-service' const AudiusCreatorNode = 'creator-node' - -let registry - +const AudiusDataContracts = 'contracts' const getDefaultAccount = async () => { let accounts = await web3.eth.getAccounts() @@ -77,9 +74,12 @@ const copySignatureSchemas = (outputPath) => { const outputJsonConfigFile = async (outputPath) => { try{ - registry = await Registry.deployed() + let migrationOutputPath = path.join(getDirectoryRoot(AudiusDataContracts), 'migrations', 'migration-output.json') + let addressInfo = require(migrationOutputPath) let outputDictionary = {} - outputDictionary['registryAddress'] = registry.address + outputDictionary['registryAddress'] = addressInfo.registryAddress + outputDictionary['audiusDataAddress'] = addressInfo.audiusDataProxyAddress + outputDictionary['ursmAddress'] = addressInfo.ursmAddress outputDictionary['ownerWallet'] = await getDefaultAccount() outputDictionary['allWallets'] = await web3.eth.getAccounts() @@ -100,10 +100,12 @@ const outputJsonConfigFile = async (outputPath) => { */ const outputFlaskConfigFile = async (outputPath) => { try { - registry = await Registry.deployed() + // registry = await Registry.deployed() + let migrationOutputPath = path.join(getDirectoryRoot(AudiusDataContracts), 'migrations', 'migration-output.json') + let addressInfo = require(migrationOutputPath) let configFileContents = '[contracts]\n' - configFileContents += 'registry = ' + registry.address + '\n' + configFileContents += 'registry = ' + addressInfo.registryAddress + '\n' configFileContents += '\n' diff --git a/service-commands/src/commands/service-commands.json b/service-commands/src/commands/service-commands.json index 6f42f93b1dc..af6ed13b9ce 100644 --- a/service-commands/src/commands/service-commands.json +++ b/service-commands/src/commands/service-commands.json @@ -33,7 +33,7 @@ "echo 'Waiting for ganache to fully come online...'", "sleep 10", "echo 'Migrating contracts'", - "cd contracts/; node_modules/.bin/truffle migrate --f 1 --to 3 >> $PROTOCOL_DIR/service-commands/output.log 2>$PROTOCOL_DIR/service-commands/error.log", + "cd contracts/; node_modules/.bin/truffle migrate >> $PROTOCOL_DIR/service-commands/output.log 2>$PROTOCOL_DIR/service-commands/error.log", "echo 'Writing configs'", "cd contracts/; node_modules/.bin/truffle exec scripts/migrate-contracts.js" ], @@ -43,7 +43,7 @@ }, "contracts-predeployed": { "up": [ - "docker run --name audius_ganache_cli -d -p 8545:8545 --network=audius_dev audius/ganache:data-contracts-predeployed-4d4a2073fb18b071f7dcf01fdcc4b26001162f48", + "docker run --name audius_ganache_cli -d -p 8545:8545 --network=audius_dev audius/ganache:data-contracts-predeployed-7a65bc3a54d5f84e1be805892935e121574b0772", "docker cp audius_ganache_cli:/app/contract-config.json $PROTOCOL_DIR/identity-service/contract-config.json", "docker cp audius_ganache_cli:/app/contract-config.json $PROTOCOL_DIR/creator-node/contract-config.json", "docker cp audius_ganache_cli:/app/contract-config.json ~/.audius/contract-config.json", diff --git a/service-commands/src/setup.js b/service-commands/src/setup.js index 1031b7f3b87..6da402a191c 100644 --- a/service-commands/src/setup.js +++ b/service-commands/src/setup.js @@ -849,9 +849,6 @@ const allUp = async ({ await runInSequence(nodeRegisterCommands.flat()) } - // URSM has to up after Creator Nodes are registered - await runInSequence([[Service.USER_REPLICA_SET_MANAGER, SetupCommand.UP]]) - const durationSeconds = Math.abs((Date.now() - start) / 1000) console.log(`All services brought up in ${durationSeconds}s`.happy) } From cc38f1640b165ef85648cd10dbe363d64a7c8d1f Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Wed, 6 Jul 2022 00:18:34 +0000 Subject: [PATCH 037/101] Improved example --- service-commands/scripts/bootstrap-eth-contracts-docker.js | 1 + 1 file changed, 1 insertion(+) diff --git a/service-commands/scripts/bootstrap-eth-contracts-docker.js b/service-commands/scripts/bootstrap-eth-contracts-docker.js index 3f1780e4170..e05eb629e72 100644 --- a/service-commands/scripts/bootstrap-eth-contracts-docker.js +++ b/service-commands/scripts/bootstrap-eth-contracts-docker.js @@ -159,6 +159,7 @@ const main = async () => { ) // TODO: Deploy to docker registry (dockerhub) + // Ex. After local tag - docker push audius/ganache:data-contracts-predeployed-7a65bc3a54d5f84e1be805892935e121574b0772 console.log('Be sure to publish the docker file if using externally') console.log( 'ie. "docker push audius/ganache:data-contracts-predeployed-latest"' From 7eb8751ef458cb0aaf8498159fc05d3dcd5852c1 Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Wed, 6 Jul 2022 02:03:22 +0000 Subject: [PATCH 038/101] Local disc prov config --- libs/initScripts/configureLocalDiscProv.js | 25 ++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/libs/initScripts/configureLocalDiscProv.js b/libs/initScripts/configureLocalDiscProv.js index 353a3fb1fc7..f451ed5ae52 100644 --- a/libs/initScripts/configureLocalDiscProv.js +++ b/libs/initScripts/configureLocalDiscProv.js @@ -2,9 +2,12 @@ const fs = require('fs') const path = require('path') const readline = require('readline') const ethContractsMigrationOutput = require('../../eth-contracts/migrations/migration-output.json') +const dataContractsMigrationOutput = require('../../contracts/migrations/migration-output.json') const solanaConfig = require('../../solana-programs/solana-program-config.json') const ETH_CONTRACTS_REGISTRY = 'audius_eth_contracts_registry' +const CONTRACTS_REGISTRY = 'audius_contracts_registry' +const CONTRACTS_DATA_ADDRESS = 'audius_contracts_data_address' const SOLANA_TRACK_LISTEN_COUNT_ADDRESS = 'audius_solana_track_listen_count_address' const SOLANA_ENDPOINT = 'audius_solana_endpoint' @@ -23,6 +26,8 @@ const SOLANA_ANCHOR_ADMIN_STORAGE_PUBLIC_KEY = 'audius_solana_anchor_admin_stora // Updates audius_eth_contracts_registry in discovery provider const configureLocalDiscProv = async () => { const ethRegistryAddress = ethContractsMigrationOutput.registryAddress + const dataRegistryAddress = dataContractsMigrationOutput.registryAddress + const dataContractAddress = dataContractsMigrationOutput.audiusDataProxyAddress const solanaTrackListenCountAddress = solanaConfig.trackListenCountAddress const signerGroup = solanaConfig.signerGroup const solanaEndpoint = solanaConfig.endpoint @@ -48,6 +53,8 @@ const configureLocalDiscProv = async () => { rewardsManagerAccount, anchorProgramId, anchorAdminStoragePublicKey, + dataRegistryAddress, + dataContractAddress ) } @@ -65,6 +72,8 @@ const _updateDiscoveryProviderEnvFile = async ( rewardsManagerAccount, anchorProgramId, anchorAdminStoragePublicKey, + dataRegistryAddress, + dataContractAddress ) => { const fileStream = fs.createReadStream(readPath) const rl = readline.createInterface({ @@ -83,6 +92,8 @@ const _updateDiscoveryProviderEnvFile = async ( let rewardsAccountFound = false let anchorProgramIdFound = false let anchorAdminStoragePublicKeyFound = false + let dataRegistryLineFound = false + let dataContractLineFound = false const ethRegistryAddressLine = `${ETH_CONTRACTS_REGISTRY}=${ethRegistryAddress}` const solanaTrackListenCountAddressLine = `${SOLANA_TRACK_LISTEN_COUNT_ADDRESS}=${solanaTrackListenCountAddress}` @@ -94,6 +105,8 @@ const _updateDiscoveryProviderEnvFile = async ( const rewardsManagerAccountLine = `${SOLANA_REWARDS_MANAGER_ACCOUNT}=${rewardsManagerAccount}` const anchorProgramIdLine = `${SOLANA_ANCHOR_PROGRAM_ID}=${anchorProgramId}` const anchorAdminStoragePublicKeyLine = `${SOLANA_ANCHOR_ADMIN_STORAGE_PUBLIC_KEY}=${anchorAdminStoragePublicKey}` + const dataRegistryLine = `${CONTRACTS_REGISTRY}=${dataRegistryAddress}` + const dataContractLine = `${CONTRACTS_DATA_ADDRESS}=${dataContractAddress}` for await (const line of rl) { if (line.includes(ETH_CONTRACTS_REGISTRY)) { @@ -126,6 +139,12 @@ const _updateDiscoveryProviderEnvFile = async ( } else if (line.includes(SOLANA_ANCHOR_ADMIN_STORAGE_PUBLIC_KEY)) { output.push(anchorAdminStoragePublicKeyLine) anchorAdminStoragePublicKeyFound = true + } else if (line.includes(CONTRACTS_REGISTRY)) { + output.push(dataRegistryLine) + dataRegistryLineFound = true + } else if (line.includes(CONTRACTS_DATA_ADDRESS)) { + output.push(dataContractLine) + dataContractLineFound = true } else { output.push(line) } @@ -160,6 +179,12 @@ const _updateDiscoveryProviderEnvFile = async ( if (!anchorAdminStoragePublicKeyFound) { output.push(anchorAdminStoragePublicKeyLine) } + if (!dataRegistryLineFound) { + output.push(dataRegistryLine) + } + if (!dataContractLineFound) { + output.push(dataContractLine) + } fs.writeFileSync(writePath, output.join('\n')) console.log(`Updated DISCOVERY PROVIDER ${writePath} ${ETH_CONTRACTS_REGISTRY}=${ethRegistryAddress} ${output}`) } From b9a13c9a88fcd6569d024fee88dd17b9d08939f7 Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Wed, 6 Jul 2022 02:06:33 +0000 Subject: [PATCH 039/101] Pull address from config --- discovery-provider/src/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/discovery-provider/src/app.py b/discovery-provider/src/app.py index 4d8bcb42260..a5c179a0937 100644 --- a/discovery-provider/src/app.py +++ b/discovery-provider/src/app.py @@ -136,7 +136,7 @@ def init_contracts(): abi=abi_values["UserReplicaSetManager"]["abi"], ) - audius_data_address = "0xaf5C4C6C7920B4883bC6252e9d9B8fE27187Cf68" + audius_data_address = web3.toChecksumAddress(shared_config["contracts"]["data_address"]) audius_data_inst = web3.eth.contract( address=audius_data_address, abi=abi_values["AudiusData"]["abi"] ) From e9c7a2d44594a92ef15ce18b100b21319c8a079b Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Wed, 6 Jul 2022 02:37:31 +0000 Subject: [PATCH 040/101] More e2e libs work --- libs/src/libs.js | 8 ++++++-- libs/src/services/dataContracts/AudiusContracts.ts | 1 + 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/libs/src/libs.js b/libs/src/libs.js index 6a0cff850b8..e182022bcab 100644 --- a/libs/src/libs.js +++ b/libs/src/libs.js @@ -93,7 +93,8 @@ class AudiusLibs { registryAddress, web3Provider, networkId, - walletOverride = null + walletOverride = null, + dataContractAddress = null ) { const web3Instance = await Utils.configureWeb3(web3Provider, networkId) if (!web3Instance) { @@ -102,6 +103,7 @@ class AudiusLibs { const wallets = await web3Instance.eth.getAccounts() return { registryAddress, + dataContractAddress, useExternalWeb3: true, externalWeb3Config: { web3: web3Instance, @@ -115,7 +117,7 @@ class AudiusLibs { * @param {string} registryAddress * @param {string | Web3 | Array} providers web3 provider endpoint(s) */ - static configInternalWeb3 (registryAddress, providers, privateKey) { + static configInternalWeb3 (registryAddress, providers, privateKey, dataContractAddress = null) { let providerList if (typeof providers === 'string') { providerList = providers.split(',') @@ -131,6 +133,7 @@ class AudiusLibs { return { registryAddress, + dataContractAddress, useExternalWeb3: false, internalWeb3Config: { web3ProviderEndpoints: providerList, @@ -461,6 +464,7 @@ class AudiusLibs { this.contracts = new AudiusContracts( this.web3Manager, this.web3Config ? this.web3Config.registryAddress : null, + this.web3Config ? this.web3Config.dataContractAddress: null, this.isServer, this.logger ) diff --git a/libs/src/services/dataContracts/AudiusContracts.ts b/libs/src/services/dataContracts/AudiusContracts.ts index d9561a56d81..f56c6f4f70b 100644 --- a/libs/src/services/dataContracts/AudiusContracts.ts +++ b/libs/src/services/dataContracts/AudiusContracts.ts @@ -65,6 +65,7 @@ export class AudiusContracts { constructor( web3Manager: Web3Manager, registryAddress: string, + dataAddress: string, isServer: boolean, logger = console ) { From 479330f6a8e524e894f1bfd46782187a166545aa Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Wed, 6 Jul 2022 02:38:50 +0000 Subject: [PATCH 041/101] Black formatting --- discovery-provider/src/app.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/discovery-provider/src/app.py b/discovery-provider/src/app.py index a5c179a0937..9c8f74a0ae8 100644 --- a/discovery-provider/src/app.py +++ b/discovery-provider/src/app.py @@ -136,7 +136,9 @@ def init_contracts(): abi=abi_values["UserReplicaSetManager"]["abi"], ) - audius_data_address = web3.toChecksumAddress(shared_config["contracts"]["data_address"]) + audius_data_address = web3.toChecksumAddress( + shared_config["contracts"]["data_address"] + ) audius_data_inst = web3.eth.contract( address=audius_data_address, abi=abi_values["AudiusData"]["abi"] ) From 01965745ffe2c1b163c1f496c46372b792693523 Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Wed, 6 Jul 2022 02:41:01 +0000 Subject: [PATCH 042/101] Fix ref --- discovery-provider/src/tasks/audius_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/discovery-provider/src/tasks/audius_data.py b/discovery-provider/src/tasks/audius_data.py index fcba2134d0d..5e12ae3e727 100644 --- a/discovery-provider/src/tasks/audius_data.py +++ b/discovery-provider/src/tasks/audius_data.py @@ -4,7 +4,7 @@ from sqlalchemy.orm.session import Session, make_transient from src.database_task import DatabaseTask -from src.models import Playlist +from src.models.playlists import Playlist from src.tasks.playlists import invalidate_old_playlist from src.utils import helpers from src.utils.user_event_constants import audius_data_event_types_arr From 701664dd14628683127224fbafb71d62b0ca4ab0 Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Wed, 6 Jul 2022 02:42:13 +0000 Subject: [PATCH 043/101] NIT --- discovery-provider/src/tasks/audius_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/discovery-provider/src/tasks/audius_data.py b/discovery-provider/src/tasks/audius_data.py index 5e12ae3e727..75c901e71c5 100644 --- a/discovery-provider/src/tasks/audius_data.py +++ b/discovery-provider/src/tasks/audius_data.py @@ -4,7 +4,7 @@ from sqlalchemy.orm.session import Session, make_transient from src.database_task import DatabaseTask -from src.models.playlists import Playlist +from src.models.playlists.playlist import Playlist from src.tasks.playlists import invalidate_old_playlist from src.utils import helpers from src.utils.user_event_constants import audius_data_event_types_arr From 9c68d72314f870c1958b1394dbecf0bd18bb59a3 Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Wed, 6 Jul 2022 15:02:27 +0000 Subject: [PATCH 044/101] Adding new config --- discovery-provider/default_config.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/discovery-provider/default_config.ini b/discovery-provider/default_config.ini index 722e1c71429..52a8e27878b 100644 --- a/discovery-provider/default_config.ini +++ b/discovery-provider/default_config.ini @@ -73,6 +73,7 @@ allow_all = false [contracts] registry = +data_address = [eth_contracts] registry = From 747da8de88bda973dc6125309471a52f7c22d8d3 Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Wed, 6 Jul 2022 16:29:48 +0000 Subject: [PATCH 045/101] Fixing local errors - e2e w/configs should be functioning now --- discovery-provider/src/tasks/audius_data.py | 2 +- libs/src/services/dataContracts/AudiusContracts.ts | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/discovery-provider/src/tasks/audius_data.py b/discovery-provider/src/tasks/audius_data.py index 75c901e71c5..64c01fd67f5 100644 --- a/discovery-provider/src/tasks/audius_data.py +++ b/discovery-provider/src/tasks/audius_data.py @@ -167,7 +167,7 @@ def parse_playlist_create_data_event( block_datetime = datetime.utcfromtimestamp(block_timestamp) block_integer_time = int(block_timestamp) playlist_record.playlist_owner_id = playlist_owner_id - playlist_record.is_album = playlist_metadata["is_album"] + playlist_record.is_album = playlist_metadata["is_album"] if "is_album" in playlist_metadata else False playlist_record.description = playlist_metadata["description"] playlist_record.playlist_image_multihash = playlist_metadata[ "playlist_image_sizes_multihash" diff --git a/libs/src/services/dataContracts/AudiusContracts.ts b/libs/src/services/dataContracts/AudiusContracts.ts index f56c6f4f70b..cf17d19a2b9 100644 --- a/libs/src/services/dataContracts/AudiusContracts.ts +++ b/libs/src/services/dataContracts/AudiusContracts.ts @@ -47,6 +47,7 @@ const UserReplicaSetManagerRegistryKey = 'UserReplicaSetManager' export class AudiusContracts { web3Manager: Web3Manager registryAddress: string + dataAddress: string isServer: boolean logger: Logger RegistryClient: RegistryClient @@ -71,6 +72,7 @@ export class AudiusContracts { ) { this.web3Manager = web3Manager this.registryAddress = registryAddress + this.dataAddress = dataAddress this.isServer = isServer this.logger = logger @@ -136,7 +138,7 @@ export class AudiusContracts { 'AudiusData', this.getRegistryAddressForContract, this.logger, - '0xaf5C4C6C7920B4883bC6252e9d9B8fE27187Cf68' // TODO: Config var + this.dataAddress ) this.contractClients = [ From cb1ddec8301e4571ec4faea27194a2f05e757926 Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Wed, 6 Jul 2022 16:38:37 +0000 Subject: [PATCH 046/101] Fix metadata parsing to match --- discovery-provider/src/tasks/audius_data.py | 4 +++- discovery-provider/src/tasks/metadata.py | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/discovery-provider/src/tasks/audius_data.py b/discovery-provider/src/tasks/audius_data.py index 64c01fd67f5..6b8b24352b5 100644 --- a/discovery-provider/src/tasks/audius_data.py +++ b/discovery-provider/src/tasks/audius_data.py @@ -167,7 +167,9 @@ def parse_playlist_create_data_event( block_datetime = datetime.utcfromtimestamp(block_timestamp) block_integer_time = int(block_timestamp) playlist_record.playlist_owner_id = playlist_owner_id - playlist_record.is_album = playlist_metadata["is_album"] if "is_album" in playlist_metadata else False + playlist_record.is_album = ( + playlist_metadata["is_album"] if "is_album" in playlist_metadata else False + ) playlist_record.description = playlist_metadata["description"] playlist_record.playlist_image_multihash = playlist_metadata[ "playlist_image_sizes_multihash" diff --git a/discovery-provider/src/tasks/metadata.py b/discovery-provider/src/tasks/metadata.py index d346e15a8ac..2470f7225f8 100644 --- a/discovery-provider/src/tasks/metadata.py +++ b/discovery-provider/src/tasks/metadata.py @@ -52,5 +52,6 @@ "playlist_name": None, "playlist_image_sizes_multihash": None, "description": None, - "isAlbum": None, + "is_album": None, + "is_private": None, } From f1f292e300be32fe3e6f7f28df9507501b2f2845 Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Wed, 6 Jul 2022 22:14:51 +0000 Subject: [PATCH 047/101] Save migration output for data contracts --- service-commands/scripts/bootstrap-eth-contracts-docker.js | 6 +++++- service-commands/src/commands/service-commands.json | 5 +++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/service-commands/scripts/bootstrap-eth-contracts-docker.js b/service-commands/scripts/bootstrap-eth-contracts-docker.js index e05eb629e72..d3c2ae7ee5c 100644 --- a/service-commands/scripts/bootstrap-eth-contracts-docker.js +++ b/service-commands/scripts/bootstrap-eth-contracts-docker.js @@ -105,7 +105,10 @@ const main = async () => { `docker cp audius_ganache_cli:/app/contracts-ganache-accounts.json ${tmpDataContracts}/contracts-ganache-accounts.json`, { stdio: 'inherit' } ) - + execSync( + `cp -r ${process.env.PROTOCOL_DIR}/contracts/migrations/migration-output.json ${tmpDataContracts}`, + { stdio: 'inherit' } + ) execSync( `cp -r ${process.env.PROTOCOL_DIR}/contracts/build ${tmpDataContracts}`, { stdio: 'inherit' } @@ -159,6 +162,7 @@ const main = async () => { ) // TODO: Deploy to docker registry (dockerhub) + // Be sure to run 'A network up' to create local docker network before running this file // Ex. After local tag - docker push audius/ganache:data-contracts-predeployed-7a65bc3a54d5f84e1be805892935e121574b0772 console.log('Be sure to publish the docker file if using externally') console.log( diff --git a/service-commands/src/commands/service-commands.json b/service-commands/src/commands/service-commands.json index c8354001c5e..b70dce67fdc 100644 --- a/service-commands/src/commands/service-commands.json +++ b/service-commands/src/commands/service-commands.json @@ -43,7 +43,7 @@ }, "contracts-predeployed": { "up": [ - "docker run --name audius_ganache_cli -d -p 8545:8545 --network=audius_dev audius/ganache:data-contracts-predeployed-7a65bc3a54d5f84e1be805892935e121574b0772", + "docker run --name audius_ganache_cli -d -p 8545:8545 --network=audius_dev audius/ganache:data-contracts-predeployed-a25931b955adce92676cec029661533f80013908", "docker cp audius_ganache_cli:/app/contract-config.json $PROTOCOL_DIR/identity-service/contract-config.json", "docker cp audius_ganache_cli:/app/contract-config.json $PROTOCOL_DIR/creator-node/contract-config.json", "docker cp audius_ganache_cli:/app/contract-config.json ~/.audius/contract-config.json", @@ -52,7 +52,8 @@ "docker cp audius_ganache_cli:/app/contract_config.ini $PROTOCOL_DIR/discovery-provider/contract_config.ini", "docker cp audius_ganache_cli:/app/contracts/. $PROTOCOL_DIR/discovery-provider/build/contracts", "docker cp audius_ganache_cli:/app/contracts/. $PROTOCOL_DIR/libs/data-contracts/ABIs", - "docker cp audius_ganache_cli:/app/build/contracts/. $PROTOCOL_DIR/contracts/build" + "docker cp audius_ganache_cli:/app/build/contracts/. $PROTOCOL_DIR/contracts/build", + "docker cp audius_ganache_cli:/app/migration-output.json $PROTOCOL_DIR/contracts/migrations/migration-output.json" ], "protocol": "http", "host": "localhost", From 33ce72b6f55158d8f66476d2a26b54112b3a2ad7 Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Thu, 7 Jul 2022 00:38:53 +0000 Subject: [PATCH 048/101] ABI nit --- libs/data-contracts/ABIs/AudiusData.json | 346 +++++++++++------------ 1 file changed, 173 insertions(+), 173 deletions(-) diff --git a/libs/data-contracts/ABIs/AudiusData.json b/libs/data-contracts/ABIs/AudiusData.json index 212b135f2fc..02f5b9651d1 100644 --- a/libs/data-contracts/ABIs/AudiusData.json +++ b/libs/data-contracts/ABIs/AudiusData.json @@ -1,174 +1,174 @@ { - "contractName": "AudiusData", - "abi": [ - { - "constant": true, - "inputs": [ - { - "name": "", - "type": "bytes32" - } - ], - "name": "usedSignatures", - "outputs": [ - { - "name": "", - "type": "bool" - } - ], - "payable": false, - "stateMutability": "view", - "type": "function" - }, - { - "anonymous": false, - "inputs": [ - { - "indexed": false, - "name": "_userId", - "type": "uint256" - }, - { - "indexed": false, - "name": "_signer", - "type": "address" - }, - { - "indexed": false, - "name": "_entityType", - "type": "string" - }, - { - "indexed": false, - "name": "_entityId", - "type": "uint256" - }, - { - "indexed": false, - "name": "_metadata", - "type": "string" - }, - { - "indexed": false, - "name": "_action", - "type": "string" - } - ], - "name": "ManageEntity", - "type": "event" - }, - { - "anonymous": false, - "inputs": [ - { - "indexed": false, - "name": "_userId", - "type": "uint256" - }, - { - "indexed": false, - "name": "_isVerified", - "type": "bool" - } - ], - "name": "ManageIsVerified", - "type": "event" - }, - { - "constant": false, - "inputs": [ - { - "name": "name", - "type": "string" - }, - { - "name": "version", - "type": "string" - }, - { - "name": "chainId", - "type": "uint256" - } - ], - "name": "initialize", - "outputs": [], - "payable": false, - "stateMutability": "nonpayable", - "type": "function" - }, - { - "constant": false, - "inputs": [ - { - "name": "_verifierAddress", - "type": "address" - }, - { - "name": "_networkId", - "type": "uint256" - } - ], - "name": "initialize", - "outputs": [], - "payable": false, - "stateMutability": "nonpayable", - "type": "function" - }, - { - "constant": false, - "inputs": [ - { - "name": "_userId", - "type": "uint256" - }, - { - "name": "_entityType", - "type": "string" - }, - { - "name": "_entityId", - "type": "uint256" - }, - { - "name": "_action", - "type": "string" - }, - { - "name": "_metadata", - "type": "string" - }, - { - "name": "_nonce", - "type": "bytes32" - }, - { - "name": "_subjectSig", - "type": "bytes" - } - ], - "name": "manageEntity", - "outputs": [], - "payable": false, - "stateMutability": "nonpayable", - "type": "function" - }, - { - "constant": false, - "inputs": [ - { - "name": "_userId", - "type": "uint256" - }, - { - "name": "_isVerified", - "type": "bool" - } - ], - "name": "manageIsVerified", - "outputs": [], - "payable": false, - "stateMutability": "nonpayable", - "type": "function" - } - ] - } \ No newline at end of file + "contractName": "AudiusData", + "abi": [ + { + "constant": true, + "inputs": [ + { + "name": "", + "type": "bytes32" + } + ], + "name": "usedSignatures", + "outputs": [ + { + "name": "", + "type": "bool" + } + ], + "payable": false, + "stateMutability": "view", + "type": "function" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": false, + "name": "_userId", + "type": "uint256" + }, + { + "indexed": false, + "name": "_signer", + "type": "address" + }, + { + "indexed": false, + "name": "_entityType", + "type": "string" + }, + { + "indexed": false, + "name": "_entityId", + "type": "uint256" + }, + { + "indexed": false, + "name": "_metadata", + "type": "string" + }, + { + "indexed": false, + "name": "_action", + "type": "string" + } + ], + "name": "ManageEntity", + "type": "event" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": false, + "name": "_userId", + "type": "uint256" + }, + { + "indexed": false, + "name": "_isVerified", + "type": "bool" + } + ], + "name": "ManageIsVerified", + "type": "event" + }, + { + "constant": false, + "inputs": [ + { + "name": "name", + "type": "string" + }, + { + "name": "version", + "type": "string" + }, + { + "name": "chainId", + "type": "uint256" + } + ], + "name": "initialize", + "outputs": [], + "payable": false, + "stateMutability": "nonpayable", + "type": "function" + }, + { + "constant": false, + "inputs": [ + { + "name": "_verifierAddress", + "type": "address" + }, + { + "name": "_networkId", + "type": "uint256" + } + ], + "name": "initialize", + "outputs": [], + "payable": false, + "stateMutability": "nonpayable", + "type": "function" + }, + { + "constant": false, + "inputs": [ + { + "name": "_userId", + "type": "uint256" + }, + { + "name": "_entityType", + "type": "string" + }, + { + "name": "_entityId", + "type": "uint256" + }, + { + "name": "_action", + "type": "string" + }, + { + "name": "_metadata", + "type": "string" + }, + { + "name": "_nonce", + "type": "bytes32" + }, + { + "name": "_subjectSig", + "type": "bytes" + } + ], + "name": "manageEntity", + "outputs": [], + "payable": false, + "stateMutability": "nonpayable", + "type": "function" + }, + { + "constant": false, + "inputs": [ + { + "name": "_userId", + "type": "uint256" + }, + { + "name": "_isVerified", + "type": "bool" + } + ], + "name": "manageIsVerified", + "outputs": [], + "payable": false, + "stateMutability": "nonpayable", + "type": "function" + } + ] +} \ No newline at end of file From 3a7d4241e1277e430391c32e5733026a92cd86fc Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Thu, 7 Jul 2022 14:20:39 +0000 Subject: [PATCH 049/101] Adding schema validation flow --- libs/src/api/audiusData.ts | 1 + libs/src/services/creatorNode/CreatorNode.ts | 9 ++- .../schemaValidator/SchemaValidator.ts | 12 +++- .../schemas/playlistSchema.json | 56 +++++++++++++++++++ 4 files changed, 76 insertions(+), 2 deletions(-) create mode 100644 libs/src/services/schemaValidator/schemas/playlistSchema.json diff --git a/libs/src/api/audiusData.ts b/libs/src/api/audiusData.ts index 46e99ee541b..79972d13d1f 100644 --- a/libs/src/api/audiusData.ts +++ b/libs/src/api/audiusData.ts @@ -10,6 +10,7 @@ export class AudiusData extends Base { /** * Calculate an unoccupied playlist ID * Maximum value is postgres integer max (2147483647) + * Minimum value is artificially set to 400000 */ async getValidPlaylistId(): Promise { let playlistId: number = 10 diff --git a/libs/src/services/creatorNode/CreatorNode.ts b/libs/src/services/creatorNode/CreatorNode.ts index 3e43c50e604..cb00a73b727 100644 --- a/libs/src/services/creatorNode/CreatorNode.ts +++ b/libs/src/services/creatorNode/CreatorNode.ts @@ -5,6 +5,7 @@ import { Utils, uuid } from '../../utils' import { userSchemaType, trackSchemaType, + playlistSchemaType, Schemas } from '../schemaValidator/SchemaValidator' import type { Web3Manager } from '../web3Manager' @@ -27,6 +28,12 @@ type Metadata = { type PlaylistMetadata = { playlist_contents: unknown + playlist_id: number + playlist_name: string + playlist_image_sizes_multihash: string + description: string + is_album: boolean + is_private: boolean } type ProgressCB = (loaded: number, total: number) => void @@ -412,7 +419,7 @@ export class CreatorNode { * @param metadata */ async uploadPlaylistMetadata(metadata: PlaylistMetadata) { - // TODO: Schema validation flow + this.schemas[playlistSchemaType].validate?.(metadata) const { data: body } = await this._makeRequest( { url: '/playlists/metadata', diff --git a/libs/src/services/schemaValidator/SchemaValidator.ts b/libs/src/services/schemaValidator/SchemaValidator.ts index a843db0a67f..a555afcb075 100644 --- a/libs/src/services/schemaValidator/SchemaValidator.ts +++ b/libs/src/services/schemaValidator/SchemaValidator.ts @@ -2,9 +2,11 @@ import { validate } from 'jsonschema' import TrackSchema from './schemas/trackSchema.json' import UserSchema from './schemas/userSchema.json' +import PlaylistSchema from './schemas/playlistSchema.json' export const trackSchemaType = 'TrackSchema' export const userSchemaType = 'UserSchema' +export const playlistSchemaType = 'PlaylistSchema' type SchemaConfig = { schema: { @@ -18,11 +20,15 @@ type SchemaConfig = { validate?: (obj: Record) => void } -type SchemaType = typeof trackSchemaType | typeof userSchemaType +type SchemaType = + | typeof trackSchemaType + | typeof userSchemaType + | typeof playlistSchemaType export type Schemas = { TrackSchema: SchemaConfig UserSchema: SchemaConfig + PlaylistSchema: SchemaConfig } export class SchemaValidator { @@ -50,6 +56,10 @@ export class SchemaValidator { [userSchemaType]: { schema: UserSchema, baseDefinition: 'User' + }, + [playlistSchemaType]: { + schema: PlaylistSchema, + baseDefinition: 'Playlist' } } diff --git a/libs/src/services/schemaValidator/schemas/playlistSchema.json b/libs/src/services/schemaValidator/schemas/playlistSchema.json new file mode 100644 index 00000000000..c7be4f3db81 --- /dev/null +++ b/libs/src/services/schemaValidator/schemas/playlistSchema.json @@ -0,0 +1,56 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$ref": "#/definitions/Playlist", + "definitions": { + "Playlist": { + "type": "object", + "additionalProperties": true, + "properties": { + "playlist_id": { + "type": ["integer", "null"], + "default": null + }, + "playlist_contents": { + "type": ["array", "null"], + "default": [] + }, + "playlist_name": { + "type": ["string", "null"], + "default": null + }, + "playlist_image_sizes_multihash": { + "type": ["string", "null"], + "default": null, + "$ref": "#/definitions/CID" + }, + "description": { + "type": ["string", "null"], + "default": null + }, + "is_album": { + "type": ["string", "null"], + "default": null + }, + "is_private": { + "type": ["string", "null"], + "default": null + } + }, + "required": [ + "playlist_name", + "playlist_id", + "description", + "is_album", + "is_private" + ], + "title": "Playlist" + }, + "CID": { + "type": ["string", "null"], + "minLength": 46, + "maxLength": 46, + "pattern": "^Qm[a-zA-Z0-9]{44}$", + "title": "CID" + } + } +} From fb85c5204cf25b09597aec44c7af4e860f79e733 Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Thu, 7 Jul 2022 14:50:49 +0000 Subject: [PATCH 050/101] Removing base.d.ts --- libs/src/api/audiusData.ts | 6 +++--- libs/src/api/base.d.ts | 4 ---- 2 files changed, 3 insertions(+), 7 deletions(-) delete mode 100644 libs/src/api/base.d.ts diff --git a/libs/src/api/audiusData.ts b/libs/src/api/audiusData.ts index 8c9b4ee28bc..13ce97d46d4 100644 --- a/libs/src/api/audiusData.ts +++ b/libs/src/api/audiusData.ts @@ -1,7 +1,7 @@ -import { Base, Services } from './base' +import { Base, BaseConstructorArgs, Services } from './base' export class AudiusData extends Base { - constructor(...args: any[]) { + constructor(...args: BaseConstructorArgs) { super(...args) this.getValidPlaylistId = this.getValidPlaylistId.bind(this) this.createPlaylist = this.createPlaylist.bind(this) @@ -53,7 +53,7 @@ export class AudiusData extends Base { playlistId: 0 } try { - const ownerId: number = this.userStateManager.getCurrentUserId() + const ownerId: number = parseInt(this.userStateManager.getCurrentUserId()) const action = 'Create' const entityType = 'Playlist' const entityId = await this.getValidPlaylistId() diff --git a/libs/src/api/base.d.ts b/libs/src/api/base.d.ts deleted file mode 100644 index 67cedbc508a..00000000000 --- a/libs/src/api/base.d.ts +++ /dev/null @@ -1,4 +0,0 @@ -// Placeholder type for base.js -declare const Base: any -declare const Services: any -export { Base, Services } From dcda6043f9129a4bf213ac403529532a7bb8fad2 Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Thu, 7 Jul 2022 14:52:50 +0000 Subject: [PATCH 051/101] LINT --- libs/src/libs.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/src/libs.js b/libs/src/libs.js index c125c497c67..e8a1d2f4fab 100644 --- a/libs/src/libs.js +++ b/libs/src/libs.js @@ -464,7 +464,7 @@ class AudiusLibs { this.contracts = new AudiusContracts( this.web3Manager, this.web3Config ? this.web3Config.registryAddress : null, - this.web3Config ? this.web3Config.dataContractAddress: null, + this.web3Config ? this.web3Config.dataContractAddress : null, this.isServer, this.logger ) From f4ca0d75d153cf333d52d809de1d07f3ff1651a5 Mon Sep 17 00:00:00 2001 From: Isaac Solo Date: Thu, 7 Jul 2022 13:12:20 -0700 Subject: [PATCH 052/101] Add delete playlist to audius data libs + indexing (#3403) --- discovery-provider/src/tasks/audius_data.py | 33 +++++++++++------- libs/src/api/audiusData.ts | 38 +++++++++++++++++++++ 2 files changed, 59 insertions(+), 12 deletions(-) diff --git a/discovery-provider/src/tasks/audius_data.py b/discovery-provider/src/tasks/audius_data.py index 6b8b24352b5..bf246cde827 100644 --- a/discovery-provider/src/tasks/audius_data.py +++ b/discovery-provider/src/tasks/audius_data.py @@ -60,7 +60,7 @@ def audius_data_state_update( ) # Handle playlist creation - if entity_type == "Playlist" and action == "Create": + if entity_type == "Playlist": playlist_id = entity_id # look up or populate existing record if playlist_id in playlist_events_lookup: @@ -77,15 +77,21 @@ def audius_data_state_update( txhash, ) - playlist_record = parse_playlist_create_data_event( - update_task, - entry, - user_id, - existing_playlist_record, - metadata, - block_timestamp, - session, - ) + if action == "Create": + playlist_id = entity_id + playlist_record = parse_playlist_create_data_event( + update_task, + entry, + user_id, + existing_playlist_record, + metadata, + block_timestamp, + session, + ) + + elif action == "Delete": + existing_playlist_record.is_delete = True + playlist_record = existing_playlist_record if playlist_record is not None: if playlist_id not in playlist_events_lookup: @@ -98,8 +104,11 @@ def audius_data_state_update( "playlist" ] = playlist_record playlist_events_lookup[playlist_id]["events"].append(event_type) - playlist_ids.add(playlist_id) - processed_entries += 1 + + playlist_ids.add(playlist_id) + + processed_entries += 1 + num_total_changes += processed_entries # Update changed entity dictionary diff --git a/libs/src/api/audiusData.ts b/libs/src/api/audiusData.ts index 79972d13d1f..1dfceb80a0b 100644 --- a/libs/src/api/audiusData.ts +++ b/libs/src/api/audiusData.ts @@ -5,6 +5,7 @@ export class AudiusData extends Base { super(...args) this.getValidPlaylistId = this.getValidPlaylistId.bind(this) this.createPlaylist = this.createPlaylist.bind(this) + this.deletePlaylist = this.deletePlaylist.bind(this) } /** @@ -96,6 +97,43 @@ export class AudiusData extends Base { return respValues } + /** + * Delete a playlist using updated data contracts flow + */ + async deletePlaylist({ + playlistId, + userId, + logger = console + }: { + playlistId: number, + userId: number, + logger: any + }): Promise<{ blockHash: any; blockNumber: any; }> { + let respValues = { + blockHash: null, + blockNumber: null, + } + try { + let resp = await this.manageEntity({ + userId, + entityType: 'Playlist', + entityId: playlistId, + action: 'Delete', + metadataMultihash: '' + }) + logger.info(`DeletePlaylistData - ${JSON.stringify(resp)}`) + let txReceipt = resp.txReceipt + respValues = { + blockHash: txReceipt.blockHash, + blockNumber: txReceipt.blockNumber, + } + } catch (e) { + logger.error(`Data delete playlist: err ${e}`) + } + return respValues + } + + /** * Manage an entity with the updated data contract flow * Leveraged to manipulate User/Track/Playlist/+ other entities From c81f1c083b4c85e5cca3899407392341f2b4bb30 Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Fri, 8 Jul 2022 14:52:15 +0000 Subject: [PATCH 053/101] Propagating name change --- libs/src/api/{audiusData.ts => entityManager.ts} | 9 +++++++-- libs/src/libs.js | 14 +++++++------- libs/src/services/dataContracts/AudiusContracts.ts | 8 ++++---- ...{AudiusDataClient.ts => EntityManagerClient.ts} | 2 +- .../schemaValidator/schemas/playlistSchema.json | 4 ++-- 5 files changed, 21 insertions(+), 16 deletions(-) rename libs/src/api/{audiusData.ts => entityManager.ts} (93%) rename libs/src/services/dataContracts/{AudiusDataClient.ts => EntityManagerClient.ts} (95%) diff --git a/libs/src/api/audiusData.ts b/libs/src/api/entityManager.ts similarity index 93% rename from libs/src/api/audiusData.ts rename to libs/src/api/entityManager.ts index 336d424d468..479a20ea113 100644 --- a/libs/src/api/audiusData.ts +++ b/libs/src/api/entityManager.ts @@ -1,6 +1,11 @@ import { Base, BaseConstructorArgs, Services } from './base' -export class AudiusData extends Base { +/* + API surface for updated data contract interactions. + Provides simplified entity management in a generic fashion + Handles metadata + file upload etc. for entities such as Playlist/Track/User +*/ +export class EntityManager extends Base { constructor(...args: BaseConstructorArgs) { super(...args) this.getValidPlaylistId = this.getValidPlaylistId.bind(this) @@ -154,7 +159,7 @@ export class AudiusData extends Base { let error: string = '' let resp: any try { - resp = await this.contracts.AudiusDataClient.manageEntity( + resp = await this.contracts.EntityManagerClient.manageEntity( userId, entityType, entityId, diff --git a/libs/src/libs.js b/libs/src/libs.js index 7a50863f083..8aec4413e4b 100644 --- a/libs/src/libs.js +++ b/libs/src/libs.js @@ -34,7 +34,7 @@ const { RewardsAttester } = require('./services/solanaWeb3Manager/rewardsAttester') const { Reactions } = require('./api/reactions') -const { AudiusData } = require('./api/audiusData') +const { EntityManager } = require('./api/audiusData') class AudiusLibs { /** @@ -94,7 +94,7 @@ class AudiusLibs { web3Provider, networkId, walletOverride = null, - dataContractAddress = null + entityManagerAddress = null ) { const web3Instance = await Utils.configureWeb3(web3Provider, networkId) if (!web3Instance) { @@ -103,7 +103,7 @@ class AudiusLibs { const wallets = await web3Instance.eth.getAccounts() return { registryAddress, - dataContractAddress, + entityManagerAddress, useExternalWeb3: true, externalWeb3Config: { web3: web3Instance, @@ -117,7 +117,7 @@ class AudiusLibs { * @param {string} registryAddress * @param {string | Web3 | Array} providers web3 provider endpoint(s) */ - static configInternalWeb3 (registryAddress, providers, privateKey, dataContractAddress = null) { + static configInternalWeb3 (registryAddress, providers, privateKey, entityManagerAddress = null) { let providerList if (typeof providers === 'string') { providerList = providers.split(',') @@ -133,7 +133,7 @@ class AudiusLibs { return { registryAddress, - dataContractAddress, + entityManagerAddress, useExternalWeb3: false, internalWeb3Config: { web3ProviderEndpoints: providerList, @@ -464,7 +464,7 @@ class AudiusLibs { this.contracts = new AudiusContracts( this.web3Manager, this.web3Config ? this.web3Config.registryAddress : null, - this.web3Config ? this.web3Config.dataContractAddress : null, + this.web3Config ? this.web3Config.entityManagerAddress : null, this.isServer, this.logger ) @@ -562,7 +562,7 @@ class AudiusLibs { this.File = new File(this.User, ...services) this.Rewards = new Rewards(this.ServiceProvider, ...services) this.Reactions = new Reactions(...services) - this.AudiusData = new AudiusData(...services) + this.EntityManager = new EntityManager(...services) } } diff --git a/libs/src/services/dataContracts/AudiusContracts.ts b/libs/src/services/dataContracts/AudiusContracts.ts index cf17d19a2b9..c882ce003c2 100644 --- a/libs/src/services/dataContracts/AudiusContracts.ts +++ b/libs/src/services/dataContracts/AudiusContracts.ts @@ -9,7 +9,7 @@ import { PlaylistFactoryClient } from './PlaylistFactoryClient' import { UserLibraryFactoryClient } from './UserLibraryFactoryClient' import { IPLDBlacklistFactoryClient } from './IPLDBlacklistFactoryClient' import { UserReplicaSetManagerClient } from './UserReplicaSetManagerClient' -import { AudiusDataClient } from './AudiusDataClient' +import { EntityManagerClient } from './EntityManagerClient' import type { Web3Manager } from '../web3Manager' import type { ContractClient } from '../contracts/ContractClient' @@ -57,7 +57,7 @@ export class AudiusContracts { PlaylistFactoryClient: PlaylistFactoryClient UserLibraryFactoryClient: UserLibraryFactoryClient IPLDBlacklistFactoryClient: IPLDBlacklistFactoryClient - AudiusDataClient: AudiusDataClient + EntityManagerClient: EntityManagerClient contractClients: ContractClient[] UserReplicaSetManagerClient: UserReplicaSetManagerClient | undefined | null contracts: Record | undefined @@ -132,7 +132,7 @@ export class AudiusContracts { this.logger ) - this.AudiusDataClient = new AudiusDataClient( + this.EntityManagerClient = new EntityManagerClient( this.web3Manager, AudiusDataABI, 'AudiusData', @@ -148,7 +148,7 @@ export class AudiusContracts { this.PlaylistFactoryClient, this.UserLibraryFactoryClient, this.IPLDBlacklistFactoryClient, - this.AudiusDataClient + this.EntityManagerClient ] } diff --git a/libs/src/services/dataContracts/AudiusDataClient.ts b/libs/src/services/dataContracts/EntityManagerClient.ts similarity index 95% rename from libs/src/services/dataContracts/AudiusDataClient.ts rename to libs/src/services/dataContracts/EntityManagerClient.ts index d3ca3f6a8f8..c1c56765c81 100644 --- a/libs/src/services/dataContracts/AudiusDataClient.ts +++ b/libs/src/services/dataContracts/EntityManagerClient.ts @@ -2,7 +2,7 @@ import { ContractClient } from '../contracts/ContractClient' import * as signatureSchemas from '../../../data-contracts/signatureSchemas' import type { Web3Manager } from '../web3Manager' -export class AudiusDataClient extends ContractClient { +export class EntityManagerClient extends ContractClient { override web3Manager!: Web3Manager async manageEntity( diff --git a/libs/src/services/schemaValidator/schemas/playlistSchema.json b/libs/src/services/schemaValidator/schemas/playlistSchema.json index c7be4f3db81..34ee4fee23b 100644 --- a/libs/src/services/schemaValidator/schemas/playlistSchema.json +++ b/libs/src/services/schemaValidator/schemas/playlistSchema.json @@ -28,11 +28,11 @@ "default": null }, "is_album": { - "type": ["string", "null"], + "type": "boolean", "default": null }, "is_private": { - "type": ["string", "null"], + "type": "boolean", "default": null } }, From a5d0e569840078d3fd7aebaea3f460f2de674c7a Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Fri, 8 Jul 2022 18:44:29 +0000 Subject: [PATCH 054/101] Further changes --- libs/src/api/entityManager.ts | 10 +++++----- libs/src/libs.js | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/libs/src/api/entityManager.ts b/libs/src/api/entityManager.ts index 479a20ea113..38d3084e43d 100644 --- a/libs/src/api/entityManager.ts +++ b/libs/src/api/entityManager.ts @@ -110,16 +110,16 @@ export class EntityManager extends Base { userId, logger = console }: { - playlistId: number, - userId: number, + playlistId: number + userId: number logger: any }): Promise<{ blockHash: any; blockNumber: any; }> { let respValues = { blockHash: null, - blockNumber: null, + blockNumber: null } try { - let resp = await this.manageEntity({ + const resp = await this.manageEntity({ userId, entityType: 'Playlist', entityId: playlistId, @@ -127,7 +127,7 @@ export class EntityManager extends Base { metadataMultihash: '' }) logger.info(`DeletePlaylistData - ${JSON.stringify(resp)}`) - let txReceipt = resp.txReceipt + const txReceipt = resp.txReceipt respValues = { blockHash: txReceipt.blockHash, blockNumber: txReceipt.blockNumber, diff --git a/libs/src/libs.js b/libs/src/libs.js index 8aec4413e4b..6ceae15e9be 100644 --- a/libs/src/libs.js +++ b/libs/src/libs.js @@ -34,7 +34,7 @@ const { RewardsAttester } = require('./services/solanaWeb3Manager/rewardsAttester') const { Reactions } = require('./api/reactions') -const { EntityManager } = require('./api/audiusData') +const { EntityManager } = require('./api/entityManager') class AudiusLibs { /** From dda6872cb76e322800759bc80df11b17d58a5d92 Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Fri, 8 Jul 2022 19:09:27 +0000 Subject: [PATCH 055/101] Conditional init --- .../services/dataContracts/AudiusContracts.ts | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/libs/src/services/dataContracts/AudiusContracts.ts b/libs/src/services/dataContracts/AudiusContracts.ts index c882ce003c2..db1718d3dfb 100644 --- a/libs/src/services/dataContracts/AudiusContracts.ts +++ b/libs/src/services/dataContracts/AudiusContracts.ts @@ -47,7 +47,7 @@ const UserReplicaSetManagerRegistryKey = 'UserReplicaSetManager' export class AudiusContracts { web3Manager: Web3Manager registryAddress: string - dataAddress: string + entityManagerAddress: string isServer: boolean logger: Logger RegistryClient: RegistryClient @@ -66,13 +66,13 @@ export class AudiusContracts { constructor( web3Manager: Web3Manager, registryAddress: string, - dataAddress: string, + entityManagerAddress: string, isServer: boolean, logger = console ) { this.web3Manager = web3Manager this.registryAddress = registryAddress - this.dataAddress = dataAddress + this.entityManagerAddress = entityManagerAddress this.isServer = isServer this.logger = logger @@ -132,14 +132,16 @@ export class AudiusContracts { this.logger ) - this.EntityManagerClient = new EntityManagerClient( - this.web3Manager, - AudiusDataABI, - 'AudiusData', - this.getRegistryAddressForContract, - this.logger, - this.dataAddress - ) + if (this.entityManagerAddress) { + this.EntityManagerClient = new EntityManagerClient( + this.web3Manager, + AudiusDataABI, + 'AudiusData', + this.getRegistryAddressForContract, + this.logger, + this.entityManagerAddress + ) + } this.contractClients = [ this.UserFactoryClient, From afc763986d700c642a31244faea5822c61835039 Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Fri, 8 Jul 2022 19:46:01 +0000 Subject: [PATCH 056/101] Address lint --- libs/src/api/entityManager.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/libs/src/api/entityManager.ts b/libs/src/api/entityManager.ts index 38d3084e43d..dcfb6c24937 100644 --- a/libs/src/api/entityManager.ts +++ b/libs/src/api/entityManager.ts @@ -113,7 +113,7 @@ export class EntityManager extends Base { playlistId: number userId: number logger: any - }): Promise<{ blockHash: any; blockNumber: any; }> { + }): Promise<{ blockHash: any; blockNumber: any }> { let respValues = { blockHash: null, blockNumber: null @@ -130,7 +130,7 @@ export class EntityManager extends Base { const txReceipt = resp.txReceipt respValues = { blockHash: txReceipt.blockHash, - blockNumber: txReceipt.blockNumber, + blockNumber: txReceipt.blockNumber } } catch (e) { logger.error(`Data delete playlist: err ${e}`) @@ -138,7 +138,6 @@ export class EntityManager extends Base { return respValues } - /** * Manage an entity with the updated data contract flow * Leveraged to manipulate User/Track/Playlist/+ other entities From 6e0f058a374602217f204d857421e5e15fa6b3f5 Mon Sep 17 00:00:00 2001 From: Isaac Solo Date: Wed, 13 Jul 2022 10:03:49 -0700 Subject: [PATCH 057/101] Update playlist on EntityManager (#3417) --- discovery-provider/src/tasks/audius_data.py | 20 +++- libs/src/api/entityManager.ts | 116 ++++++++++++++++---- 2 files changed, 107 insertions(+), 29 deletions(-) diff --git a/discovery-provider/src/tasks/audius_data.py b/discovery-provider/src/tasks/audius_data.py index bf246cde827..e3608edbb52 100644 --- a/discovery-provider/src/tasks/audius_data.py +++ b/discovery-provider/src/tasks/audius_data.py @@ -1,5 +1,6 @@ import logging from datetime import datetime +from enum import Enum from typing import Any, Dict, Set, Tuple from sqlalchemy.orm.session import Session, make_transient @@ -12,6 +13,16 @@ logger = logging.getLogger(__name__) +class Action(Enum): + CREATE = "Create" + UPDATE = "Update" + DELETE = "Delete" + + +class EntityType(Enum): + PLAYLIST = "Playlist" + + def audius_data_state_update( self, update_task: DatabaseTask, @@ -60,7 +71,7 @@ def audius_data_state_update( ) # Handle playlist creation - if entity_type == "Playlist": + if entity_type == EntityType.PLAYLIST: playlist_id = entity_id # look up or populate existing record if playlist_id in playlist_events_lookup: @@ -77,8 +88,7 @@ def audius_data_state_update( txhash, ) - if action == "Create": - playlist_id = entity_id + if action == Action.CREATE or Action.UPDATE: playlist_record = parse_playlist_create_data_event( update_task, entry, @@ -89,7 +99,7 @@ def audius_data_state_update( session, ) - elif action == "Delete": + elif Action.DELETE: existing_playlist_record.is_delete = True playlist_record = existing_playlist_record @@ -104,9 +114,7 @@ def audius_data_state_update( "playlist" ] = playlist_record playlist_events_lookup[playlist_id]["events"].append(event_type) - playlist_ids.add(playlist_id) - processed_entries += 1 num_total_changes += processed_entries diff --git a/libs/src/api/entityManager.ts b/libs/src/api/entityManager.ts index dcfb6c24937..1b310c32e84 100644 --- a/libs/src/api/entityManager.ts +++ b/libs/src/api/entityManager.ts @@ -1,5 +1,15 @@ import { Base, BaseConstructorArgs, Services } from './base' +export enum Action { + CREATE = 'Create', + UPDATE = 'Update', + DELETE = 'Delete' +} + +export enum EntityType { + PLAYLIST = 'Playlist' +} + /* API surface for updated data contract interactions. Provides simplified entity management in a generic fashion @@ -50,18 +60,13 @@ export class EntityManager extends Base { description: string isAlbum: boolean isPrivate: boolean - coverArt: any - logger: any - }): Promise<{ blockHash: any; blockNumber: any; playlistId: number }> { - let respValues = { - blockHash: null, - blockNumber: null, - playlistId: 0 - } + coverArt: string + logger: Console + }): Promise<{ blockHash: string; blockNumber: number; playlistId: number }> { try { const ownerId: number = parseInt(this.userStateManager.getCurrentUserId()) - const action = 'Create' - const entityType = 'Playlist' + const createAction = Action.CREATE + const entityType = EntityType.PLAYLIST const entityId = await this.getValidPlaylistId() this.REQUIRES(Services.CREATOR_NODE) const updatedPlaylistImage = await this.creatorNode.uploadImage( @@ -70,7 +75,7 @@ export class EntityManager extends Base { ) const dirCID = updatedPlaylistImage.dirCID const metadata = { - action, + createAction, entity_type: entityType, playlist_id: entityId, playlist_contents: trackIds, @@ -86,20 +91,20 @@ export class EntityManager extends Base { userId: ownerId, entityType, entityId, - action, + createAction, metadataMultihash }) logger.info(`CreatePlaylistData - ${JSON.stringify(resp)}`) const txReceipt = resp.txReceipt - respValues = { + return { blockHash: txReceipt.blockHash, blockNumber: txReceipt.blockNumber, playlistId: entityId } } catch (e) { logger.error(`Data create playlist: err ${e}`) + throw e } - return respValues } /** @@ -114,10 +119,6 @@ export class EntityManager extends Base { userId: number logger: any }): Promise<{ blockHash: any; blockNumber: any }> { - let respValues = { - blockHash: null, - blockNumber: null - } try { const resp = await this.manageEntity({ userId, @@ -128,14 +129,83 @@ export class EntityManager extends Base { }) logger.info(`DeletePlaylistData - ${JSON.stringify(resp)}`) const txReceipt = resp.txReceipt - respValues = { + return { blockHash: txReceipt.blockHash, blockNumber: txReceipt.blockNumber } } catch (e) { logger.error(`Data delete playlist: err ${e}`) + throw e + } + } + /** + * Update a playlist using updated data contracts flow + **/ + async updatePlaylist({ + playlistId, + playlistName, + trackIds, + description, + isAlbum, + isPrivate, + coverArt, + logger = console + }: { + playlistId: number + playlistName: string + trackIds: number[] + description: string + isAlbum: boolean + isPrivate: boolean + coverArt: string + logger: Console + }): Promise<{ blockHash: string; blockNumber: number; playlistId: number }> { + try { + const userId: number = parseInt(this.userStateManager.getCurrentUserId()) + const updateAction = Action.UPDATE + const entityType = 'Playlist' + this.REQUIRES(Services.CREATOR_NODE) + let dirCID; + if (coverArt) { + const updatedPlaylistImage = await this.creatorNode.uploadImage( + coverArt, + true // square + ) + dirCID = updatedPlaylistImage.dirCID + } + + const playlist = (await this.discoveryProvider.getPlaylists(1, 0, [playlistId]))[0] + + const metadata = { + action: updateAction, // why include action here? + entityType, + playlist_id: playlistId, + playlist_contents: trackIds || playlist.playlist_contents, + playlist_name: playlistName || playlist.playlist_name, + playlist_image_sizes_multihash: dirCID || playlist.playlist_image_sizes_multihash, + description: description || playlist.description, + is_album: isAlbum || playlist.is_album, + is_private: isPrivate || playlist.is_private + } + const { metadataMultihash } = await this.creatorNode.uploadPlaylistMetadata(metadata) + + const resp = await this.manageEntity({ + userId, + entityType, + entityId: playlistId, + action: updateAction, + metadataMultihash + }) + const txReceipt = resp.txReceipt + return { + blockHash: txReceipt.blockHash, + blockNumber: txReceipt.blockNumber, + playlistId + } + } catch (e) { + logger.error(`Data update playlist: err ${e}`) + throw e } - return respValues } /** @@ -150,11 +220,11 @@ export class EntityManager extends Base { metadataMultihash }: { userId: number - entityType: string + entityType: EntityType entityId: number - action: string + action: Action metadataMultihash: string - }): Promise<{ txReceipt: {}; error: any }> { + }): Promise<{ txReceipt: {blockHash: string, blockNumber: number}; error: any }> { let error: string = '' let resp: any try { From 1fdb31cb96f76eac00ac6140d890f872d340b9ae Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Wed, 13 Jul 2022 19:47:46 +0000 Subject: [PATCH 058/101] Fix init bug --- .../services/dataContracts/AudiusContracts.ts | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/libs/src/services/dataContracts/AudiusContracts.ts b/libs/src/services/dataContracts/AudiusContracts.ts index db1718d3dfb..3c91f9c4d90 100644 --- a/libs/src/services/dataContracts/AudiusContracts.ts +++ b/libs/src/services/dataContracts/AudiusContracts.ts @@ -57,7 +57,7 @@ export class AudiusContracts { PlaylistFactoryClient: PlaylistFactoryClient UserLibraryFactoryClient: UserLibraryFactoryClient IPLDBlacklistFactoryClient: IPLDBlacklistFactoryClient - EntityManagerClient: EntityManagerClient + EntityManagerClient: EntityManagerClient | undefined contractClients: ContractClient[] UserReplicaSetManagerClient: UserReplicaSetManagerClient | undefined | null contracts: Record | undefined @@ -132,6 +132,15 @@ export class AudiusContracts { this.logger ) + this.contractClients = [ + this.UserFactoryClient, + this.TrackFactoryClient, + this.SocialFeatureFactoryClient, + this.PlaylistFactoryClient, + this.UserLibraryFactoryClient, + this.IPLDBlacklistFactoryClient + ] + if (this.entityManagerAddress) { this.EntityManagerClient = new EntityManagerClient( this.web3Manager, @@ -141,17 +150,8 @@ export class AudiusContracts { this.logger, this.entityManagerAddress ) + this.contractClients.push(this.EntityManagerClient) } - - this.contractClients = [ - this.UserFactoryClient, - this.TrackFactoryClient, - this.SocialFeatureFactoryClient, - this.PlaylistFactoryClient, - this.UserLibraryFactoryClient, - this.IPLDBlacklistFactoryClient, - this.EntityManagerClient - ] } async init() { From 854cdfdcd3b9e0ea418bd0f171de91dd857301bf Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Wed, 13 Jul 2022 22:35:10 +0000 Subject: [PATCH 059/101] Debug commit --- libs/src/api/entityManager.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libs/src/api/entityManager.ts b/libs/src/api/entityManager.ts index 1b310c32e84..8bd5a984007 100644 --- a/libs/src/api/entityManager.ts +++ b/libs/src/api/entityManager.ts @@ -91,7 +91,7 @@ export class EntityManager extends Base { userId: ownerId, entityType, entityId, - createAction, + action: createAction, metadataMultihash }) logger.info(`CreatePlaylistData - ${JSON.stringify(resp)}`) @@ -237,6 +237,7 @@ export class EntityManager extends Base { ) } catch (e) { error = (e as Error).message + console.log(error) } return { txReceipt: resp.txReceipt, error } } From 8a2f5455f90a82922bf365660f879627da5527f8 Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Thu, 14 Jul 2022 15:47:56 +0000 Subject: [PATCH 060/101] Fixing up enum during processing --- discovery-provider/src/tasks/audius_data.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/discovery-provider/src/tasks/audius_data.py b/discovery-provider/src/tasks/audius_data.py index e3608edbb52..dbfb72566a3 100644 --- a/discovery-provider/src/tasks/audius_data.py +++ b/discovery-provider/src/tasks/audius_data.py @@ -2,6 +2,7 @@ from datetime import datetime from enum import Enum from typing import Any, Dict, Set, Tuple +from xml.dom.minidom import Entity from sqlalchemy.orm.session import Session, make_transient from src.database_task import DatabaseTask @@ -46,6 +47,7 @@ def audius_data_state_update( playlist_ids: Set[int] = set() for tx_receipt in audius_data_txs: + logger.info(f"AudiusData.py | Processing {tx_receipt}") txhash = update_task.web3.toHex(tx_receipt.transactionHash) for event_type in audius_data_event_types_arr: audius_data_event_tx = get_audius_data_events_tx( @@ -70,9 +72,12 @@ def audius_data_state_update( f"index.py | AudiusData state update: {user_id}, entity_id={entity_id}, entity_type={entity_type}, action={action}, metadata_cid={metadata_cid}, metadata={metadata} signer={signer}" ) + is_playlist = entity_type == EntityType.PLAYLIST.value + logger.info(f"index.py | AudiusData - playlist_events_lookup {playlist_events_lookup}, is_playlist={is_playlist}, entity_type={entity_type}, expected={EntityType.PLAYLIST}, actions={Action.CREATE},{Action.DELETE},{Action.UPDATE}") # Handle playlist creation - if entity_type == EntityType.PLAYLIST: + if entity_type == EntityType.PLAYLIST.value: playlist_id = entity_id + logger.info(f"index.py | AudiusData - playlist detected, id={playlist_id}") # look up or populate existing record if playlist_id in playlist_events_lookup: existing_playlist_record = playlist_events_lookup[playlist_id][ @@ -89,6 +94,7 @@ def audius_data_state_update( ) if action == Action.CREATE or Action.UPDATE: + logger.info(f"index.py | AudiusData - handling {action}, events_lookup={playlist_events_lookup}") playlist_record = parse_playlist_create_data_event( update_task, entry, @@ -123,7 +129,7 @@ def audius_data_state_update( changed_entity_ids["playlist"] = playlist_ids for playlist_id, value_obj in playlist_events_lookup.items(): - logger.info(f"index.py | playlists.py | Adding {value_obj['playlist']})") + logger.info(f"index.py | AudiusData | playlists.py | Adding {value_obj['playlist']})") if value_obj["events"]: invalidate_old_playlist(session, playlist_id) session.add(value_obj["playlist"]) From 7403d8ceeaf43cff9bd212c43dc8fd782470045c Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Thu, 14 Jul 2022 16:21:24 +0000 Subject: [PATCH 061/101] Use .value everywhere --- discovery-provider/src/tasks/audius_data.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/discovery-provider/src/tasks/audius_data.py b/discovery-provider/src/tasks/audius_data.py index dbfb72566a3..f20e9ebd0d2 100644 --- a/discovery-provider/src/tasks/audius_data.py +++ b/discovery-provider/src/tasks/audius_data.py @@ -93,7 +93,7 @@ def audius_data_state_update( txhash, ) - if action == Action.CREATE or Action.UPDATE: + if action == Action.CREATE.value or Action.UPDATE.value: logger.info(f"index.py | AudiusData - handling {action}, events_lookup={playlist_events_lookup}") playlist_record = parse_playlist_create_data_event( update_task, @@ -105,7 +105,7 @@ def audius_data_state_update( session, ) - elif Action.DELETE: + elif Action.DELETE.value: existing_playlist_record.is_delete = True playlist_record = existing_playlist_record From c34a17619385bec3a25f44880e645e003fc9f1aa Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Thu, 14 Jul 2022 16:22:39 +0000 Subject: [PATCH 062/101] Incorrect import --- discovery-provider/src/tasks/audius_data.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/discovery-provider/src/tasks/audius_data.py b/discovery-provider/src/tasks/audius_data.py index f20e9ebd0d2..1e2069e7699 100644 --- a/discovery-provider/src/tasks/audius_data.py +++ b/discovery-provider/src/tasks/audius_data.py @@ -2,7 +2,6 @@ from datetime import datetime from enum import Enum from typing import Any, Dict, Set, Tuple -from xml.dom.minidom import Entity from sqlalchemy.orm.session import Session, make_transient from src.database_task import DatabaseTask @@ -73,11 +72,15 @@ def audius_data_state_update( ) is_playlist = entity_type == EntityType.PLAYLIST.value - logger.info(f"index.py | AudiusData - playlist_events_lookup {playlist_events_lookup}, is_playlist={is_playlist}, entity_type={entity_type}, expected={EntityType.PLAYLIST}, actions={Action.CREATE},{Action.DELETE},{Action.UPDATE}") + logger.info( + f"index.py | AudiusData - playlist_events_lookup {playlist_events_lookup}, is_playlist={is_playlist}, entity_type={entity_type}, expected={EntityType.PLAYLIST}, actions={Action.CREATE},{Action.DELETE},{Action.UPDATE}" + ) # Handle playlist creation if entity_type == EntityType.PLAYLIST.value: playlist_id = entity_id - logger.info(f"index.py | AudiusData - playlist detected, id={playlist_id}") + logger.info( + f"index.py | AudiusData - playlist detected, id={playlist_id}" + ) # look up or populate existing record if playlist_id in playlist_events_lookup: existing_playlist_record = playlist_events_lookup[playlist_id][ @@ -94,7 +97,9 @@ def audius_data_state_update( ) if action == Action.CREATE.value or Action.UPDATE.value: - logger.info(f"index.py | AudiusData - handling {action}, events_lookup={playlist_events_lookup}") + logger.info( + f"index.py | AudiusData - handling {action}, events_lookup={playlist_events_lookup}" + ) playlist_record = parse_playlist_create_data_event( update_task, entry, @@ -129,7 +134,9 @@ def audius_data_state_update( changed_entity_ids["playlist"] = playlist_ids for playlist_id, value_obj in playlist_events_lookup.items(): - logger.info(f"index.py | AudiusData | playlists.py | Adding {value_obj['playlist']})") + logger.info( + f"index.py | AudiusData | playlists.py | Adding {value_obj['playlist']})" + ) if value_obj["events"]: invalidate_old_playlist(session, playlist_id) session.add(value_obj["playlist"]) From 7b5c82fadee3442b11c43f6fce560102c206d1d4 Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Thu, 14 Jul 2022 16:28:58 +0000 Subject: [PATCH 063/101] Remove debug logs --- discovery-provider/src/tasks/audius_data.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/discovery-provider/src/tasks/audius_data.py b/discovery-provider/src/tasks/audius_data.py index 1e2069e7699..d79d67e56bd 100644 --- a/discovery-provider/src/tasks/audius_data.py +++ b/discovery-provider/src/tasks/audius_data.py @@ -70,11 +70,6 @@ def audius_data_state_update( logger.info( f"index.py | AudiusData state update: {user_id}, entity_id={entity_id}, entity_type={entity_type}, action={action}, metadata_cid={metadata_cid}, metadata={metadata} signer={signer}" ) - - is_playlist = entity_type == EntityType.PLAYLIST.value - logger.info( - f"index.py | AudiusData - playlist_events_lookup {playlist_events_lookup}, is_playlist={is_playlist}, entity_type={entity_type}, expected={EntityType.PLAYLIST}, actions={Action.CREATE},{Action.DELETE},{Action.UPDATE}" - ) # Handle playlist creation if entity_type == EntityType.PLAYLIST.value: playlist_id = entity_id From fa60d11cd999b7556356e09a88d01af7d2c6f7fe Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Thu, 14 Jul 2022 18:05:09 +0000 Subject: [PATCH 064/101] Calculate valid playlist ID --- libs/src/api/entityManager.ts | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/libs/src/api/entityManager.ts b/libs/src/api/entityManager.ts index 8bd5a984007..34e97a71de9 100644 --- a/libs/src/api/entityManager.ts +++ b/libs/src/api/entityManager.ts @@ -10,6 +10,11 @@ export enum EntityType { PLAYLIST = 'Playlist' } +// Minimum playlist ID, intentionally higher than legacy playlist ID range +const MIN_PLAYLIST_ID = 400000 +// Maximum playlist ID, reflects postgres max integer value +const MAX_PLAYLIST_ID = 2147483647 + /* API surface for updated data contract interactions. Provides simplified entity management in a generic fashion @@ -23,19 +28,27 @@ export class EntityManager extends Base { this.deletePlaylist = this.deletePlaylist.bind(this) } + /** + * Generate random integer between two known values + */ + getRandomInt(min: number, max: number): number { + min = Math.ceil(min); + max = Math.floor(max); + return Math.floor(Math.random() * (max - min) + min); + } + /** * Calculate an unoccupied playlist ID * Maximum value is postgres integer max (2147483647) * Minimum value is artificially set to 400000 */ async getValidPlaylistId(): Promise { - let playlistId: number = 10 + let playlistId: number = this.getRandomInt(MIN_PLAYLIST_ID, MAX_PLAYLIST_ID) let validIdFound: boolean = false while (!validIdFound) { - const resp = await this.discoveryProvider.getPlaylists(1, 0, [playlistId]) + const resp: any = await this.discoveryProvider.getPlaylists(1, 0, [playlistId]) if (resp.length !== 0) { - // TODO: Playlist ID Min offset - playlistId = Math.floor(Math.random() * 2147483647) + playlistId = this.getRandomInt(MIN_PLAYLIST_ID, MAX_PLAYLIST_ID) } else { validIdFound = true } @@ -122,9 +135,9 @@ export class EntityManager extends Base { try { const resp = await this.manageEntity({ userId, - entityType: 'Playlist', + entityType: EntityType.PLAYLIST, entityId: playlistId, - action: 'Delete', + action: Action.DELETE, metadataMultihash: '' }) logger.info(`DeletePlaylistData - ${JSON.stringify(resp)}`) From d168c39fc0ab396d972405f138f6942624a014c5 Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Thu, 14 Jul 2022 18:40:55 +0000 Subject: [PATCH 065/101] Another entity type NIT fix --- libs/src/api/entityManager.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/src/api/entityManager.ts b/libs/src/api/entityManager.ts index 34e97a71de9..076972eb6dd 100644 --- a/libs/src/api/entityManager.ts +++ b/libs/src/api/entityManager.ts @@ -176,7 +176,7 @@ export class EntityManager extends Base { try { const userId: number = parseInt(this.userStateManager.getCurrentUserId()) const updateAction = Action.UPDATE - const entityType = 'Playlist' + const entityType = EntityType.PLAYLIST this.REQUIRES(Services.CREATOR_NODE) let dirCID; if (coverArt) { From aaa03b7653809c6b67c65a4e57226ce74a10a793 Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Thu, 14 Jul 2022 22:39:00 +0000 Subject: [PATCH 066/101] Use new image w/EntityManager deployed --- service-commands/src/commands/service-commands.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service-commands/src/commands/service-commands.json b/service-commands/src/commands/service-commands.json index f983613114e..890ad8adad4 100644 --- a/service-commands/src/commands/service-commands.json +++ b/service-commands/src/commands/service-commands.json @@ -43,7 +43,7 @@ }, "contracts-predeployed": { "up": [ - "docker run --name audius_ganache_cli -d -p 8545:8545 --network=audius_dev audius/ganache:data-contracts-predeployed-a25931b955adce92676cec029661533f80013908", + "docker run --name audius_ganache_cli -d -p 8545:8545 --network=audius_dev audius/ganache:data-contracts-predeployed-fd3ba540307ae1df8ece8ba5995a1954a414b2f9", "docker cp audius_ganache_cli:/app/contract-config.json $PROTOCOL_DIR/identity-service/contract-config.json", "docker cp audius_ganache_cli:/app/contract-config.json $PROTOCOL_DIR/creator-node/contract-config.json", "docker cp audius_ganache_cli:/app/contract-config.json ~/.audius/contract-config.json", From 2347bc03f37bf7522a8471aab7709a333d7e7c79 Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Fri, 15 Jul 2022 01:21:22 +0000 Subject: [PATCH 067/101] Config --- discovery-provider/src/app.py | 3 +-- discovery-provider/src/tasks/audius_data.py | 3 +-- libs/initScripts/configureLocalDiscProv.js | 16 ++++++++-------- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/discovery-provider/src/app.py b/discovery-provider/src/app.py index 069fffd3807..a6d137c24e5 100644 --- a/discovery-provider/src/app.py +++ b/discovery-provider/src/app.py @@ -128,7 +128,7 @@ def init_contracts(): ) audius_data_address = web3.toChecksumAddress( - shared_config["contracts"]["data_address"] + shared_config["contracts"]["entity_manager_address"] ) audius_data_inst = web3.eth.contract( address=audius_data_address, abi=abi_values["AudiusData"]["abi"] @@ -201,7 +201,6 @@ def create_celery(test_config=None): audius_data, contract_addresses, ) = init_contracts() - logger.info(f"contract_addresses_dict {contract_addresses}") return create(test_config, mode="celery") diff --git a/discovery-provider/src/tasks/audius_data.py b/discovery-provider/src/tasks/audius_data.py index d79d67e56bd..68fe3de9b99 100644 --- a/discovery-provider/src/tasks/audius_data.py +++ b/discovery-provider/src/tasks/audius_data.py @@ -31,8 +31,7 @@ def audius_data_state_update( block_number, block_timestamp, block_hash, - ipfs_metadata, # prefix unused args with underscore to prevent pylint - _blacklisted_cids, + ipfs_metadata, ) -> Tuple[int, Dict[str, Set[(int)]]]: num_total_changes = 0 diff --git a/libs/initScripts/configureLocalDiscProv.js b/libs/initScripts/configureLocalDiscProv.js index f451ed5ae52..d79dffa6fcd 100644 --- a/libs/initScripts/configureLocalDiscProv.js +++ b/libs/initScripts/configureLocalDiscProv.js @@ -7,7 +7,7 @@ const solanaConfig = require('../../solana-programs/solana-program-config.json') const ETH_CONTRACTS_REGISTRY = 'audius_eth_contracts_registry' const CONTRACTS_REGISTRY = 'audius_contracts_registry' -const CONTRACTS_DATA_ADDRESS = 'audius_contracts_data_address' +const ENTITY_MANAGER_ADDRESS = 'audius_contracts_entity_manager_address' const SOLANA_TRACK_LISTEN_COUNT_ADDRESS = 'audius_solana_track_listen_count_address' const SOLANA_ENDPOINT = 'audius_solana_endpoint' @@ -27,7 +27,7 @@ const SOLANA_ANCHOR_ADMIN_STORAGE_PUBLIC_KEY = 'audius_solana_anchor_admin_stora const configureLocalDiscProv = async () => { const ethRegistryAddress = ethContractsMigrationOutput.registryAddress const dataRegistryAddress = dataContractsMigrationOutput.registryAddress - const dataContractAddress = dataContractsMigrationOutput.audiusDataProxyAddress + const entityManagerAddress = dataContractsMigrationOutput.entityManagerProxyAddress const solanaTrackListenCountAddress = solanaConfig.trackListenCountAddress const signerGroup = solanaConfig.signerGroup const solanaEndpoint = solanaConfig.endpoint @@ -54,7 +54,7 @@ const configureLocalDiscProv = async () => { anchorProgramId, anchorAdminStoragePublicKey, dataRegistryAddress, - dataContractAddress + entityManagerAddress ) } @@ -73,7 +73,7 @@ const _updateDiscoveryProviderEnvFile = async ( anchorProgramId, anchorAdminStoragePublicKey, dataRegistryAddress, - dataContractAddress + entityManagerAddress ) => { const fileStream = fs.createReadStream(readPath) const rl = readline.createInterface({ @@ -106,7 +106,7 @@ const _updateDiscoveryProviderEnvFile = async ( const anchorProgramIdLine = `${SOLANA_ANCHOR_PROGRAM_ID}=${anchorProgramId}` const anchorAdminStoragePublicKeyLine = `${SOLANA_ANCHOR_ADMIN_STORAGE_PUBLIC_KEY}=${anchorAdminStoragePublicKey}` const dataRegistryLine = `${CONTRACTS_REGISTRY}=${dataRegistryAddress}` - const dataContractLine = `${CONTRACTS_DATA_ADDRESS}=${dataContractAddress}` + const entityManagerContractLine = `${ENTITY_MANAGER_ADDRESS}=${entityManagerAddress}` for await (const line of rl) { if (line.includes(ETH_CONTRACTS_REGISTRY)) { @@ -142,8 +142,8 @@ const _updateDiscoveryProviderEnvFile = async ( } else if (line.includes(CONTRACTS_REGISTRY)) { output.push(dataRegistryLine) dataRegistryLineFound = true - } else if (line.includes(CONTRACTS_DATA_ADDRESS)) { - output.push(dataContractLine) + } else if (line.includes(ENTITY_MANAGER_ADDRESS)) { + output.push(entityManagerContractLine) dataContractLineFound = true } else { output.push(line) @@ -183,7 +183,7 @@ const _updateDiscoveryProviderEnvFile = async ( output.push(dataRegistryLine) } if (!dataContractLineFound) { - output.push(dataContractLine) + output.push(entityManagerContractLine) } fs.writeFileSync(writePath, output.join('\n')) console.log(`Updated DISCOVERY PROVIDER ${writePath} ${ETH_CONTRACTS_REGISTRY}=${ethRegistryAddress} ${output}`) From f06dbfade0172e99d368882ab192e7377d6d6761 Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Fri, 15 Jul 2022 01:29:05 +0000 Subject: [PATCH 068/101] Fix output --- contracts/scripts/migrate-contracts.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contracts/scripts/migrate-contracts.js b/contracts/scripts/migrate-contracts.js index 2e75a365f1e..85386f9a19a 100644 --- a/contracts/scripts/migrate-contracts.js +++ b/contracts/scripts/migrate-contracts.js @@ -78,7 +78,7 @@ const outputJsonConfigFile = async (outputPath) => { let addressInfo = require(migrationOutputPath) let outputDictionary = {} outputDictionary['registryAddress'] = addressInfo.registryAddress - outputDictionary['audiusDataAddress'] = addressInfo.audiusDataProxyAddress + outputDictionary['entityManagerProxyAddress'] = addressInfo.entityManagerProxyAddress outputDictionary['ursmAddress'] = addressInfo.ursmAddress outputDictionary['ownerWallet'] = await getDefaultAccount() outputDictionary['allWallets'] = await web3.eth.getAccounts() From 44227843a0947d134e9e4493d99271eef89c8054 Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Fri, 15 Jul 2022 01:56:32 +0000 Subject: [PATCH 069/101] Config in libs for rename --- .../ABIs/{AudiusData.json => EntityManager.json} | 2 +- libs/src/services/ABIDecoder/AudiusABIDecoder.ts | 2 +- libs/src/services/dataContracts/AudiusContracts.ts | 4 ++-- service-commands/src/commands/service-commands.json | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) rename libs/data-contracts/ABIs/{AudiusData.json => EntityManager.json} (98%) diff --git a/libs/data-contracts/ABIs/AudiusData.json b/libs/data-contracts/ABIs/EntityManager.json similarity index 98% rename from libs/data-contracts/ABIs/AudiusData.json rename to libs/data-contracts/ABIs/EntityManager.json index 02f5b9651d1..f7398fb21e8 100644 --- a/libs/data-contracts/ABIs/AudiusData.json +++ b/libs/data-contracts/ABIs/EntityManager.json @@ -1,5 +1,5 @@ { - "contractName": "AudiusData", + "contractName": "EntityManager", "abi": [ { "constant": true, diff --git a/libs/src/services/ABIDecoder/AudiusABIDecoder.ts b/libs/src/services/ABIDecoder/AudiusABIDecoder.ts index 28e6b831aa5..3f6a4af771a 100644 --- a/libs/src/services/ABIDecoder/AudiusABIDecoder.ts +++ b/libs/src/services/ABIDecoder/AudiusABIDecoder.ts @@ -19,7 +19,7 @@ loadABI('SocialFeatureFactory.json') loadABI('PlaylistFactory.json') loadABI('UserLibraryFactory.json') loadABI('UserReplicaSetManager.json') -loadABI('AudiusData.json') +loadABI('EntityManager.json') // eslint-disable-next-line @typescript-eslint/no-extraneous-class -- should just use esm export class AudiusABIDecoder { diff --git a/libs/src/services/dataContracts/AudiusContracts.ts b/libs/src/services/dataContracts/AudiusContracts.ts index 3c91f9c4d90..10b3ba0ea71 100644 --- a/libs/src/services/dataContracts/AudiusContracts.ts +++ b/libs/src/services/dataContracts/AudiusContracts.ts @@ -33,7 +33,7 @@ const IPLDBlacklistFactoryABI = Utils.importDataContractABI( const UserReplicaSetManagerABI = Utils.importDataContractABI( 'UserReplicaSetManager.json' ).abi -const AudiusDataABI = Utils.importDataContractABI('AudiusData.json').abi +const EntityManagerABI = Utils.importDataContractABI('EntityManager.json').abi // define contract registry keys const UserFactoryRegistryKey = 'UserFactory' @@ -144,7 +144,7 @@ export class AudiusContracts { if (this.entityManagerAddress) { this.EntityManagerClient = new EntityManagerClient( this.web3Manager, - AudiusDataABI, + EntityManagerABI, 'AudiusData', this.getRegistryAddressForContract, this.logger, diff --git a/service-commands/src/commands/service-commands.json b/service-commands/src/commands/service-commands.json index 890ad8adad4..95d3042d962 100644 --- a/service-commands/src/commands/service-commands.json +++ b/service-commands/src/commands/service-commands.json @@ -43,7 +43,7 @@ }, "contracts-predeployed": { "up": [ - "docker run --name audius_ganache_cli -d -p 8545:8545 --network=audius_dev audius/ganache:data-contracts-predeployed-fd3ba540307ae1df8ece8ba5995a1954a414b2f9", + "docker run --name audius_ganache_cli -d -p 8545:8545 --network=audius_dev audius/ganache:data-contracts-predeployed-f06dbfade0172e99d368882ab192e7377d6d6761", "docker cp audius_ganache_cli:/app/contract-config.json $PROTOCOL_DIR/identity-service/contract-config.json", "docker cp audius_ganache_cli:/app/contract-config.json $PROTOCOL_DIR/creator-node/contract-config.json", "docker cp audius_ganache_cli:/app/contract-config.json ~/.audius/contract-config.json", From 71480157ba1785a1c4f9cd32e0c705575bf12763 Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Fri, 15 Jul 2022 02:02:24 +0000 Subject: [PATCH 070/101] default config update --- discovery-provider/default_config.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/discovery-provider/default_config.ini b/discovery-provider/default_config.ini index 0d35168e893..123e94ed5c9 100644 --- a/discovery-provider/default_config.ini +++ b/discovery-provider/default_config.ini @@ -71,7 +71,7 @@ allow_all = false [contracts] registry = -data_address = +entity_manager_address = [eth_contracts] registry = From 9994bb40ce15c1a3f23b77d4ac8a117d70cfd4f4 Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Fri, 15 Jul 2022 02:18:19 +0000 Subject: [PATCH 071/101] Updated schema --- libs/data-contracts/signatureSchemas.ts | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/libs/data-contracts/signatureSchemas.ts b/libs/data-contracts/signatureSchemas.ts index 9462effc106..332064b8a3e 100644 --- a/libs/data-contracts/signatureSchemas.ts +++ b/libs/data-contracts/signatureSchemas.ts @@ -64,8 +64,8 @@ const getUserReplicaSetManagerDomain: DomainFn = (chainId, contractAddress) => { ) } -const getAudiusDataDomain: DomainFn = (chainId, contractAddress) => { - return getDomainData("Audius Data", "1", chainId, contractAddress) +const getEntityManagerDomain: DomainFn = (chainId, contractAddress) => { + return getDomainData('Entity Manager', '1', chainId, contractAddress) } export const domains = { @@ -76,7 +76,7 @@ export const domains = { getUserLibraryFactoryDomain, getIPLDBlacklistFactoryDomain, getUserReplicaSetManagerDomain, - getAudiusDataDomain + getEntityManagerDomain } /* contract signing domain */ @@ -264,12 +264,12 @@ const updateReplicaSet = [ ] const manageEntity = [ - { name: 'userId', type: 'uint'}, - { name: 'entityType', type: 'string'}, - { name: 'entityId', type: 'uint'}, - { name: 'action', type: 'string'}, - { name: 'metadata', type: 'string'}, - { name: 'nonce', type: 'bytes32'}, + { name: 'userId', type: 'uint' }, + { name: 'entityType', type: 'string' }, + { name: 'entityId', type: 'uint' }, + { name: 'action', type: 'string' }, + { name: 'metadata', type: 'string' }, + { name: 'nonce', type: 'bytes32' } ] export const schemas = { @@ -1179,7 +1179,7 @@ const getManageEntityData = ( nonce } return getRequestData( - domains.getAudiusDataDomain, + domains.getEntityManagerDomain, chainId, contractAddress, 'ManageEntity', From c43503634b17a6a2c061e723eacb92c267b82ed5 Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Fri, 15 Jul 2022 14:29:56 +0000 Subject: [PATCH 072/101] Lint --- libs/src/api/entityManager.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libs/src/api/entityManager.ts b/libs/src/api/entityManager.ts index 076972eb6dd..78d65282f27 100644 --- a/libs/src/api/entityManager.ts +++ b/libs/src/api/entityManager.ts @@ -32,9 +32,9 @@ export class EntityManager extends Base { * Generate random integer between two known values */ getRandomInt(min: number, max: number): number { - min = Math.ceil(min); - max = Math.floor(max); - return Math.floor(Math.random() * (max - min) + min); + min = Math.ceil(min) + max = Math.floor(max) + return Math.floor(Math.random() * (max - min) + min) } /** From ee8ecd2195bd4c45b0d0d20cad60e3c06097e549 Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Fri, 15 Jul 2022 15:06:51 +0000 Subject: [PATCH 073/101] PARITY E2E w/CREATE - Contract Name fix --- libs/src/services/dataContracts/AudiusContracts.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/src/services/dataContracts/AudiusContracts.ts b/libs/src/services/dataContracts/AudiusContracts.ts index 10b3ba0ea71..ccd384613ef 100644 --- a/libs/src/services/dataContracts/AudiusContracts.ts +++ b/libs/src/services/dataContracts/AudiusContracts.ts @@ -145,7 +145,7 @@ export class AudiusContracts { this.EntityManagerClient = new EntityManagerClient( this.web3Manager, EntityManagerABI, - 'AudiusData', + 'EntityManager', this.getRegistryAddressForContract, this.logger, this.entityManagerAddress From 1a43ce115534c8afdb12bb4bc823754247080ef5 Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Fri, 15 Jul 2022 16:51:41 +0000 Subject: [PATCH 074/101] Fixing types and such --- libs/src/api/entityManager.ts | 64 ++++++++++++++----- .../dataContracts/EntityManagerClient.ts | 3 +- .../discoveryProvider/DiscoveryProvider.ts | 2 +- 3 files changed, 50 insertions(+), 19 deletions(-) diff --git a/libs/src/api/entityManager.ts b/libs/src/api/entityManager.ts index 78d65282f27..741e35918ed 100644 --- a/libs/src/api/entityManager.ts +++ b/libs/src/api/entityManager.ts @@ -1,4 +1,5 @@ import { Base, BaseConstructorArgs, Services } from './base' +import type { Nullable } from '../utils' export enum Action { CREATE = 'Create', @@ -46,7 +47,9 @@ export class EntityManager extends Base { let playlistId: number = this.getRandomInt(MIN_PLAYLIST_ID, MAX_PLAYLIST_ID) let validIdFound: boolean = false while (!validIdFound) { - const resp: any = await this.discoveryProvider.getPlaylists(1, 0, [playlistId]) + const resp: any = await this.discoveryProvider.getPlaylists(1, 0, [ + playlistId + ]) if (resp.length !== 0) { playlistId = this.getRandomInt(MIN_PLAYLIST_ID, MAX_PLAYLIST_ID) } else { @@ -75,9 +78,25 @@ export class EntityManager extends Base { isPrivate: boolean coverArt: string logger: Console - }): Promise<{ blockHash: string; blockNumber: number; playlistId: number }> { + }): Promise<{ + blockHash: Nullable + blockNumber: Nullable + playlistId: Nullable + error: Nullable + }> { + let error = null try { - const ownerId: number = parseInt(this.userStateManager.getCurrentUserId()) + const currentUserId: string | null = + this.userStateManager.getCurrentUserId() + if (!currentUserId) { + return { + blockHash: null, + blockNumber: null, + playlistId: null, + error: 'Missing current user ID' + } + } + const userId: number = parseInt(currentUserId) const createAction = Action.CREATE const entityType = EntityType.PLAYLIST const entityId = await this.getValidPlaylistId() @@ -101,7 +120,7 @@ export class EntityManager extends Base { const { metadataMultihash } = await this.creatorNode.uploadPlaylistMetadata(metadata) const resp = await this.manageEntity({ - userId: ownerId, + userId: userId, entityType, entityId, action: createAction, @@ -112,11 +131,17 @@ export class EntityManager extends Base { return { blockHash: txReceipt.blockHash, blockNumber: txReceipt.blockNumber, - playlistId: entityId + playlistId: entityId, + error } } catch (e) { - logger.error(`Data create playlist: err ${e}`) - throw e + error = (e as Error).message + return { + blockHash: null, + blockNumber: null, + playlistId: null, + error + } } } @@ -151,6 +176,7 @@ export class EntityManager extends Base { throw e } } + /** * Update a playlist using updated data contracts flow **/ @@ -178,7 +204,7 @@ export class EntityManager extends Base { const updateAction = Action.UPDATE const entityType = EntityType.PLAYLIST this.REQUIRES(Services.CREATOR_NODE) - let dirCID; + let dirCID if (coverArt) { const updatedPlaylistImage = await this.creatorNode.uploadImage( coverArt, @@ -187,7 +213,9 @@ export class EntityManager extends Base { dirCID = updatedPlaylistImage.dirCID } - const playlist = (await this.discoveryProvider.getPlaylists(1, 0, [playlistId]))[0] + const playlist: any = ( + await this.discoveryProvider.getPlaylists(1, 0, [playlistId]) + )[0] const metadata = { action: updateAction, // why include action here? @@ -195,12 +223,14 @@ export class EntityManager extends Base { playlist_id: playlistId, playlist_contents: trackIds || playlist.playlist_contents, playlist_name: playlistName || playlist.playlist_name, - playlist_image_sizes_multihash: dirCID || playlist.playlist_image_sizes_multihash, + playlist_image_sizes_multihash: + dirCID || playlist.playlist_image_sizes_multihash, description: description || playlist.description, is_album: isAlbum || playlist.is_album, is_private: isPrivate || playlist.is_private } - const { metadataMultihash } = await this.creatorNode.uploadPlaylistMetadata(metadata) + const { metadataMultihash } = + await this.creatorNode.uploadPlaylistMetadata(metadata) const resp = await this.manageEntity({ userId, @@ -237,21 +267,23 @@ export class EntityManager extends Base { entityId: number action: Action metadataMultihash: string - }): Promise<{ txReceipt: {blockHash: string, blockNumber: number}; error: any }> { - let error: string = '' + }): Promise< + { txReceipt: any; error: null } | { txReceipt: null; error: string } + > { + let error = null let resp: any try { - resp = await this.contracts.EntityManagerClient.manageEntity( + resp = await this.contracts.EntityManagerClient?.manageEntity( userId, entityType, entityId, action, metadataMultihash ) + return { txReceipt: resp.txReceipt, error } } catch (e) { error = (e as Error).message - console.log(error) + return { txReceipt: null, error } } - return { txReceipt: resp.txReceipt, error } } } diff --git a/libs/src/services/dataContracts/EntityManagerClient.ts b/libs/src/services/dataContracts/EntityManagerClient.ts index c1c56765c81..dbb8765c79b 100644 --- a/libs/src/services/dataContracts/EntityManagerClient.ts +++ b/libs/src/services/dataContracts/EntityManagerClient.ts @@ -11,7 +11,7 @@ export class EntityManagerClient extends ContractClient { entityId: number, action: string, metadata: string - ) { + ): Promise<{ txReceipt: any }> { const nonce = signatureSchemas.getNonce() const chainId = await this.getEthNetId() const contractAddress = await this.getAddress() @@ -41,7 +41,6 @@ export class EntityManagerClient extends ContractClient { this.contractRegistryKey, contractAddress ) - console.log(tx) return { txReceipt: tx } diff --git a/libs/src/services/discoveryProvider/DiscoveryProvider.ts b/libs/src/services/discoveryProvider/DiscoveryProvider.ts index da5935e4610..22d270f84a4 100644 --- a/libs/src/services/discoveryProvider/DiscoveryProvider.ts +++ b/libs/src/services/discoveryProvider/DiscoveryProvider.ts @@ -399,7 +399,7 @@ export class DiscoveryProvider { idsArray = null, targetUserId = null, withUsers = false - ) { + ): Promise { const req = Requests.getPlaylists( limit, offset, From 7d0c08102ec90141c422844aee5ee1ebe30e207a Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Fri, 15 Jul 2022 17:00:57 +0000 Subject: [PATCH 075/101] Aggregating types --- libs/src/api/entityManager.ts | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/libs/src/api/entityManager.ts b/libs/src/api/entityManager.ts index 741e35918ed..7b0aada93e9 100644 --- a/libs/src/api/entityManager.ts +++ b/libs/src/api/entityManager.ts @@ -198,9 +198,25 @@ export class EntityManager extends Base { isPrivate: boolean coverArt: string logger: Console - }): Promise<{ blockHash: string; blockNumber: number; playlistId: number }> { + }): Promise<{ + blockHash: Nullable + blockNumber: Nullable + playlistId: Nullable + error: Nullable + }> { + let error = null try { - const userId: number = parseInt(this.userStateManager.getCurrentUserId()) + const currentUserId: string | null = + this.userStateManager.getCurrentUserId() + if (!currentUserId) { + return { + blockHash: null, + blockNumber: null, + playlistId: null, + error: 'Missing current user ID' + } + } + const userId: number = parseInt(currentUserId) const updateAction = Action.UPDATE const entityType = EntityType.PLAYLIST this.REQUIRES(Services.CREATOR_NODE) @@ -243,11 +259,17 @@ export class EntityManager extends Base { return { blockHash: txReceipt.blockHash, blockNumber: txReceipt.blockNumber, - playlistId + playlistId, + error } } catch (e) { - logger.error(`Data update playlist: err ${e}`) - throw e + error = (e as Error).message + return { + blockHash: null, + blockNumber: null, + playlistId: null, + error + } } } From 2da18dd706c4077a53f0e72b85c8db9ea104b4bb Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Fri, 15 Jul 2022 17:42:08 +0000 Subject: [PATCH 076/101] More cleanup work --- libs/src/api/entityManager.ts | 94 ++++++++++++++++++++--------------- 1 file changed, 55 insertions(+), 39 deletions(-) diff --git a/libs/src/api/entityManager.ts b/libs/src/api/entityManager.ts index 7b0aada93e9..e18e0b6df00 100644 --- a/libs/src/api/entityManager.ts +++ b/libs/src/api/entityManager.ts @@ -11,6 +11,25 @@ export enum EntityType { PLAYLIST = 'Playlist' } +export interface PlaylistOperationResponse { + /** + * Blockhash of playlist transaction + */ + blockHash: Nullable + /** + * Block number of playlist transaction + */ + blockNumber: Nullable + /** + * ID of playlist being modified + */ + playlistId: Nullable + /** + * String error message returned + */ + error: Nullable +} + // Minimum playlist ID, intentionally higher than legacy playlist ID range const MIN_PLAYLIST_ID = 400000 // Maximum playlist ID, reflects postgres max integer value @@ -59,6 +78,18 @@ export class EntityManager extends Base { return playlistId } + /** + * Playlist default response values + */ + getDefaultPlaylistReponseValues(): PlaylistOperationResponse { + return { + blockHash: null, + blockNumber: null, + playlistId: null, + error: null + } + } + /** * Create a playlist using updated data contracts flow */ @@ -78,13 +109,9 @@ export class EntityManager extends Base { isPrivate: boolean coverArt: string logger: Console - }): Promise<{ - blockHash: Nullable - blockNumber: Nullable - playlistId: Nullable - error: Nullable - }> { - let error = null + }): Promise { + const responseValues: PlaylistOperationResponse = + this.getDefaultPlaylistReponseValues() try { const currentUserId: string | null = this.userStateManager.getCurrentUserId() @@ -119,29 +146,22 @@ export class EntityManager extends Base { } const { metadataMultihash } = await this.creatorNode.uploadPlaylistMetadata(metadata) - const resp = await this.manageEntity({ + const manageEntityResponse = await this.manageEntity({ userId: userId, entityType, entityId, action: createAction, metadataMultihash }) - logger.info(`CreatePlaylistData - ${JSON.stringify(resp)}`) - const txReceipt = resp.txReceipt - return { - blockHash: txReceipt.blockHash, - blockNumber: txReceipt.blockNumber, - playlistId: entityId, - error - } + const txReceipt = manageEntityResponse.txReceipt + responseValues.blockHash = txReceipt.blockHash + responseValues.blockNumber = txReceipt.blockNumber + responseValues.playlistId = entityId + return responseValues } catch (e) { - error = (e as Error).message - return { - blockHash: null, - blockNumber: null, - playlistId: null, - error - } + const error = (e as Error).message + responseValues.error = error + return responseValues } } @@ -150,13 +170,14 @@ export class EntityManager extends Base { */ async deletePlaylist({ playlistId, - userId, - logger = console + userId }: { playlistId: number userId: number logger: any - }): Promise<{ blockHash: any; blockNumber: any }> { + }): Promise { + const responseValues: PlaylistOperationResponse = + this.getDefaultPlaylistReponseValues() try { const resp = await this.manageEntity({ userId, @@ -165,15 +186,15 @@ export class EntityManager extends Base { action: Action.DELETE, metadataMultihash: '' }) - logger.info(`DeletePlaylistData - ${JSON.stringify(resp)}`) const txReceipt = resp.txReceipt - return { - blockHash: txReceipt.blockHash, - blockNumber: txReceipt.blockNumber - } + responseValues.blockHash = txReceipt.blockHash + responseValues.blockNumber = txReceipt.blockNumber + responseValues.playlistId = playlistId + return responseValues } catch (e) { - logger.error(`Data delete playlist: err ${e}`) - throw e + const error = (e as Error).message + responseValues.error = error + return responseValues } } @@ -198,12 +219,7 @@ export class EntityManager extends Base { isPrivate: boolean coverArt: string logger: Console - }): Promise<{ - blockHash: Nullable - blockNumber: Nullable - playlistId: Nullable - error: Nullable - }> { + }): Promise { let error = null try { const currentUserId: string | null = From 0eb7c5f0eda45526ce98b2b286708e40d3bc933d Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Fri, 15 Jul 2022 18:05:30 +0000 Subject: [PATCH 077/101] Add EntityManager functionality to libs, minor local dev updates, new pre-baked image --- contracts/scripts/migrate-contracts.js | 2 +- libs/data-contracts/ABIs/EntityManager.json | 174 ++++++++++ libs/data-contracts/signatureSchemas.ts | 50 ++- libs/initScripts/configureLocalDiscProv.js | 25 ++ libs/src/api/entityManager.ts | 327 ++++++++++++++++++ libs/src/libs.js | 10 +- .../services/ABIDecoder/AudiusABIDecoder.ts | 3 +- libs/src/services/creatorNode/CreatorNode.ts | 57 +++ .../services/dataContracts/AudiusContracts.ts | 18 + .../dataContracts/EntityManagerClient.ts | 48 +++ .../discoveryProvider/DiscoveryProvider.ts | 2 +- .../schemaValidator/SchemaValidator.ts | 12 +- .../schemas/playlistSchema.json | 56 +++ .../src/commands/service-commands.json | 2 +- 14 files changed, 776 insertions(+), 10 deletions(-) create mode 100644 libs/data-contracts/ABIs/EntityManager.json create mode 100644 libs/src/api/entityManager.ts create mode 100644 libs/src/services/dataContracts/EntityManagerClient.ts create mode 100644 libs/src/services/schemaValidator/schemas/playlistSchema.json diff --git a/contracts/scripts/migrate-contracts.js b/contracts/scripts/migrate-contracts.js index 2e75a365f1e..85386f9a19a 100644 --- a/contracts/scripts/migrate-contracts.js +++ b/contracts/scripts/migrate-contracts.js @@ -78,7 +78,7 @@ const outputJsonConfigFile = async (outputPath) => { let addressInfo = require(migrationOutputPath) let outputDictionary = {} outputDictionary['registryAddress'] = addressInfo.registryAddress - outputDictionary['audiusDataAddress'] = addressInfo.audiusDataProxyAddress + outputDictionary['entityManagerProxyAddress'] = addressInfo.entityManagerProxyAddress outputDictionary['ursmAddress'] = addressInfo.ursmAddress outputDictionary['ownerWallet'] = await getDefaultAccount() outputDictionary['allWallets'] = await web3.eth.getAccounts() diff --git a/libs/data-contracts/ABIs/EntityManager.json b/libs/data-contracts/ABIs/EntityManager.json new file mode 100644 index 00000000000..f7398fb21e8 --- /dev/null +++ b/libs/data-contracts/ABIs/EntityManager.json @@ -0,0 +1,174 @@ +{ + "contractName": "EntityManager", + "abi": [ + { + "constant": true, + "inputs": [ + { + "name": "", + "type": "bytes32" + } + ], + "name": "usedSignatures", + "outputs": [ + { + "name": "", + "type": "bool" + } + ], + "payable": false, + "stateMutability": "view", + "type": "function" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": false, + "name": "_userId", + "type": "uint256" + }, + { + "indexed": false, + "name": "_signer", + "type": "address" + }, + { + "indexed": false, + "name": "_entityType", + "type": "string" + }, + { + "indexed": false, + "name": "_entityId", + "type": "uint256" + }, + { + "indexed": false, + "name": "_metadata", + "type": "string" + }, + { + "indexed": false, + "name": "_action", + "type": "string" + } + ], + "name": "ManageEntity", + "type": "event" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": false, + "name": "_userId", + "type": "uint256" + }, + { + "indexed": false, + "name": "_isVerified", + "type": "bool" + } + ], + "name": "ManageIsVerified", + "type": "event" + }, + { + "constant": false, + "inputs": [ + { + "name": "name", + "type": "string" + }, + { + "name": "version", + "type": "string" + }, + { + "name": "chainId", + "type": "uint256" + } + ], + "name": "initialize", + "outputs": [], + "payable": false, + "stateMutability": "nonpayable", + "type": "function" + }, + { + "constant": false, + "inputs": [ + { + "name": "_verifierAddress", + "type": "address" + }, + { + "name": "_networkId", + "type": "uint256" + } + ], + "name": "initialize", + "outputs": [], + "payable": false, + "stateMutability": "nonpayable", + "type": "function" + }, + { + "constant": false, + "inputs": [ + { + "name": "_userId", + "type": "uint256" + }, + { + "name": "_entityType", + "type": "string" + }, + { + "name": "_entityId", + "type": "uint256" + }, + { + "name": "_action", + "type": "string" + }, + { + "name": "_metadata", + "type": "string" + }, + { + "name": "_nonce", + "type": "bytes32" + }, + { + "name": "_subjectSig", + "type": "bytes" + } + ], + "name": "manageEntity", + "outputs": [], + "payable": false, + "stateMutability": "nonpayable", + "type": "function" + }, + { + "constant": false, + "inputs": [ + { + "name": "_userId", + "type": "uint256" + }, + { + "name": "_isVerified", + "type": "bool" + } + ], + "name": "manageIsVerified", + "outputs": [], + "payable": false, + "stateMutability": "nonpayable", + "type": "function" + } + ] +} \ No newline at end of file diff --git a/libs/data-contracts/signatureSchemas.ts b/libs/data-contracts/signatureSchemas.ts index fa6c64c5f7f..332064b8a3e 100644 --- a/libs/data-contracts/signatureSchemas.ts +++ b/libs/data-contracts/signatureSchemas.ts @@ -64,6 +64,10 @@ const getUserReplicaSetManagerDomain: DomainFn = (chainId, contractAddress) => { ) } +const getEntityManagerDomain: DomainFn = (chainId, contractAddress) => { + return getDomainData('Entity Manager', '1', chainId, contractAddress) +} + export const domains = { getSocialFeatureFactoryDomain, getUserFactoryDomain, @@ -71,7 +75,8 @@ export const domains = { getPlaylistFactoryDomain, getUserLibraryFactoryDomain, getIPLDBlacklistFactoryDomain, - getUserReplicaSetManagerDomain + getUserReplicaSetManagerDomain, + getEntityManagerDomain } /* contract signing domain */ @@ -258,6 +263,15 @@ const updateReplicaSet = [ { name: 'nonce', type: 'bytes32' } ] +const manageEntity = [ + { name: 'userId', type: 'uint' }, + { name: 'entityType', type: 'string' }, + { name: 'entityId', type: 'uint' }, + { name: 'action', type: 'string' }, + { name: 'metadata', type: 'string' }, + { name: 'nonce', type: 'bytes32' } +] + export const schemas = { domain, addUserRequest, @@ -289,7 +303,8 @@ export const schemas = { deletePlaylistSaveRequest, addIPLDBlacklist, proposeAddOrUpdateContentNode, - updateReplicaSet + updateReplicaSet, + manageEntity } type MessageSchema = readonly EIP712TypeProperty[] @@ -1145,6 +1160,34 @@ const getUpdateReplicaSetRequestData = ( ) } +const getManageEntityData = ( + chainId: number, + contractAddress: string, + userId: number, + entityType: string, + entityId: number, + action: string, + metadata: string, + nonce: string +) => { + const message = { + userId, + entityType, + entityId, + action, + metadata, + nonce + } + return getRequestData( + domains.getEntityManagerDomain, + chainId, + contractAddress, + 'ManageEntity', + schemas.manageEntity, + message + ) +} + export const generators = { getUpdateUserMultihashRequestData, getAddUserRequestData, @@ -1181,7 +1224,8 @@ export const generators = { getUpdatePlaylistDescriptionRequestData, addIPLDToBlacklistRequestData, getProposeAddOrUpdateContentNodeRequestData, - getUpdateReplicaSetRequestData + getUpdateReplicaSetRequestData, + getManageEntityData } type NodeCrypto = { randomBytes: (size: number) => Buffer } diff --git a/libs/initScripts/configureLocalDiscProv.js b/libs/initScripts/configureLocalDiscProv.js index 353a3fb1fc7..d79dffa6fcd 100644 --- a/libs/initScripts/configureLocalDiscProv.js +++ b/libs/initScripts/configureLocalDiscProv.js @@ -2,9 +2,12 @@ const fs = require('fs') const path = require('path') const readline = require('readline') const ethContractsMigrationOutput = require('../../eth-contracts/migrations/migration-output.json') +const dataContractsMigrationOutput = require('../../contracts/migrations/migration-output.json') const solanaConfig = require('../../solana-programs/solana-program-config.json') const ETH_CONTRACTS_REGISTRY = 'audius_eth_contracts_registry' +const CONTRACTS_REGISTRY = 'audius_contracts_registry' +const ENTITY_MANAGER_ADDRESS = 'audius_contracts_entity_manager_address' const SOLANA_TRACK_LISTEN_COUNT_ADDRESS = 'audius_solana_track_listen_count_address' const SOLANA_ENDPOINT = 'audius_solana_endpoint' @@ -23,6 +26,8 @@ const SOLANA_ANCHOR_ADMIN_STORAGE_PUBLIC_KEY = 'audius_solana_anchor_admin_stora // Updates audius_eth_contracts_registry in discovery provider const configureLocalDiscProv = async () => { const ethRegistryAddress = ethContractsMigrationOutput.registryAddress + const dataRegistryAddress = dataContractsMigrationOutput.registryAddress + const entityManagerAddress = dataContractsMigrationOutput.entityManagerProxyAddress const solanaTrackListenCountAddress = solanaConfig.trackListenCountAddress const signerGroup = solanaConfig.signerGroup const solanaEndpoint = solanaConfig.endpoint @@ -48,6 +53,8 @@ const configureLocalDiscProv = async () => { rewardsManagerAccount, anchorProgramId, anchorAdminStoragePublicKey, + dataRegistryAddress, + entityManagerAddress ) } @@ -65,6 +72,8 @@ const _updateDiscoveryProviderEnvFile = async ( rewardsManagerAccount, anchorProgramId, anchorAdminStoragePublicKey, + dataRegistryAddress, + entityManagerAddress ) => { const fileStream = fs.createReadStream(readPath) const rl = readline.createInterface({ @@ -83,6 +92,8 @@ const _updateDiscoveryProviderEnvFile = async ( let rewardsAccountFound = false let anchorProgramIdFound = false let anchorAdminStoragePublicKeyFound = false + let dataRegistryLineFound = false + let dataContractLineFound = false const ethRegistryAddressLine = `${ETH_CONTRACTS_REGISTRY}=${ethRegistryAddress}` const solanaTrackListenCountAddressLine = `${SOLANA_TRACK_LISTEN_COUNT_ADDRESS}=${solanaTrackListenCountAddress}` @@ -94,6 +105,8 @@ const _updateDiscoveryProviderEnvFile = async ( const rewardsManagerAccountLine = `${SOLANA_REWARDS_MANAGER_ACCOUNT}=${rewardsManagerAccount}` const anchorProgramIdLine = `${SOLANA_ANCHOR_PROGRAM_ID}=${anchorProgramId}` const anchorAdminStoragePublicKeyLine = `${SOLANA_ANCHOR_ADMIN_STORAGE_PUBLIC_KEY}=${anchorAdminStoragePublicKey}` + const dataRegistryLine = `${CONTRACTS_REGISTRY}=${dataRegistryAddress}` + const entityManagerContractLine = `${ENTITY_MANAGER_ADDRESS}=${entityManagerAddress}` for await (const line of rl) { if (line.includes(ETH_CONTRACTS_REGISTRY)) { @@ -126,6 +139,12 @@ const _updateDiscoveryProviderEnvFile = async ( } else if (line.includes(SOLANA_ANCHOR_ADMIN_STORAGE_PUBLIC_KEY)) { output.push(anchorAdminStoragePublicKeyLine) anchorAdminStoragePublicKeyFound = true + } else if (line.includes(CONTRACTS_REGISTRY)) { + output.push(dataRegistryLine) + dataRegistryLineFound = true + } else if (line.includes(ENTITY_MANAGER_ADDRESS)) { + output.push(entityManagerContractLine) + dataContractLineFound = true } else { output.push(line) } @@ -160,6 +179,12 @@ const _updateDiscoveryProviderEnvFile = async ( if (!anchorAdminStoragePublicKeyFound) { output.push(anchorAdminStoragePublicKeyLine) } + if (!dataRegistryLineFound) { + output.push(dataRegistryLine) + } + if (!dataContractLineFound) { + output.push(entityManagerContractLine) + } fs.writeFileSync(writePath, output.join('\n')) console.log(`Updated DISCOVERY PROVIDER ${writePath} ${ETH_CONTRACTS_REGISTRY}=${ethRegistryAddress} ${output}`) } diff --git a/libs/src/api/entityManager.ts b/libs/src/api/entityManager.ts new file mode 100644 index 00000000000..e18e0b6df00 --- /dev/null +++ b/libs/src/api/entityManager.ts @@ -0,0 +1,327 @@ +import { Base, BaseConstructorArgs, Services } from './base' +import type { Nullable } from '../utils' + +export enum Action { + CREATE = 'Create', + UPDATE = 'Update', + DELETE = 'Delete' +} + +export enum EntityType { + PLAYLIST = 'Playlist' +} + +export interface PlaylistOperationResponse { + /** + * Blockhash of playlist transaction + */ + blockHash: Nullable + /** + * Block number of playlist transaction + */ + blockNumber: Nullable + /** + * ID of playlist being modified + */ + playlistId: Nullable + /** + * String error message returned + */ + error: Nullable +} + +// Minimum playlist ID, intentionally higher than legacy playlist ID range +const MIN_PLAYLIST_ID = 400000 +// Maximum playlist ID, reflects postgres max integer value +const MAX_PLAYLIST_ID = 2147483647 + +/* + API surface for updated data contract interactions. + Provides simplified entity management in a generic fashion + Handles metadata + file upload etc. for entities such as Playlist/Track/User +*/ +export class EntityManager extends Base { + constructor(...args: BaseConstructorArgs) { + super(...args) + this.getValidPlaylistId = this.getValidPlaylistId.bind(this) + this.createPlaylist = this.createPlaylist.bind(this) + this.deletePlaylist = this.deletePlaylist.bind(this) + } + + /** + * Generate random integer between two known values + */ + getRandomInt(min: number, max: number): number { + min = Math.ceil(min) + max = Math.floor(max) + return Math.floor(Math.random() * (max - min) + min) + } + + /** + * Calculate an unoccupied playlist ID + * Maximum value is postgres integer max (2147483647) + * Minimum value is artificially set to 400000 + */ + async getValidPlaylistId(): Promise { + let playlistId: number = this.getRandomInt(MIN_PLAYLIST_ID, MAX_PLAYLIST_ID) + let validIdFound: boolean = false + while (!validIdFound) { + const resp: any = await this.discoveryProvider.getPlaylists(1, 0, [ + playlistId + ]) + if (resp.length !== 0) { + playlistId = this.getRandomInt(MIN_PLAYLIST_ID, MAX_PLAYLIST_ID) + } else { + validIdFound = true + } + } + return playlistId + } + + /** + * Playlist default response values + */ + getDefaultPlaylistReponseValues(): PlaylistOperationResponse { + return { + blockHash: null, + blockNumber: null, + playlistId: null, + error: null + } + } + + /** + * Create a playlist using updated data contracts flow + */ + async createPlaylist({ + playlistName, + trackIds, + description, + isAlbum, + isPrivate, + coverArt, + logger = console + }: { + playlistName: string + trackIds: number[] + description: string + isAlbum: boolean + isPrivate: boolean + coverArt: string + logger: Console + }): Promise { + const responseValues: PlaylistOperationResponse = + this.getDefaultPlaylistReponseValues() + try { + const currentUserId: string | null = + this.userStateManager.getCurrentUserId() + if (!currentUserId) { + return { + blockHash: null, + blockNumber: null, + playlistId: null, + error: 'Missing current user ID' + } + } + const userId: number = parseInt(currentUserId) + const createAction = Action.CREATE + const entityType = EntityType.PLAYLIST + const entityId = await this.getValidPlaylistId() + this.REQUIRES(Services.CREATOR_NODE) + const updatedPlaylistImage = await this.creatorNode.uploadImage( + coverArt, + true // square + ) + const dirCID = updatedPlaylistImage.dirCID + const metadata = { + createAction, + entity_type: entityType, + playlist_id: entityId, + playlist_contents: trackIds, + playlist_name: playlistName, + playlist_image_sizes_multihash: dirCID, + description, + is_album: isAlbum, + is_private: isPrivate + } + const { metadataMultihash } = + await this.creatorNode.uploadPlaylistMetadata(metadata) + const manageEntityResponse = await this.manageEntity({ + userId: userId, + entityType, + entityId, + action: createAction, + metadataMultihash + }) + const txReceipt = manageEntityResponse.txReceipt + responseValues.blockHash = txReceipt.blockHash + responseValues.blockNumber = txReceipt.blockNumber + responseValues.playlistId = entityId + return responseValues + } catch (e) { + const error = (e as Error).message + responseValues.error = error + return responseValues + } + } + + /** + * Delete a playlist using updated data contracts flow + */ + async deletePlaylist({ + playlistId, + userId + }: { + playlistId: number + userId: number + logger: any + }): Promise { + const responseValues: PlaylistOperationResponse = + this.getDefaultPlaylistReponseValues() + try { + const resp = await this.manageEntity({ + userId, + entityType: EntityType.PLAYLIST, + entityId: playlistId, + action: Action.DELETE, + metadataMultihash: '' + }) + const txReceipt = resp.txReceipt + responseValues.blockHash = txReceipt.blockHash + responseValues.blockNumber = txReceipt.blockNumber + responseValues.playlistId = playlistId + return responseValues + } catch (e) { + const error = (e as Error).message + responseValues.error = error + return responseValues + } + } + + /** + * Update a playlist using updated data contracts flow + **/ + async updatePlaylist({ + playlistId, + playlistName, + trackIds, + description, + isAlbum, + isPrivate, + coverArt, + logger = console + }: { + playlistId: number + playlistName: string + trackIds: number[] + description: string + isAlbum: boolean + isPrivate: boolean + coverArt: string + logger: Console + }): Promise { + let error = null + try { + const currentUserId: string | null = + this.userStateManager.getCurrentUserId() + if (!currentUserId) { + return { + blockHash: null, + blockNumber: null, + playlistId: null, + error: 'Missing current user ID' + } + } + const userId: number = parseInt(currentUserId) + const updateAction = Action.UPDATE + const entityType = EntityType.PLAYLIST + this.REQUIRES(Services.CREATOR_NODE) + let dirCID + if (coverArt) { + const updatedPlaylistImage = await this.creatorNode.uploadImage( + coverArt, + true // square + ) + dirCID = updatedPlaylistImage.dirCID + } + + const playlist: any = ( + await this.discoveryProvider.getPlaylists(1, 0, [playlistId]) + )[0] + + const metadata = { + action: updateAction, // why include action here? + entityType, + playlist_id: playlistId, + playlist_contents: trackIds || playlist.playlist_contents, + playlist_name: playlistName || playlist.playlist_name, + playlist_image_sizes_multihash: + dirCID || playlist.playlist_image_sizes_multihash, + description: description || playlist.description, + is_album: isAlbum || playlist.is_album, + is_private: isPrivate || playlist.is_private + } + const { metadataMultihash } = + await this.creatorNode.uploadPlaylistMetadata(metadata) + + const resp = await this.manageEntity({ + userId, + entityType, + entityId: playlistId, + action: updateAction, + metadataMultihash + }) + const txReceipt = resp.txReceipt + return { + blockHash: txReceipt.blockHash, + blockNumber: txReceipt.blockNumber, + playlistId, + error + } + } catch (e) { + error = (e as Error).message + return { + blockHash: null, + blockNumber: null, + playlistId: null, + error + } + } + } + + /** + * Manage an entity with the updated data contract flow + * Leveraged to manipulate User/Track/Playlist/+ other entities + */ + async manageEntity({ + userId, + entityType, + entityId, + action, + metadataMultihash + }: { + userId: number + entityType: EntityType + entityId: number + action: Action + metadataMultihash: string + }): Promise< + { txReceipt: any; error: null } | { txReceipt: null; error: string } + > { + let error = null + let resp: any + try { + resp = await this.contracts.EntityManagerClient?.manageEntity( + userId, + entityType, + entityId, + action, + metadataMultihash + ) + return { txReceipt: resp.txReceipt, error } + } catch (e) { + error = (e as Error).message + return { txReceipt: null, error } + } + } +} diff --git a/libs/src/libs.js b/libs/src/libs.js index 7b09608ac3e..17ba556d8e5 100644 --- a/libs/src/libs.js +++ b/libs/src/libs.js @@ -34,6 +34,7 @@ const { RewardsAttester } = require('./services/solanaWeb3Manager/rewardsAttester') const { Reactions } = require('./api/reactions') +const { EntityManager } = require('./api/entityManager') const { getPlatformLocalStorage } = require('./utils/localStorage') class AudiusLibs { @@ -96,7 +97,8 @@ class AudiusLibs { registryAddress, web3Provider, networkId, - walletOverride = null + walletOverride = null, + entityManagerAddress = null ) { const web3Instance = await Utils.configureWeb3(web3Provider, networkId) if (!web3Instance) { @@ -105,6 +107,7 @@ class AudiusLibs { const wallets = await web3Instance.eth.getAccounts() return { registryAddress, + entityManagerAddress, useExternalWeb3: true, externalWeb3Config: { web3: web3Instance, @@ -118,7 +121,7 @@ class AudiusLibs { * @param {string} registryAddress * @param {string | Web3 | Array} providers web3 provider endpoint(s) */ - static configInternalWeb3 (registryAddress, providers, privateKey) { + static configInternalWeb3 (registryAddress, providers, privateKey, entityManagerAddress = null) { let providerList if (typeof providers === 'string') { providerList = providers.split(',') @@ -134,6 +137,7 @@ class AudiusLibs { return { registryAddress, + entityManagerAddress, useExternalWeb3: false, internalWeb3Config: { web3ProviderEndpoints: providerList, @@ -470,6 +474,7 @@ class AudiusLibs { this.contracts = new AudiusContracts( this.web3Manager, this.web3Config ? this.web3Config.registryAddress : null, + this.web3Config ? this.web3Config.entityManagerAddress : null, this.isServer, this.logger ) @@ -569,6 +574,7 @@ class AudiusLibs { this.File = new File(this.User, ...services) this.Rewards = new Rewards(this.ServiceProvider, ...services) this.Reactions = new Reactions(...services) + this.EntityManager = new EntityManager(...services) } } diff --git a/libs/src/services/ABIDecoder/AudiusABIDecoder.ts b/libs/src/services/ABIDecoder/AudiusABIDecoder.ts index 7e9b95d1543..3f6a4af771a 100644 --- a/libs/src/services/ABIDecoder/AudiusABIDecoder.ts +++ b/libs/src/services/ABIDecoder/AudiusABIDecoder.ts @@ -19,6 +19,7 @@ loadABI('SocialFeatureFactory.json') loadABI('PlaylistFactory.json') loadABI('UserLibraryFactory.json') loadABI('UserReplicaSetManager.json') +loadABI('EntityManager.json') // eslint-disable-next-line @typescript-eslint/no-extraneous-class -- should just use esm export class AudiusABIDecoder { @@ -32,7 +33,7 @@ export class AudiusABIDecoder { // namespace of functions) const abi = abiMap[contractName] if (!abi) { - throw new Error('Unrecognized contract name') + throw new Error(`Unrecognized contract name ${contractName}`) } let foundFunction: AbiItem | undefined diff --git a/libs/src/services/creatorNode/CreatorNode.ts b/libs/src/services/creatorNode/CreatorNode.ts index 3a0b67c951b..6149898756c 100644 --- a/libs/src/services/creatorNode/CreatorNode.ts +++ b/libs/src/services/creatorNode/CreatorNode.ts @@ -5,6 +5,7 @@ import { Utils, uuid } from '../../utils' import { userSchemaType, trackSchemaType, + playlistSchemaType, Schemas } from '../schemaValidator/SchemaValidator' import type { Web3Manager } from '../web3Manager' @@ -25,6 +26,16 @@ type Metadata = { cover_art_sizes: string } +type PlaylistMetadata = { + playlist_contents: unknown + playlist_id: number + playlist_name: string + playlist_image_sizes_multihash: string + description: string + is_album: boolean + is_private: boolean +} + type ProgressCB = (loaded: number, total: number) => void export type MonitoringCallbacks = { @@ -406,6 +417,52 @@ export class CreatorNode { return body } + /** + * Uploads playlist metadata to a creator node + * source file must be provided (returned from uploading track content). + * @param metadata + */ + async uploadPlaylistMetadata(metadata: PlaylistMetadata) { + this.schemas[playlistSchemaType].validate?.(metadata) + const { data: body } = await this._makeRequest( + { + url: '/playlists/metadata', + method: 'post', + data: { + metadata + } + }, + true + ) + return body + } + + /** + * Associate an uploaded playlist metadata file with a blockchainId + * @param blockchainId - Valid ID assigned to playlist + * @param metadataFileUUID unique ID for metadata playlist + * @param blockNumber + */ + async associatePlaylistMetadata( + blockchainId: number, + metadataFileUUID: string, + blockNumber: number + ) { + const { data: body } = await this._makeRequest( + { + url: '/playlists', + method: 'post', + data: { + blockchainId, + metadataFileUUID, + blockNumber + } + }, + true + ) + return body + } + /** * Creates a track on the content node, associating track id with file content * @param audiusTrackId returned by track creation on-blockchain diff --git a/libs/src/services/dataContracts/AudiusContracts.ts b/libs/src/services/dataContracts/AudiusContracts.ts index 8c170e02dd6..ccd384613ef 100644 --- a/libs/src/services/dataContracts/AudiusContracts.ts +++ b/libs/src/services/dataContracts/AudiusContracts.ts @@ -9,6 +9,7 @@ import { PlaylistFactoryClient } from './PlaylistFactoryClient' import { UserLibraryFactoryClient } from './UserLibraryFactoryClient' import { IPLDBlacklistFactoryClient } from './IPLDBlacklistFactoryClient' import { UserReplicaSetManagerClient } from './UserReplicaSetManagerClient' +import { EntityManagerClient } from './EntityManagerClient' import type { Web3Manager } from '../web3Manager' import type { ContractClient } from '../contracts/ContractClient' @@ -32,6 +33,7 @@ const IPLDBlacklistFactoryABI = Utils.importDataContractABI( const UserReplicaSetManagerABI = Utils.importDataContractABI( 'UserReplicaSetManager.json' ).abi +const EntityManagerABI = Utils.importDataContractABI('EntityManager.json').abi // define contract registry keys const UserFactoryRegistryKey = 'UserFactory' @@ -45,6 +47,7 @@ const UserReplicaSetManagerRegistryKey = 'UserReplicaSetManager' export class AudiusContracts { web3Manager: Web3Manager registryAddress: string + entityManagerAddress: string isServer: boolean logger: Logger RegistryClient: RegistryClient @@ -54,6 +57,7 @@ export class AudiusContracts { PlaylistFactoryClient: PlaylistFactoryClient UserLibraryFactoryClient: UserLibraryFactoryClient IPLDBlacklistFactoryClient: IPLDBlacklistFactoryClient + EntityManagerClient: EntityManagerClient | undefined contractClients: ContractClient[] UserReplicaSetManagerClient: UserReplicaSetManagerClient | undefined | null contracts: Record | undefined @@ -62,11 +66,13 @@ export class AudiusContracts { constructor( web3Manager: Web3Manager, registryAddress: string, + entityManagerAddress: string, isServer: boolean, logger = console ) { this.web3Manager = web3Manager this.registryAddress = registryAddress + this.entityManagerAddress = entityManagerAddress this.isServer = isServer this.logger = logger @@ -134,6 +140,18 @@ export class AudiusContracts { this.UserLibraryFactoryClient, this.IPLDBlacklistFactoryClient ] + + if (this.entityManagerAddress) { + this.EntityManagerClient = new EntityManagerClient( + this.web3Manager, + EntityManagerABI, + 'EntityManager', + this.getRegistryAddressForContract, + this.logger, + this.entityManagerAddress + ) + this.contractClients.push(this.EntityManagerClient) + } } async init() { diff --git a/libs/src/services/dataContracts/EntityManagerClient.ts b/libs/src/services/dataContracts/EntityManagerClient.ts new file mode 100644 index 00000000000..dbb8765c79b --- /dev/null +++ b/libs/src/services/dataContracts/EntityManagerClient.ts @@ -0,0 +1,48 @@ +import { ContractClient } from '../contracts/ContractClient' +import * as signatureSchemas from '../../../data-contracts/signatureSchemas' +import type { Web3Manager } from '../web3Manager' + +export class EntityManagerClient extends ContractClient { + override web3Manager!: Web3Manager + + async manageEntity( + userId: number, + entityType: string, + entityId: number, + action: string, + metadata: string + ): Promise<{ txReceipt: any }> { + const nonce = signatureSchemas.getNonce() + const chainId = await this.getEthNetId() + const contractAddress = await this.getAddress() + const signatureData = signatureSchemas.generators.getManageEntityData( + chainId, + contractAddress, + userId, + entityType, + entityId, + action, + metadata, + nonce + ) + const sig = await this.web3Manager.signTypedData(signatureData) + const method = await this.getMethod( + 'manageEntity', + userId, + entityType, + entityId, + action, + metadata, + nonce, + sig + ) + const tx = await this.web3Manager.sendTransaction( + method, + this.contractRegistryKey, + contractAddress + ) + return { + txReceipt: tx + } + } +} diff --git a/libs/src/services/discoveryProvider/DiscoveryProvider.ts b/libs/src/services/discoveryProvider/DiscoveryProvider.ts index da5935e4610..22d270f84a4 100644 --- a/libs/src/services/discoveryProvider/DiscoveryProvider.ts +++ b/libs/src/services/discoveryProvider/DiscoveryProvider.ts @@ -399,7 +399,7 @@ export class DiscoveryProvider { idsArray = null, targetUserId = null, withUsers = false - ) { + ): Promise { const req = Requests.getPlaylists( limit, offset, diff --git a/libs/src/services/schemaValidator/SchemaValidator.ts b/libs/src/services/schemaValidator/SchemaValidator.ts index a843db0a67f..a555afcb075 100644 --- a/libs/src/services/schemaValidator/SchemaValidator.ts +++ b/libs/src/services/schemaValidator/SchemaValidator.ts @@ -2,9 +2,11 @@ import { validate } from 'jsonschema' import TrackSchema from './schemas/trackSchema.json' import UserSchema from './schemas/userSchema.json' +import PlaylistSchema from './schemas/playlistSchema.json' export const trackSchemaType = 'TrackSchema' export const userSchemaType = 'UserSchema' +export const playlistSchemaType = 'PlaylistSchema' type SchemaConfig = { schema: { @@ -18,11 +20,15 @@ type SchemaConfig = { validate?: (obj: Record) => void } -type SchemaType = typeof trackSchemaType | typeof userSchemaType +type SchemaType = + | typeof trackSchemaType + | typeof userSchemaType + | typeof playlistSchemaType export type Schemas = { TrackSchema: SchemaConfig UserSchema: SchemaConfig + PlaylistSchema: SchemaConfig } export class SchemaValidator { @@ -50,6 +56,10 @@ export class SchemaValidator { [userSchemaType]: { schema: UserSchema, baseDefinition: 'User' + }, + [playlistSchemaType]: { + schema: PlaylistSchema, + baseDefinition: 'Playlist' } } diff --git a/libs/src/services/schemaValidator/schemas/playlistSchema.json b/libs/src/services/schemaValidator/schemas/playlistSchema.json new file mode 100644 index 00000000000..34ee4fee23b --- /dev/null +++ b/libs/src/services/schemaValidator/schemas/playlistSchema.json @@ -0,0 +1,56 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$ref": "#/definitions/Playlist", + "definitions": { + "Playlist": { + "type": "object", + "additionalProperties": true, + "properties": { + "playlist_id": { + "type": ["integer", "null"], + "default": null + }, + "playlist_contents": { + "type": ["array", "null"], + "default": [] + }, + "playlist_name": { + "type": ["string", "null"], + "default": null + }, + "playlist_image_sizes_multihash": { + "type": ["string", "null"], + "default": null, + "$ref": "#/definitions/CID" + }, + "description": { + "type": ["string", "null"], + "default": null + }, + "is_album": { + "type": "boolean", + "default": null + }, + "is_private": { + "type": "boolean", + "default": null + } + }, + "required": [ + "playlist_name", + "playlist_id", + "description", + "is_album", + "is_private" + ], + "title": "Playlist" + }, + "CID": { + "type": ["string", "null"], + "minLength": 46, + "maxLength": 46, + "pattern": "^Qm[a-zA-Z0-9]{44}$", + "title": "CID" + } + } +} diff --git a/service-commands/src/commands/service-commands.json b/service-commands/src/commands/service-commands.json index f983613114e..95d3042d962 100644 --- a/service-commands/src/commands/service-commands.json +++ b/service-commands/src/commands/service-commands.json @@ -43,7 +43,7 @@ }, "contracts-predeployed": { "up": [ - "docker run --name audius_ganache_cli -d -p 8545:8545 --network=audius_dev audius/ganache:data-contracts-predeployed-a25931b955adce92676cec029661533f80013908", + "docker run --name audius_ganache_cli -d -p 8545:8545 --network=audius_dev audius/ganache:data-contracts-predeployed-f06dbfade0172e99d368882ab192e7377d6d6761", "docker cp audius_ganache_cli:/app/contract-config.json $PROTOCOL_DIR/identity-service/contract-config.json", "docker cp audius_ganache_cli:/app/contract-config.json $PROTOCOL_DIR/creator-node/contract-config.json", "docker cp audius_ganache_cli:/app/contract-config.json ~/.audius/contract-config.json", From 246cefeb19507138c203157090cce62d1fe434bd Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Mon, 18 Jul 2022 14:37:58 +0000 Subject: [PATCH 078/101] Reorder imports --- dev-tools/startup/socks5-proxy-pac.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-tools/startup/socks5-proxy-pac.py b/dev-tools/startup/socks5-proxy-pac.py index cab5b340874..e2c30f3eb32 100644 --- a/dev-tools/startup/socks5-proxy-pac.py +++ b/dev-tools/startup/socks5-proxy-pac.py @@ -1,8 +1,8 @@ #!/usr/bin/env python -import socket import http import http.server +import socket IP_ADDRESS = socket.gethostbyname(socket.gethostname()) TLD = socket.getfqdn().split(".")[-1] From 52fc104773e340aa912ccf125aeb028433511a37 Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Mon, 18 Jul 2022 16:12:19 +0000 Subject: [PATCH 079/101] Addressing PR comments, removed entity type / entity action from metadata --- libs/src/api/entityManager.ts | 9 +++------ libs/src/services/creatorNode/CreatorNode.ts | 2 +- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/libs/src/api/entityManager.ts b/libs/src/api/entityManager.ts index e18e0b6df00..4d2041934a5 100644 --- a/libs/src/api/entityManager.ts +++ b/libs/src/api/entityManager.ts @@ -1,4 +1,5 @@ import { Base, BaseConstructorArgs, Services } from './base' +import type { PlaylistMetadata } from '../services/creatorNode' import type { Nullable } from '../utils' export enum Action { @@ -133,9 +134,7 @@ export class EntityManager extends Base { true // square ) const dirCID = updatedPlaylistImage.dirCID - const metadata = { - createAction, - entity_type: entityType, + const metadata: PlaylistMetadata = { playlist_id: entityId, playlist_contents: trackIds, playlist_name: playlistName, @@ -249,9 +248,7 @@ export class EntityManager extends Base { await this.discoveryProvider.getPlaylists(1, 0, [playlistId]) )[0] - const metadata = { - action: updateAction, // why include action here? - entityType, + const metadata: PlaylistMetadata = { playlist_id: playlistId, playlist_contents: trackIds || playlist.playlist_contents, playlist_name: playlistName || playlist.playlist_name, diff --git a/libs/src/services/creatorNode/CreatorNode.ts b/libs/src/services/creatorNode/CreatorNode.ts index 6149898756c..74e64c95332 100644 --- a/libs/src/services/creatorNode/CreatorNode.ts +++ b/libs/src/services/creatorNode/CreatorNode.ts @@ -26,7 +26,7 @@ type Metadata = { cover_art_sizes: string } -type PlaylistMetadata = { +export type PlaylistMetadata = { playlist_contents: unknown playlist_id: number playlist_name: string From f08e4d8257a3a8a907472b283a17bcf58908de93 Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Mon, 18 Jul 2022 23:01:51 +0000 Subject: [PATCH 080/101] Minor touchups, fixed playlist_content parsing --- libs/src/api/entityManager.ts | 38 +++++++++++++------- libs/src/services/creatorNode/CreatorNode.ts | 8 ++++- 2 files changed, 32 insertions(+), 14 deletions(-) diff --git a/libs/src/api/entityManager.ts b/libs/src/api/entityManager.ts index 4d2041934a5..612a71ae834 100644 --- a/libs/src/api/entityManager.ts +++ b/libs/src/api/entityManager.ts @@ -44,9 +44,6 @@ const MAX_PLAYLIST_ID = 2147483647 export class EntityManager extends Base { constructor(...args: BaseConstructorArgs) { super(...args) - this.getValidPlaylistId = this.getValidPlaylistId.bind(this) - this.createPlaylist = this.createPlaylist.bind(this) - this.deletePlaylist = this.deletePlaylist.bind(this) } /** @@ -64,6 +61,7 @@ export class EntityManager extends Base { * Minimum value is artificially set to 400000 */ async getValidPlaylistId(): Promise { + // TODO: Confirm collision of ID with disc prov endpoint to account for hidden / private let playlistId: number = this.getRandomInt(MIN_PLAYLIST_ID, MAX_PLAYLIST_ID) let validIdFound: boolean = false while (!validIdFound) { @@ -211,23 +209,31 @@ export class EntityManager extends Base { logger = console }: { playlistId: number - playlistName: string - trackIds: number[] - description: string - isAlbum: boolean - isPrivate: boolean - coverArt: string + playlistName: Nullable + trackIds: Nullable + description: Nullable + isAlbum: Nullable + isPrivate: Nullable + coverArt: Nullable logger: Console }): Promise { let error = null try { const currentUserId: string | null = this.userStateManager.getCurrentUserId() + if (!playlistId || playlistId === undefined) { + return { + blockHash: null, + blockNumber: null, + playlistId, + error: 'Missing current playlistId' + } + } if (!currentUserId) { return { blockHash: null, blockNumber: null, - playlistId: null, + playlistId, error: 'Missing current user ID' } } @@ -243,14 +249,20 @@ export class EntityManager extends Base { ) dirCID = updatedPlaylistImage.dirCID } - const playlist: any = ( - await this.discoveryProvider.getPlaylists(1, 0, [playlistId]) + await this.discoveryProvider.getPlaylists(1, 0, [playlistId], userId) )[0] + let playlistContents = trackIds + if (playlist.playlist_contents) { + playlistContents = playlist.playlist_contents.track_ids.map( + (x: { [x: string]: any }) => x['track'] + ) + } + const metadata: PlaylistMetadata = { playlist_id: playlistId, - playlist_contents: trackIds || playlist.playlist_contents, + playlist_contents: playlistContents, playlist_name: playlistName || playlist.playlist_name, playlist_image_sizes_multihash: dirCID || playlist.playlist_image_sizes_multihash, diff --git a/libs/src/services/creatorNode/CreatorNode.ts b/libs/src/services/creatorNode/CreatorNode.ts index 74e64c95332..aa799297ede 100644 --- a/libs/src/services/creatorNode/CreatorNode.ts +++ b/libs/src/services/creatorNode/CreatorNode.ts @@ -423,7 +423,13 @@ export class CreatorNode { * @param metadata */ async uploadPlaylistMetadata(metadata: PlaylistMetadata) { - this.schemas[playlistSchemaType].validate?.(metadata) + // Validate object before sending + try { + this.schemas[playlistSchemaType].validate?.(metadata) + } catch (e) { + console.error('Error validating playlist metadata', e) + } + const { data: body } = await this._makeRequest( { url: '/playlists/metadata', From 4fb6ce556c3a742a9041dff74c9d7c6586920203 Mon Sep 17 00:00:00 2001 From: Isaac Solo Date: Mon, 18 Jul 2022 16:26:03 -0700 Subject: [PATCH 081/101] Playlist validation and batching (#3484) --- discovery-provider/src/tasks/audius_data.py | 402 +++++++++++++------- discovery-provider/src/tasks/index.py | 3 +- libs/eth-contracts/ABIs/Wormhole.json | 132 ++++++- libs/src/api/entityManager.ts | 9 +- 4 files changed, 393 insertions(+), 153 deletions(-) diff --git a/discovery-provider/src/tasks/audius_data.py b/discovery-provider/src/tasks/audius_data.py index 68fe3de9b99..2a19f57d079 100644 --- a/discovery-provider/src/tasks/audius_data.py +++ b/discovery-provider/src/tasks/audius_data.py @@ -1,27 +1,36 @@ import logging +from collections import defaultdict from datetime import datetime from enum import Enum -from typing import Any, Dict, Set, Tuple +from typing import Any, Dict, List, Set, Tuple from sqlalchemy.orm.session import Session, make_transient from src.database_task import DatabaseTask from src.models.playlists.playlist import Playlist +from src.models.users.user import User from src.tasks.playlists import invalidate_old_playlist from src.utils import helpers from src.utils.user_event_constants import audius_data_event_types_arr +from web3.datastructures import AttributeDict logger = logging.getLogger(__name__) -class Action(Enum): +class Action(str, Enum): CREATE = "Create" UPDATE = "Update" DELETE = "Delete" + def __str__(self) -> str: + return str.__str__(self) -class EntityType(Enum): + +class EntityType(str, Enum): PLAYLIST = "Playlist" + def __str__(self) -> str: + return str.__str__(self) + def audius_data_state_update( self, @@ -31,166 +40,287 @@ def audius_data_state_update( block_number, block_timestamp, block_hash, - ipfs_metadata, + ipfs_metadata ) -> Tuple[int, Dict[str, Set[(int)]]]: - num_total_changes = 0 + try: + num_total_changes = 0 + event_blockhash = update_task.web3.toHex(block_hash) - changed_entity_ids: Dict[str, Set[(int)]] = {} + changed_entity_ids: Dict[str, Set[(int)]] = defaultdict(set) - if not audius_data_txs: - return num_total_changes, changed_entity_ids + if not audius_data_txs: + return num_total_changes, changed_entity_ids - playlist_events_lookup: Dict[int, Dict[str, Any]] = {} - # This stores the playlist_ids created or updated in the set of transactions - playlist_ids: Set[int] = set() + entities_to_fetch: Dict[EntityType, Set[int]] = defaultdict(set) + users_to_fetch: Set[int] = set() - for tx_receipt in audius_data_txs: - logger.info(f"AudiusData.py | Processing {tx_receipt}") - txhash = update_task.web3.toHex(tx_receipt.transactionHash) - for event_type in audius_data_event_types_arr: - audius_data_event_tx = get_audius_data_events_tx( - update_task, event_type, tx_receipt - ) + # collect events by entity type and action + collect_entities_to_fetch( + update_task, audius_data_txs, entities_to_fetch, users_to_fetch + ) - processed_entries = 0 - # TODO: Batch reject operations for mismatched signer/userId - for entry in audius_data_event_tx: - user_id = helpers.get_tx_arg(entry, "_userId") - entity_id = helpers.get_tx_arg(entry, "_entityId") - entity_type = helpers.get_tx_arg(entry, "_entityType") - action = helpers.get_tx_arg(entry, "_action") - metadata_cid = helpers.get_tx_arg(entry, "_metadata") - signer = helpers.get_tx_arg(entry, "_signer") - metadata = ( - ipfs_metadata[metadata_cid] - if metadata_cid in ipfs_metadata - else None - ) - logger.info( - f"index.py | AudiusData state update: {user_id}, entity_id={entity_id}, entity_type={entity_type}, action={action}, metadata_cid={metadata_cid}, metadata={metadata} signer={signer}" + # fetch existing playlists + existing_playlist_id_to_playlist: Dict[int, Playlist] = fetch_existing_entities( + session, entities_to_fetch + ) + + # fetch users + existing_user_id_to_user: Dict[int, User] = fetch_users(session, users_to_fetch) + + playlists_to_save: Dict[int, List[Playlist]] = defaultdict(list) + # process in tx order and populate playlists_to_save + for tx_receipt in audius_data_txs: + txhash = update_task.web3.toHex(tx_receipt.transactionHash) + for event_type in audius_data_event_types_arr: + audius_data_event_tx = get_audius_data_events_tx( + update_task, event_type, tx_receipt ) - # Handle playlist creation - if entity_type == EntityType.PLAYLIST.value: - playlist_id = entity_id - logger.info( - f"index.py | AudiusData - playlist detected, id={playlist_id}" + for event in audius_data_event_tx: + params = ManagePlaylistParameters( + event, + playlists_to_save, # actions below populate these records + existing_playlist_id_to_playlist, + existing_user_id_to_user, + ipfs_metadata, + block_timestamp, + block_number, + event_blockhash, + txhash, ) - # look up or populate existing record - if playlist_id in playlist_events_lookup: - existing_playlist_record = playlist_events_lookup[playlist_id][ - "playlist" - ] - else: - existing_playlist_record = lookup_playlist_data_record( - update_task, - session, - playlist_id, - block_number, - block_hash, - txhash, - ) - - if action == Action.CREATE.value or Action.UPDATE.value: - logger.info( - f"index.py | AudiusData - handling {action}, events_lookup={playlist_events_lookup}" - ) - playlist_record = parse_playlist_create_data_event( - update_task, - entry, - user_id, - existing_playlist_record, - metadata, - block_timestamp, - session, - ) - - elif Action.DELETE.value: - existing_playlist_record.is_delete = True - playlist_record = existing_playlist_record - - if playlist_record is not None: - if playlist_id not in playlist_events_lookup: - playlist_events_lookup[playlist_id] = { - "playlist": playlist_record, - "events": [], - } - else: - playlist_events_lookup[playlist_id][ - "playlist" - ] = playlist_record - playlist_events_lookup[playlist_id]["events"].append(event_type) - playlist_ids.add(playlist_id) - processed_entries += 1 - - num_total_changes += processed_entries - - # Update changed entity dictionary - changed_entity_ids["playlist"] = playlist_ids - - for playlist_id, value_obj in playlist_events_lookup.items(): - logger.info( - f"index.py | AudiusData | playlists.py | Adding {value_obj['playlist']})" - ) - if value_obj["events"]: - invalidate_old_playlist(session, playlist_id) - session.add(value_obj["playlist"]) + if ( + params.action == Action.CREATE + and params.entity_type == EntityType.PLAYLIST + ): + create_playlist(params) + elif ( + params.action == Action.UPDATE + and params.entity_type == EntityType.PLAYLIST + ): + update_playlist(params) + + elif ( + params.action == Action.DELETE + and params.entity_type == EntityType.PLAYLIST + ): + delete_playlist(params) + + # compile records_to_save + records_to_save = [] + for _, playlist_records in playlists_to_save.items(): + # flip is_current to true for the last tx in each playlist + playlist_records[-1].is_current = True + records_to_save.extend(playlist_records) + # insert/update all playlist records in this block + session.bulk_save_objects(records_to_save) + num_total_changes += len(records_to_save) + + except Exception as e: + logger.error(f"Exception occurred {e}", exc_info=True) return num_total_changes, changed_entity_ids -def get_audius_data_events_tx(update_task, event_type, tx_receipt): - return getattr( - update_task.audius_data_contract.events, event_type - )().processReceipt(tx_receipt) +class ManagePlaylistParameters: + def __init__( + self, + event: AttributeDict, + playlists_to_save: Dict[int, List[Playlist]], + existing_playlist_id_to_playlist: Dict[int, Playlist], + existing_user_id_to_user: Dict[int, User], + ipfs_metadata: Dict[str, Dict[str, Any]], + block_timestamp: int, + block_number: int, + event_blockhash: str, + txhash: str, + ): + self.user_id = helpers.get_tx_arg(event, "_userId") + self.entity_id = helpers.get_tx_arg(event, "_entityId") + self.entity_type = helpers.get_tx_arg(event, "_entityType") + self.action = helpers.get_tx_arg(event, "_action") + self.metadata_cid = helpers.get_tx_arg(event, "_metadata") + self.signer = helpers.get_tx_arg(event, "_signer") + self.block_datetime = datetime.utcfromtimestamp(block_timestamp) + self.block_integer_time = int(block_timestamp) + self.event = event + self.ipfs_metadata = ipfs_metadata + self.existing_playlist_id_to_playlist = existing_playlist_id_to_playlist + self.block_number = block_number + self.event_blockhash = event_blockhash + self.txhash = txhash + self.existing_user_id_to_user = existing_user_id_to_user + self.playlists_to_save = playlists_to_save -def lookup_playlist_data_record( - update_task, session, playlist_id, block_number, block_hash, txhash -): - event_blockhash = update_task.web3.toHex(block_hash) - # Check if playlist record is in the DB - playlist_exists = ( - session.query(Playlist).filter_by(playlist_id=playlist_id).count() > 0 + +def create_playlist(params: ManagePlaylistParameters): + metadata = params.ipfs_metadata[params.metadata_cid] + # check if playlist already exists + if params.entity_id in params.existing_playlist_id_to_playlist: + return + + track_ids = metadata.get("playlist_contents", []) + playlist_contents = [] + for track_id in track_ids: + playlist_contents.append({"track": track_id, "time": params.block_integer_time}) + create_playlist_record = Playlist( + playlist_id=params.entity_id, + playlist_owner_id=params.user_id, + is_album=metadata.get("is_album", False), + description=metadata["description"], + playlist_image_multihash=metadata["playlist_image_sizes_multihash"], + playlist_image_sizes_multihash=metadata["playlist_image_sizes_multihash"], + playlist_name=metadata["playlist_name"], + is_private=metadata.get("is_private", False), + playlist_contents={"track_ids": playlist_contents}, + created_at=params.block_datetime, + updated_at=params.block_datetime, + blocknumber=params.block_number, + blockhash=params.event_blockhash, + txhash=params.txhash, + is_current=False, + is_delete=False, + ) + + params.playlists_to_save[params.entity_id].append(create_playlist_record) + + +def update_playlist(params: ManagePlaylistParameters): + metadata = params.ipfs_metadata[params.metadata_cid] + # check user owns playlist + if ( + params.signer.lower() + != params.existing_user_id_to_user[params.user_id].wallet.lower() + ): + return + existing_playlist = params.existing_playlist_id_to_playlist[params.entity_id] + existing_playlist.is_current = False # invalidate + if ( + params.entity_id in params.playlists_to_save + ): # override with last updated playlist is in this block + existing_playlist = params.playlists_to_save[params.entity_id][-1] + + updated_playlist = copy_record( + existing_playlist, params.block_number, params.event_blockhash, params.txhash + ) + parse_playlist_create_data_event( + updated_playlist, + metadata, + params.block_integer_time, + params.block_datetime, + ) + params.playlists_to_save[params.entity_id].append(updated_playlist) + + +def delete_playlist(params: ManagePlaylistParameters): + # check user owns playlist + if ( + params.signer.lower() + != params.existing_user_id_to_user[params.user_id].wallet.lower() + ): + return + existing_playlist = params.existing_playlist_id_to_playlist[params.entity_id] + existing_playlist.is_current = False # invalidate old playlist + if params.entity_id in params.playlists_to_save: + # override with last updated playlist is in this block + existing_playlist = params.playlists_to_save[params.entity_id][-1] + + deleted_playlist = copy_record( + existing_playlist, params.block_number, params.event_blockhash, params.txhash ) + deleted_playlist.is_delete = True + + params.playlists_to_save[params.entity_id].append(deleted_playlist) + + +def collect_entities_to_fetch( + update_task, + audius_data_txs, + entities_to_fetch: Dict[EntityType, Set[int]], + users_to_fetch: Set[int], +): + for tx_receipt in audius_data_txs: + for event_type in audius_data_event_types_arr: + audius_data_event_tx = get_audius_data_events_tx( + update_task, event_type, tx_receipt + ) + for event in audius_data_event_tx: + entity_id = helpers.get_tx_arg(event, "_entityId") + entity_type = helpers.get_tx_arg(event, "_entityType") + user_id = helpers.get_tx_arg(event, "_userId") + + entities_to_fetch[entity_type].add(entity_id) + users_to_fetch.add(user_id) - playlist_record = None - if playlist_exists: - playlist_record = ( - session.query(Playlist) - .filter(Playlist.playlist_id == playlist_id, Playlist.is_current == True) - .first() + +def fetch_existing_entities( + session: Session, entities_to_fetch: Dict[EntityType, Set[int]] +): + existing_playlist_id_to_playlist: Dict[int, Playlist] = {} + existing_playlists_query = ( + session.query(Playlist) + .filter( + Playlist.playlist_id.in_(entities_to_fetch[EntityType.PLAYLIST]), + Playlist.is_current == True, ) + .all() + ) + for existing_playlist in existing_playlists_query: + existing_playlist_id_to_playlist[ + existing_playlist.playlist_id + ] = existing_playlist + return existing_playlist_id_to_playlist + - # expunge the result from sqlalchemy so we can modify it without UPDATE statements being made - # https://stackoverflow.com/questions/28871406/how-to-clone-a-sqlalchemy-db-object-with-new-primary-key - session.expunge(playlist_record) - make_transient(playlist_record) - else: - playlist_record = Playlist( - playlist_id=playlist_id, is_current=True, is_delete=False +def fetch_users(session: Session, users_to_fetch: Set[int]): + existing_user_id_to_user: Dict[int, User] = {} + existing_users_query = ( + session.query(User) + .filter( + User.user_id.in_(users_to_fetch), + User.is_current == True, ) + .all() + ) + for user in existing_users_query: + existing_user_id_to_user[user.user_id] = user + return existing_user_id_to_user - # update these fields regardless of type - playlist_record.blocknumber = block_number - playlist_record.blockhash = event_blockhash - playlist_record.txhash = txhash - return playlist_record +def copy_record(old_playlist: Playlist, block_number, event_blockhash, txhash): + new_playlist = Playlist( + playlist_id=old_playlist.playlist_id, + playlist_owner_id=old_playlist.playlist_owner_id, + is_album=old_playlist.is_album, + description=old_playlist.description, + playlist_image_multihash=old_playlist.playlist_image_multihash, + playlist_image_sizes_multihash=old_playlist.playlist_image_sizes_multihash, + playlist_name=old_playlist.playlist_name, + is_private=old_playlist.is_private, + playlist_contents=old_playlist.playlist_contents, + created_at=old_playlist.created_at, + updated_at=old_playlist.updated_at, + blocknumber=block_number, + blockhash=event_blockhash, + txhash=txhash, + is_current=False, + is_delete=old_playlist.is_delete, + ) + return new_playlist + + +def get_audius_data_events_tx(update_task, event_type, tx_receipt): + return getattr( + update_task.audius_data_contract.events, event_type + )().processReceipt(tx_receipt) # Create playlist specific def parse_playlist_create_data_event( - update_task, - entry, - playlist_owner_id, - playlist_record, + playlist_record: Playlist, playlist_metadata, - block_timestamp, - session, + block_integer_time, + block_datetime, ): - block_datetime = datetime.utcfromtimestamp(block_timestamp) - block_integer_time = int(block_timestamp) - playlist_record.playlist_owner_id = playlist_owner_id playlist_record.is_album = ( playlist_metadata["is_album"] if "is_album" in playlist_metadata else False ) diff --git a/discovery-provider/src/tasks/index.py b/discovery-provider/src/tasks/index.py index cf3348243ec..dfc322234a1 100644 --- a/discovery-provider/src/tasks/index.py +++ b/discovery-provider/src/tasks/index.py @@ -328,7 +328,8 @@ def fetch_cid_metadata(db, user_factory_txs, track_factory_txs, audius_data_txs) user_id = event_args._userId entity_type = event_args._entityType cid = event_args._metadata - # TODO - skip if not a multihash + if not cid: + continue logger.info( f"index.py | newcontract {txhash}, {event_args}, {entity_type}, {cid}" ) diff --git a/libs/eth-contracts/ABIs/Wormhole.json b/libs/eth-contracts/ABIs/Wormhole.json index 830697357d2..7dc4a28bee0 100644 --- a/libs/eth-contracts/ABIs/Wormhole.json +++ b/libs/eth-contracts/ABIs/Wormhole.json @@ -1,12 +1,71 @@ { "contractName": "Wormhole", "abi": [ + { + "constant": true, + "inputs": [], + "name": "DOMAIN_SEPARATOR", + "outputs": [ + { + "internalType": "bytes32", + "name": "", + "type": "bytes32" + } + ], + "payable": false, + "stateMutability": "view", + "type": "function" + }, + { + "constant": true, + "inputs": [], + "name": "LOCK_ASSETS_TYPEHASH", + "outputs": [ + { + "internalType": "bytes32", + "name": "", + "type": "bytes32" + } + ], + "payable": false, + "stateMutability": "view", + "type": "function" + }, + { + "constant": false, + "inputs": [ + { + "internalType": "address", + "name": "_tokenAddress", + "type": "address" + }, + { + "internalType": "address", + "name": "_wormholeAddress", + "type": "address" + } + ], + "name": "initialize", + "outputs": [], + "payable": false, + "stateMutability": "nonpayable", + "type": "function" + }, + { + "constant": false, + "inputs": [], + "name": "initialize", + "outputs": [], + "payable": false, + "stateMutability": "nonpayable", + "type": "function" + }, { "constant": false, "inputs": [ { "internalType": "address", - "name": "token", + "name": "from", "type": "address" }, { @@ -14,32 +73,83 @@ "name": "amount", "type": "uint256" }, - { - "internalType": "uint16", - "name": "recipientChain", - "type": "uint16" - }, { "internalType": "bytes32", "name": "recipient", "type": "bytes32" }, + { + "internalType": "uint8", + "name": "targetChain", + "type": "uint8" + }, + { + "internalType": "bool", + "name": "refundDust", + "type": "bool" + }, { "internalType": "uint256", - "name": "arbiterFee", + "name": "deadline", "type": "uint256" }, { - "internalType": "uint32", - "name": "nonce", - "type": "uint32" + "internalType": "uint8", + "name": "v", + "type": "uint8" + }, + { + "internalType": "bytes32", + "name": "r", + "type": "bytes32" + }, + { + "internalType": "bytes32", + "name": "s", + "type": "bytes32" } ], - "name": "transferTokens", + "name": "lockAssets", "outputs": [], "payable": false, "stateMutability": "nonpayable", "type": "function" + }, + { + "constant": true, + "inputs": [ + { + "internalType": "address", + "name": "", + "type": "address" + } + ], + "name": "nonces", + "outputs": [ + { + "internalType": "uint32", + "name": "", + "type": "uint32" + } + ], + "payable": false, + "stateMutability": "view", + "type": "function" + }, + { + "constant": true, + "inputs": [], + "name": "token", + "outputs": [ + { + "internalType": "address", + "name": "", + "type": "address" + } + ], + "payable": false, + "stateMutability": "view", + "type": "function" } ] } \ No newline at end of file diff --git a/libs/src/api/entityManager.ts b/libs/src/api/entityManager.ts index 612a71ae834..e4be874474c 100644 --- a/libs/src/api/entityManager.ts +++ b/libs/src/api/entityManager.ts @@ -167,15 +167,14 @@ export class EntityManager extends Base { */ async deletePlaylist({ playlistId, - userId + logger = console }: { playlistId: number - userId: number logger: any - }): Promise { - const responseValues: PlaylistOperationResponse = - this.getDefaultPlaylistReponseValues() + }): Promise<{ blockHash: any; blockNumber: any }> { + const userId: number = parseInt(this.userStateManager.getCurrentUserId()) try { + const resp = await this.manageEntity({ userId, entityType: EntityType.PLAYLIST, From 3d8fb12179c7a52dcf5a119acc84dd2e27b4f8c1 Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Thu, 21 Jul 2022 15:19:14 +0000 Subject: [PATCH 082/101] Fixing lint issues --- libs/src/api/entityManager.ts | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/libs/src/api/entityManager.ts b/libs/src/api/entityManager.ts index e4be874474c..7e7815439be 100644 --- a/libs/src/api/entityManager.ts +++ b/libs/src/api/entityManager.ts @@ -1,4 +1,4 @@ -import { Base, BaseConstructorArgs, Services } from './base' +import { Base, Services } from './base' import type { PlaylistMetadata } from '../services/creatorNode' import type { Nullable } from '../utils' @@ -42,10 +42,6 @@ const MAX_PLAYLIST_ID = 2147483647 Handles metadata + file upload etc. for entities such as Playlist/Track/User */ export class EntityManager extends Base { - constructor(...args: BaseConstructorArgs) { - super(...args) - } - /** * Generate random integer between two known values */ @@ -174,7 +170,6 @@ export class EntityManager extends Base { }): Promise<{ blockHash: any; blockNumber: any }> { const userId: number = parseInt(this.userStateManager.getCurrentUserId()) try { - const resp = await this.manageEntity({ userId, entityType: EntityType.PLAYLIST, @@ -242,6 +237,7 @@ export class EntityManager extends Base { this.REQUIRES(Services.CREATOR_NODE) let dirCID if (coverArt) { + // @ts-expect-error const updatedPlaylistImage = await this.creatorNode.uploadImage( coverArt, true // square @@ -254,6 +250,7 @@ export class EntityManager extends Base { let playlistContents = trackIds if (playlist.playlist_contents) { + // CONVERT TO FOREACH playlistContents = playlist.playlist_contents.track_ids.map( (x: { [x: string]: any }) => x['track'] ) @@ -262,12 +259,12 @@ export class EntityManager extends Base { const metadata: PlaylistMetadata = { playlist_id: playlistId, playlist_contents: playlistContents, - playlist_name: playlistName || playlist.playlist_name, + playlist_name: playlistName ?? playlist.playlist_name, playlist_image_sizes_multihash: dirCID || playlist.playlist_image_sizes_multihash, - description: description || playlist.description, - is_album: isAlbum || playlist.is_album, - is_private: isPrivate || playlist.is_private + description: description ?? playlist.description, + is_album: isAlbum ?? playlist.is_album, + is_private: isPrivate ?? playlist.is_private } const { metadataMultihash } = await this.creatorNode.uploadPlaylistMetadata(metadata) From f797ab783132840f46f10672152b19381a64b86b Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Thu, 21 Jul 2022 17:24:05 +0000 Subject: [PATCH 083/101] Fix missing values --- libs/src/api/entityManager.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libs/src/api/entityManager.ts b/libs/src/api/entityManager.ts index 7e7815439be..e258d3546b0 100644 --- a/libs/src/api/entityManager.ts +++ b/libs/src/api/entityManager.ts @@ -169,6 +169,8 @@ export class EntityManager extends Base { logger: any }): Promise<{ blockHash: any; blockNumber: any }> { const userId: number = parseInt(this.userStateManager.getCurrentUserId()) + const responseValues: PlaylistOperationResponse = + this.getDefaultPlaylistReponseValues() try { const resp = await this.manageEntity({ userId, From 9af986c5ca84d81a57ae57992049988f8a1f8593 Mon Sep 17 00:00:00 2001 From: Hareesh Nagaraj Date: Fri, 22 Jul 2022 14:51:53 +0000 Subject: [PATCH 084/101] Nullable and type added to CreatorNote.ts --- libs/src/api/entityManager.ts | 65 +++++++------------- libs/src/services/creatorNode/CreatorNode.ts | 3 +- 2 files changed, 25 insertions(+), 43 deletions(-) diff --git a/libs/src/api/entityManager.ts b/libs/src/api/entityManager.ts index e258d3546b0..01298eb2707 100644 --- a/libs/src/api/entityManager.ts +++ b/libs/src/api/entityManager.ts @@ -111,12 +111,8 @@ export class EntityManager extends Base { const currentUserId: string | null = this.userStateManager.getCurrentUserId() if (!currentUserId) { - return { - blockHash: null, - blockNumber: null, - playlistId: null, - error: 'Missing current user ID' - } + responseValues.error = 'Missing current user ID' + return responseValues } const userId: number = parseInt(currentUserId) const createAction = Action.CREATE @@ -168,9 +164,15 @@ export class EntityManager extends Base { playlistId: number logger: any }): Promise<{ blockHash: any; blockNumber: any }> { - const userId: number = parseInt(this.userStateManager.getCurrentUserId()) const responseValues: PlaylistOperationResponse = this.getDefaultPlaylistReponseValues() + const currentUserId: string | null = + this.userStateManager.getCurrentUserId() + if (!currentUserId) { + responseValues.error = 'Missing current user ID' + return responseValues + } + const userId: number = parseInt(currentUserId) try { const resp = await this.manageEntity({ userId, @@ -213,25 +215,18 @@ export class EntityManager extends Base { coverArt: Nullable logger: Console }): Promise { - let error = null + const responseValues: PlaylistOperationResponse = + this.getDefaultPlaylistReponseValues() try { const currentUserId: string | null = this.userStateManager.getCurrentUserId() if (!playlistId || playlistId === undefined) { - return { - blockHash: null, - blockNumber: null, - playlistId, - error: 'Missing current playlistId' - } + responseValues.error = 'Missing current playlistId' + return responseValues } if (!currentUserId) { - return { - blockHash: null, - blockNumber: null, - playlistId, - error: 'Missing current user ID' - } + responseValues.error = 'Missing current user ID' + return responseValues } const userId: number = parseInt(currentUserId) const updateAction = Action.UPDATE @@ -250,17 +245,9 @@ export class EntityManager extends Base { await this.discoveryProvider.getPlaylists(1, 0, [playlistId], userId) )[0] - let playlistContents = trackIds - if (playlist.playlist_contents) { - // CONVERT TO FOREACH - playlistContents = playlist.playlist_contents.track_ids.map( - (x: { [x: string]: any }) => x['track'] - ) - } - const metadata: PlaylistMetadata = { playlist_id: playlistId, - playlist_contents: playlistContents, + playlist_contents: trackIds, playlist_name: playlistName ?? playlist.playlist_name, playlist_image_sizes_multihash: dirCID || playlist.playlist_image_sizes_multihash, @@ -279,20 +266,14 @@ export class EntityManager extends Base { metadataMultihash }) const txReceipt = resp.txReceipt - return { - blockHash: txReceipt.blockHash, - blockNumber: txReceipt.blockNumber, - playlistId, - error - } + responseValues.blockHash = txReceipt.blockHash + responseValues.blockNumber = txReceipt.blockNumber + responseValues.playlistId = playlistId + return responseValues } catch (e) { - error = (e as Error).message - return { - blockHash: null, - blockNumber: null, - playlistId: null, - error - } + const error = (e as Error).message + responseValues.error = error + return responseValues } } diff --git a/libs/src/services/creatorNode/CreatorNode.ts b/libs/src/services/creatorNode/CreatorNode.ts index aa799297ede..1f55a29e0a3 100644 --- a/libs/src/services/creatorNode/CreatorNode.ts +++ b/libs/src/services/creatorNode/CreatorNode.ts @@ -8,6 +8,7 @@ import { playlistSchemaType, Schemas } from '../schemaValidator/SchemaValidator' +import type { Nullable } from '../../utils' import type { Web3Manager } from '../web3Manager' import type { CurrentUser, UserStateManager } from '../../userStateManager' @@ -27,7 +28,7 @@ type Metadata = { } export type PlaylistMetadata = { - playlist_contents: unknown + playlist_contents: Nullable playlist_id: number playlist_name: string playlist_image_sizes_multihash: string From 3e1cd777b3478794510e2dee19048eef5bb9359c Mon Sep 17 00:00:00 2001 From: Isaac Date: Mon, 1 Aug 2022 16:23:04 +0000 Subject: [PATCH 085/101] Add entity manager playlist tests --- .../tasks/test_entity_manager.py | 322 ++++++++++++++++++ discovery-provider/src/tasks/audius_data.py | 144 ++++---- 2 files changed, 404 insertions(+), 62 deletions(-) create mode 100644 discovery-provider/integration_tests/tasks/test_entity_manager.py diff --git a/discovery-provider/integration_tests/tasks/test_entity_manager.py b/discovery-provider/integration_tests/tasks/test_entity_manager.py new file mode 100644 index 00000000000..5adae51d32e --- /dev/null +++ b/discovery-provider/integration_tests/tasks/test_entity_manager.py @@ -0,0 +1,322 @@ +from typing import List + +from integration_tests.challenges.index_helpers import UpdateTask +from integration_tests.utils import populate_mock_db +from src.models.playlists.playlist import Playlist +from src.tasks.audius_data import audius_data_state_update +from src.utils.db_session import get_db +from web3 import Web3 +from web3.datastructures import AttributeDict + + +def test_index_valid_playlists(app, mocker): + "Tests valid batch of playlists create/update/delete actions" + + # setup db and mocked txs + with app.app_context(): + db = get_db() + web3 = Web3() + update_task = UpdateTask(None, web3, None) + + tx_receipts = { + "CreatePlaylist1Tx": [ + { + "args": AttributeDict( + { + "_entityId": 1, + "_entityType": "Playlist", + "_userId": 1, + "_action": "Create", + "_metadata": "QmCreatePlaylist1", + "_signer": "user1wallet", + } + ) + }, + ], + "UpdatePlaylist1Tx": [ + { + "args": AttributeDict( + { + "_entityId": 1, + "_entityType": "Playlist", + "_userId": 1, + "_action": "Update", + "_metadata": "QmUpdatePlaylist1", + "_signer": "user1wallet", + } + ) + }, + ], + "DeletePlaylist1Tx": [ + { + "args": AttributeDict( + { + "_entityId": 1, + "_entityType": "Playlist", + "_userId": 1, + "_action": "Delete", + "_metadata": "", + "_signer": "user1wallet", + } + ) + }, + ], + "CreatePlaylist2Tx": [ + { + "args": AttributeDict( + { + "_entityId": 2, + "_entityType": "Playlist", + "_userId": 1, + "_action": "Create", + "_metadata": "QmCreatePlaylist2", + "_signer": "user1wallet", + } + ) + }, + ], + } + + audius_data_txs = [ + AttributeDict({"transactionHash": update_task.web3.toBytes(text=tx_receipt)}) + for tx_receipt in tx_receipts + ] + + def get_events_side_effect(_, tx_receipt): + return tx_receipts[tx_receipt.transactionHash.decode("utf-8")] + + mocker.patch( + "src.tasks.audius_data.get_audius_data_events_tx", + side_effect=get_events_side_effect, + autospec=True, + ) + test_metadata = { + "QmCreatePlaylist1": { + "playlist_contents": {"track_ids": []}, + "description": "", + "playlist_image_sizes_multihash": "", + "playlist_name": "playlist 1", + }, + "QmCreatePlaylist2": { + "playlist_contents": {"track_ids": []}, + "description": "test description", + "playlist_image_sizes_multihash": "", + "playlist_name": "playlist 2", + }, + "QmUpdatePlaylist1": { + "playlist_contents": {"track_ids": []}, + "description": "", + "playlist_image_sizes_multihash": "", + "playlist_name": "playlist 1 updated", + }, + } + + entities = { + "users": [ + {"user_id": 1, "handle": "user-1", "wallet": "user1wallet"}, + ] + } + populate_mock_db(db, entities) + + with db.scoped_session() as session: + # index transactions + audius_data_state_update( + None, + update_task, + session, + audius_data_txs, + block_number=0, + block_timestamp=1585336422, + block_hash=0, + ipfs_metadata=test_metadata, + ) + + # validate db records + all_playlists: List[Playlist] = session.query(Playlist).all() + assert len(all_playlists) == 4 + + playlist_1: Playlist = ( + session.query(Playlist) + .filter(Playlist.is_current == True, Playlist.playlist_id == 1) + .first() + ) + assert playlist_1.playlist_name == "playlist 1 updated" + assert playlist_1.is_delete == True + + playlist_2: Playlist = ( + session.query(Playlist) + .filter(Playlist.is_current == True, Playlist.playlist_id == 2) + .first() + ) + assert playlist_2.playlist_name == "playlist 2" + assert playlist_2.is_delete == False + + +def test_index_invalid_playlists(app, mocker): + "Tests invalid batch of playlists create/update/delete actions" + + # setup db and mocked txs + with app.app_context(): + db = get_db() + web3 = Web3() + update_task = UpdateTask(None, web3, None) + + tx_receipts = { + # invalid create + "CreatePlaylistUserDoesNotExist": [ + { + "args": AttributeDict( + { + "_entityId": 2, + "_entityType": "Playlist", + "_userId": 2, + "_action": "Create", + "_metadata": "", + "_signer": "user1wallet", + } + ) + }, + ], + "CreatePlaylistUserDoesNotMatchSigner": [ + { + "args": AttributeDict( + { + "_entityId": 2, + "_entityType": "Playlist", + "_userId": 1, + "_action": "Create", + "_metadata": "", + "_signer": "InvalidWallet", + } + ) + }, + ], + "CreatePlaylistAlreadyExists": [ + { + "args": AttributeDict( + { + "_entityId": 1, + "_entityType": "Playlist", + "_userId": 1, + "_action": "Create", + "_metadata": "", + "_signer": "user1wallet", + } + ) + }, + ], + # invalid updates + "UpdatePlaylistInvalidOwnership": [ + { + "args": AttributeDict( + { + "_entityId": 1, + "_entityType": "Playlist", + "_userId": 1, + "_action": "Update", + "_metadata": "", + "_signer": "InvalidWallet", + } + ) + }, + ], + "UpdatePlaylistDoesNotMatchPlaylistOwner": [ + { + "args": AttributeDict( + { + "_entityId": 1, + "_entityType": "Playlist", + "_userId": 2, + "_action": "Update", + "_metadata": "", + "_signer": "User2Wallet", + } + ) + }, + ], + # invalid deletes + "DeletePlaylistInvalidOwnership": [ + { + "args": AttributeDict( + { + "_entityId": 1, + "_entityType": "Playlist", + "_userId": 1, + "_action": "Delete", + "_metadata": "", + "_signer": "InvalidWallet", + } + ) + }, + ], + "DeletePlaylistDoesNotExist": [ + { + "args": AttributeDict( + { + "_entityId": 2, + "_entityType": "Playlist", + "_userId": 1, + "_action": "Update", + "_metadata": "", + "_signer": "user1wallet", + } + ) + }, + ], + "DeletePlaylistDoesNotMatchPlaylistOwner": [ + { + "args": AttributeDict( + { + "_entityId": 1, + "_entityType": "Playlist", + "_userId": 2, + "_action": "Update", + "_metadata": "", + "_signer": "User2Wallet", + } + ) + }, + ], + } + + audius_data_txs = [ + AttributeDict({"transactionHash": update_task.web3.toBytes(text=tx_receipt)}) + for tx_receipt in tx_receipts + ] + + def get_events_side_effect(_, tx_receipt): + return tx_receipts[tx_receipt.transactionHash.decode("utf-8")] + + mocker.patch( + "src.tasks.audius_data.get_audius_data_events_tx", + side_effect=get_events_side_effect, + autospec=True, + ) + + entities = { + "users": [ + {"user_id": 1, "handle": "user-1", "wallet": "user1wallet"}, + {"user_id": 2, "handle": "user-1", "wallet": "User2Wallet"}, + ], + "playlists": [ + {"playlist_id": 1, "playlist_owner_id": 1}, + ], + } + populate_mock_db(db, entities) + + with db.scoped_session() as session: + # index transactions + audius_data_state_update( + None, + update_task, + session, + audius_data_txs, + block_number=0, + block_timestamp=1585336422, + block_hash=0, + ipfs_metadata={}, + ) + + # validate db records + all_playlists: List[Playlist] = session.query(Playlist).all() + assert len(all_playlists) == 1 # no new playlists indexed diff --git a/discovery-provider/src/tasks/audius_data.py b/discovery-provider/src/tasks/audius_data.py index 2a19f57d079..fbb9c8056a2 100644 --- a/discovery-provider/src/tasks/audius_data.py +++ b/discovery-provider/src/tasks/audius_data.py @@ -4,13 +4,11 @@ from enum import Enum from typing import Any, Dict, List, Set, Tuple -from sqlalchemy.orm.session import Session, make_transient +from sqlalchemy.orm.session import Session from src.database_task import DatabaseTask from src.models.playlists.playlist import Playlist from src.models.users.user import User -from src.tasks.playlists import invalidate_old_playlist from src.utils import helpers -from src.utils.user_event_constants import audius_data_event_types_arr from web3.datastructures import AttributeDict logger = logging.getLogger(__name__) @@ -32,6 +30,9 @@ def __str__(self) -> str: return str.__str__(self) +MANAGE_ENTITY_EVENT_TYPE = "ManageEntity" + + def audius_data_state_update( self, update_task: DatabaseTask, @@ -40,7 +41,7 @@ def audius_data_state_update( block_number, block_timestamp, block_hash, - ipfs_metadata + ipfs_metadata, ) -> Tuple[int, Dict[str, Set[(int)]]]: try: num_total_changes = 0 @@ -71,38 +72,35 @@ def audius_data_state_update( # process in tx order and populate playlists_to_save for tx_receipt in audius_data_txs: txhash = update_task.web3.toHex(tx_receipt.transactionHash) - for event_type in audius_data_event_types_arr: - audius_data_event_tx = get_audius_data_events_tx( - update_task, event_type, tx_receipt + audius_data_event_tx = get_audius_data_events_tx(update_task, tx_receipt) + for event in audius_data_event_tx: + params = ManagePlaylistParameters( + event, + playlists_to_save, # actions below populate these records + existing_playlist_id_to_playlist, + existing_user_id_to_user, + ipfs_metadata, + block_timestamp, + block_number, + event_blockhash, + txhash, ) - for event in audius_data_event_tx: - params = ManagePlaylistParameters( - event, - playlists_to_save, # actions below populate these records - existing_playlist_id_to_playlist, - existing_user_id_to_user, - ipfs_metadata, - block_timestamp, - block_number, - event_blockhash, - txhash, - ) - if ( - params.action == Action.CREATE - and params.entity_type == EntityType.PLAYLIST - ): - create_playlist(params) - elif ( - params.action == Action.UPDATE - and params.entity_type == EntityType.PLAYLIST - ): - update_playlist(params) - - elif ( - params.action == Action.DELETE - and params.entity_type == EntityType.PLAYLIST - ): - delete_playlist(params) + if ( + params.action == Action.CREATE + and params.entity_type == EntityType.PLAYLIST + ): + create_playlist(params) + elif ( + params.action == Action.UPDATE + and params.entity_type == EntityType.PLAYLIST + ): + update_playlist(params) + + elif ( + params.action == Action.DELETE + and params.entity_type == EntityType.PLAYLIST + ): + delete_playlist(params) # compile records_to_save records_to_save = [] @@ -117,6 +115,7 @@ def audius_data_state_update( except Exception as e: logger.error(f"Exception occurred {e}", exc_info=True) + raise e return num_total_changes, changed_entity_ids @@ -152,13 +151,40 @@ def __init__( self.playlists_to_save = playlists_to_save +def is_valid_playlist_owner(params: ManagePlaylistParameters): + if params.user_id not in params.existing_user_id_to_user: + # user does not exist + return False + + wallet = params.existing_user_id_to_user[params.user_id].wallet + if wallet and wallet.lower() != params.signer.lower(): + # user does not match signer + return False + if params.action == Action.CREATE: + if params.entity_id in params.existing_playlist_id_to_playlist: + # playlist already exists + return False + else: + if params.entity_id not in params.existing_playlist_id_to_playlist: + # playlist does not exist + return False + existing_playlist: Playlist = params.existing_playlist_id_to_playlist[ + params.entity_id + ] + if existing_playlist.playlist_owner_id != params.user_id: + # existing playlist does not match user + return False + + return True + + def create_playlist(params: ManagePlaylistParameters): - metadata = params.ipfs_metadata[params.metadata_cid] - # check if playlist already exists - if params.entity_id in params.existing_playlist_id_to_playlist: + if not is_valid_playlist_owner(params): return - track_ids = metadata.get("playlist_contents", []) + metadata = params.ipfs_metadata[params.metadata_cid] + + track_ids = metadata["playlist_contents"] playlist_contents = [] for track_id in track_ids: playlist_contents.append({"track": track_id, "time": params.block_integer_time}) @@ -182,16 +208,15 @@ def create_playlist(params: ManagePlaylistParameters): ) params.playlists_to_save[params.entity_id].append(create_playlist_record) + params.existing_playlist_id_to_playlist[params.entity_id] = create_playlist_record def update_playlist(params: ManagePlaylistParameters): - metadata = params.ipfs_metadata[params.metadata_cid] - # check user owns playlist - if ( - params.signer.lower() - != params.existing_user_id_to_user[params.user_id].wallet.lower() - ): + if not is_valid_playlist_owner(params): return + # TODO ignore updates on deleted playlists? + + metadata = params.ipfs_metadata[params.metadata_cid] existing_playlist = params.existing_playlist_id_to_playlist[params.entity_id] existing_playlist.is_current = False # invalidate if ( @@ -212,12 +237,10 @@ def update_playlist(params: ManagePlaylistParameters): def delete_playlist(params: ManagePlaylistParameters): - # check user owns playlist - if ( - params.signer.lower() - != params.existing_user_id_to_user[params.user_id].wallet.lower() - ): + + if not is_valid_playlist_owner(params): return + existing_playlist = params.existing_playlist_id_to_playlist[params.entity_id] existing_playlist.is_current = False # invalidate old playlist if params.entity_id in params.playlists_to_save: @@ -239,17 +262,14 @@ def collect_entities_to_fetch( users_to_fetch: Set[int], ): for tx_receipt in audius_data_txs: - for event_type in audius_data_event_types_arr: - audius_data_event_tx = get_audius_data_events_tx( - update_task, event_type, tx_receipt - ) - for event in audius_data_event_tx: - entity_id = helpers.get_tx_arg(event, "_entityId") - entity_type = helpers.get_tx_arg(event, "_entityType") - user_id = helpers.get_tx_arg(event, "_userId") + audius_data_event_tx = get_audius_data_events_tx(update_task, tx_receipt) + for event in audius_data_event_tx: + entity_id = helpers.get_tx_arg(event, "_entityId") + entity_type = helpers.get_tx_arg(event, "_entityType") + user_id = helpers.get_tx_arg(event, "_userId") - entities_to_fetch[entity_type].add(entity_id) - users_to_fetch.add(user_id) + entities_to_fetch[entity_type].add(entity_id) + users_to_fetch.add(user_id) def fetch_existing_entities( @@ -308,9 +328,9 @@ def copy_record(old_playlist: Playlist, block_number, event_blockhash, txhash): return new_playlist -def get_audius_data_events_tx(update_task, event_type, tx_receipt): +def get_audius_data_events_tx(update_task, tx_receipt): return getattr( - update_task.audius_data_contract.events, event_type + update_task.audius_data_contract.events, MANAGE_ENTITY_EVENT_TYPE )().processReceipt(tx_receipt) From 796aed310445a4cf1715e8c2295fadcac1df21f5 Mon Sep 17 00:00:00 2001 From: Isaac Date: Mon, 1 Aug 2022 16:26:02 +0000 Subject: [PATCH 086/101] add migration --- .../f1e86fba0357_add_playlist_metadata.py | 32 +++++++++++++++++++ .../src/models/playlists/playlist.py | 1 + 2 files changed, 33 insertions(+) create mode 100644 discovery-provider/alembic/versions/f1e86fba0357_add_playlist_metadata.py diff --git a/discovery-provider/alembic/versions/f1e86fba0357_add_playlist_metadata.py b/discovery-provider/alembic/versions/f1e86fba0357_add_playlist_metadata.py new file mode 100644 index 00000000000..83fc886234b --- /dev/null +++ b/discovery-provider/alembic/versions/f1e86fba0357_add_playlist_metadata.py @@ -0,0 +1,32 @@ +"""add playlist metadata + +Revision ID: f1e86fba0357 +Revises: ab56e2d974a6 +Create Date: 2022-07-29 20:31:41.996125 + +""" +from alembic import op + +# revision identifiers, used by Alembic. +revision = "f1e86fba0357" +down_revision = "ab56e2d974a6" +branch_labels = None +depends_on = None + + +def upgrade(): + connection = op.get_bind() + connection.execute( + """ + ALTER TABLE playlists ADD COLUMN IF NOT EXISTS metadata_multihash varchar; + """ + ) + + +def downgrade(): + connection = op.get_bind() + connection.execute( + """ + ALTER TABLE playlists DROP COLUMN IF EXISTS metadata_multihash; + """ + ) diff --git a/discovery-provider/src/models/playlists/playlist.py b/discovery-provider/src/models/playlists/playlist.py index 9d06394913f..e105ac411fb 100644 --- a/discovery-provider/src/models/playlists/playlist.py +++ b/discovery-provider/src/models/playlists/playlist.py @@ -33,6 +33,7 @@ class Playlist(Base, RepresentableMixin): ) last_added_to = Column(DateTime) slot = Column(Integer) + metadata_multihash = Column(String) block = relationship( # type: ignore "Block", primaryjoin="Playlist.blockhash == Block.blockhash" From 3d2f60b7ecdb8fe01d22c36ce9d32e0a253e733b Mon Sep 17 00:00:00 2001 From: Isaac Date: Mon, 1 Aug 2022 16:33:28 +0000 Subject: [PATCH 087/101] Add offset logic --- discovery-provider/src/tasks/audius_data.py | 19 +++++++++++-------- discovery-provider/src/tasks/playlists.py | 8 +++++++- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/discovery-provider/src/tasks/audius_data.py b/discovery-provider/src/tasks/audius_data.py index fbb9c8056a2..1f1470f8788 100644 --- a/discovery-provider/src/tasks/audius_data.py +++ b/discovery-provider/src/tasks/audius_data.py @@ -13,6 +13,8 @@ logger = logging.getLogger(__name__) +PLAYLIST_ID_OFFSET = 400000 + class Action(str, Enum): CREATE = "Create" @@ -151,7 +153,7 @@ def __init__( self.playlists_to_save = playlists_to_save -def is_valid_playlist_owner(params: ManagePlaylistParameters): +def is_valid_playlist_tx(params: ManagePlaylistParameters): if params.user_id not in params.existing_user_id_to_user: # user does not exist return False @@ -164,7 +166,10 @@ def is_valid_playlist_owner(params: ManagePlaylistParameters): if params.entity_id in params.existing_playlist_id_to_playlist: # playlist already exists return False + if params.entity_id < PLAYLIST_ID_OFFSET: + return False else: + # update / delete specific validations if params.entity_id not in params.existing_playlist_id_to_playlist: # playlist does not exist return False @@ -179,7 +184,7 @@ def is_valid_playlist_owner(params: ManagePlaylistParameters): def create_playlist(params: ManagePlaylistParameters): - if not is_valid_playlist_owner(params): + if not is_valid_playlist_tx(params): return metadata = params.ipfs_metadata[params.metadata_cid] @@ -212,7 +217,7 @@ def create_playlist(params: ManagePlaylistParameters): def update_playlist(params: ManagePlaylistParameters): - if not is_valid_playlist_owner(params): + if not is_valid_playlist_tx(params): return # TODO ignore updates on deleted playlists? @@ -227,7 +232,7 @@ def update_playlist(params: ManagePlaylistParameters): updated_playlist = copy_record( existing_playlist, params.block_number, params.event_blockhash, params.txhash ) - parse_playlist_create_data_event( + parse_playlist_data_event( updated_playlist, metadata, params.block_integer_time, @@ -237,8 +242,7 @@ def update_playlist(params: ManagePlaylistParameters): def delete_playlist(params: ManagePlaylistParameters): - - if not is_valid_playlist_owner(params): + if not is_valid_playlist_tx(params): return existing_playlist = params.existing_playlist_id_to_playlist[params.entity_id] @@ -334,8 +338,7 @@ def get_audius_data_events_tx(update_task, tx_receipt): )().processReceipt(tx_receipt) -# Create playlist specific -def parse_playlist_create_data_event( +def parse_playlist_data_event( playlist_record: Playlist, playlist_metadata, block_integer_time, diff --git a/discovery-provider/src/tasks/playlists.py b/discovery-provider/src/tasks/playlists.py index 52544cd021a..5ece9c16c69 100644 --- a/discovery-provider/src/tasks/playlists.py +++ b/discovery-provider/src/tasks/playlists.py @@ -7,6 +7,7 @@ from src.database_task import DatabaseTask from src.models.playlists.playlist import Playlist from src.queries.skipped_transactions import add_node_level_skipped_transaction +from src.tasks.audius_data import PLAYLIST_ID_OFFSET from src.utils import helpers from src.utils.indexing_errors import EntityMissingRequiredFieldError, IndexingError from src.utils.model_nullable_validator import all_required_fields_present @@ -63,7 +64,7 @@ def playlist_state_update( ) # parse playlist event to add metadata to record - playlist_record = parse_playlist_event( + playlist_record: Playlist = parse_playlist_event( self, update_task, entry, @@ -72,6 +73,11 @@ def playlist_state_update( block_timestamp, session, ) + if playlist_record.playlist_id >= PLAYLIST_ID_OFFSET: + logger.info( + f"index.py | playlists.py | Playlist {playlist_record.playlist_id} is above the playlist ID offset {PLAYLIST_ID_OFFSET}. Skipping transaction." + ) + continue # process playlist record if playlist_record is not None: From 1d5dfceed3012c46b7dded514067bdd7e61b95b2 Mon Sep 17 00:00:00 2001 From: Isaac Date: Mon, 1 Aug 2022 18:36:07 +0000 Subject: [PATCH 088/101] rename entity manager --- .../tasks/test_entity_manager.py | 75 ++++++++++++------- discovery-provider/src/app.py | 16 ++-- .../{audius_data.py => entity_manager.py} | 30 ++++---- discovery-provider/src/tasks/index.py | 39 +++++----- discovery-provider/src/tasks/playlists.py | 2 +- discovery-provider/src/utils/constants.py | 4 +- .../src/utils/user_event_constants.py | 4 +- 7 files changed, 97 insertions(+), 73 deletions(-) rename discovery-provider/src/tasks/{audius_data.py => entity_manager.py} (94%) diff --git a/discovery-provider/integration_tests/tasks/test_entity_manager.py b/discovery-provider/integration_tests/tasks/test_entity_manager.py index 5adae51d32e..b0c2351e98e 100644 --- a/discovery-provider/integration_tests/tasks/test_entity_manager.py +++ b/discovery-provider/integration_tests/tasks/test_entity_manager.py @@ -3,7 +3,7 @@ from integration_tests.challenges.index_helpers import UpdateTask from integration_tests.utils import populate_mock_db from src.models.playlists.playlist import Playlist -from src.tasks.audius_data import audius_data_state_update +from src.tasks.entity_manager import PLAYLIST_ID_OFFSET, entity_manager_update from src.utils.db_session import get_db from web3 import Web3 from web3.datastructures import AttributeDict @@ -23,7 +23,7 @@ def test_index_valid_playlists(app, mocker): { "args": AttributeDict( { - "_entityId": 1, + "_entityId": PLAYLIST_ID_OFFSET, "_entityType": "Playlist", "_userId": 1, "_action": "Create", @@ -37,7 +37,7 @@ def test_index_valid_playlists(app, mocker): { "args": AttributeDict( { - "_entityId": 1, + "_entityId": PLAYLIST_ID_OFFSET, "_entityType": "Playlist", "_userId": 1, "_action": "Update", @@ -51,7 +51,7 @@ def test_index_valid_playlists(app, mocker): { "args": AttributeDict( { - "_entityId": 1, + "_entityId": PLAYLIST_ID_OFFSET, "_entityType": "Playlist", "_userId": 1, "_action": "Delete", @@ -65,7 +65,7 @@ def test_index_valid_playlists(app, mocker): { "args": AttributeDict( { - "_entityId": 2, + "_entityId": PLAYLIST_ID_OFFSET + 1, "_entityType": "Playlist", "_userId": 1, "_action": "Create", @@ -77,7 +77,7 @@ def test_index_valid_playlists(app, mocker): ], } - audius_data_txs = [ + entity_manager_txs = [ AttributeDict({"transactionHash": update_task.web3.toBytes(text=tx_receipt)}) for tx_receipt in tx_receipts ] @@ -86,7 +86,7 @@ def get_events_side_effect(_, tx_receipt): return tx_receipts[tx_receipt.transactionHash.decode("utf-8")] mocker.patch( - "src.tasks.audius_data.get_audius_data_events_tx", + "src.tasks.entity_manager.get_entity_manager_events_tx", side_effect=get_events_side_effect, autospec=True, ) @@ -120,11 +120,11 @@ def get_events_side_effect(_, tx_receipt): with db.scoped_session() as session: # index transactions - audius_data_state_update( + entity_manager_update( None, update_task, session, - audius_data_txs, + entity_manager_txs, block_number=0, block_timestamp=1585336422, block_hash=0, @@ -137,7 +137,9 @@ def get_events_side_effect(_, tx_receipt): playlist_1: Playlist = ( session.query(Playlist) - .filter(Playlist.is_current == True, Playlist.playlist_id == 1) + .filter( + Playlist.is_current == True, Playlist.playlist_id == PLAYLIST_ID_OFFSET + ) .first() ) assert playlist_1.playlist_name == "playlist 1 updated" @@ -145,7 +147,10 @@ def get_events_side_effect(_, tx_receipt): playlist_2: Playlist = ( session.query(Playlist) - .filter(Playlist.is_current == True, Playlist.playlist_id == 2) + .filter( + Playlist.is_current == True, + Playlist.playlist_id == PLAYLIST_ID_OFFSET + 1, + ) .first() ) assert playlist_2.playlist_name == "playlist 2" @@ -163,11 +168,25 @@ def test_index_invalid_playlists(app, mocker): tx_receipts = { # invalid create + "CreatePlaylistBelowOffset": [ + { + "args": AttributeDict( + { + "_entityId": 1, + "_entityType": "Playlist", + "_userId": 1, + "_action": "Create", + "_metadata": "", + "_signer": "user1wallet", + } + ) + }, + ], "CreatePlaylistUserDoesNotExist": [ { "args": AttributeDict( { - "_entityId": 2, + "_entityId": PLAYLIST_ID_OFFSET + 1, "_entityType": "Playlist", "_userId": 2, "_action": "Create", @@ -181,7 +200,7 @@ def test_index_invalid_playlists(app, mocker): { "args": AttributeDict( { - "_entityId": 2, + "_entityId": PLAYLIST_ID_OFFSET + 1, "_entityType": "Playlist", "_userId": 1, "_action": "Create", @@ -195,7 +214,7 @@ def test_index_invalid_playlists(app, mocker): { "args": AttributeDict( { - "_entityId": 1, + "_entityId": PLAYLIST_ID_OFFSET, "_entityType": "Playlist", "_userId": 1, "_action": "Create", @@ -206,11 +225,11 @@ def test_index_invalid_playlists(app, mocker): }, ], # invalid updates - "UpdatePlaylistInvalidOwnership": [ + "UpdatePlaylistInvalidSigner": [ { "args": AttributeDict( { - "_entityId": 1, + "_entityId": PLAYLIST_ID_OFFSET, "_entityType": "Playlist", "_userId": 1, "_action": "Update", @@ -220,11 +239,11 @@ def test_index_invalid_playlists(app, mocker): ) }, ], - "UpdatePlaylistDoesNotMatchPlaylistOwner": [ + "UpdatePlaylistInvalidOwner": [ { "args": AttributeDict( { - "_entityId": 1, + "_entityId": PLAYLIST_ID_OFFSET, "_entityType": "Playlist", "_userId": 2, "_action": "Update", @@ -235,11 +254,11 @@ def test_index_invalid_playlists(app, mocker): }, ], # invalid deletes - "DeletePlaylistInvalidOwnership": [ + "DeletePlaylistInvalidSigner": [ { "args": AttributeDict( { - "_entityId": 1, + "_entityId": PLAYLIST_ID_OFFSET, "_entityType": "Playlist", "_userId": 1, "_action": "Delete", @@ -253,7 +272,7 @@ def test_index_invalid_playlists(app, mocker): { "args": AttributeDict( { - "_entityId": 2, + "_entityId": PLAYLIST_ID_OFFSET + 1, "_entityType": "Playlist", "_userId": 1, "_action": "Update", @@ -263,11 +282,11 @@ def test_index_invalid_playlists(app, mocker): ) }, ], - "DeletePlaylistDoesNotMatchPlaylistOwner": [ + "DeletePlaylistInvalidOwner": [ { "args": AttributeDict( { - "_entityId": 1, + "_entityId": PLAYLIST_ID_OFFSET + 1, "_entityType": "Playlist", "_userId": 2, "_action": "Update", @@ -279,7 +298,7 @@ def test_index_invalid_playlists(app, mocker): ], } - audius_data_txs = [ + entity_manager_txs = [ AttributeDict({"transactionHash": update_task.web3.toBytes(text=tx_receipt)}) for tx_receipt in tx_receipts ] @@ -288,7 +307,7 @@ def get_events_side_effect(_, tx_receipt): return tx_receipts[tx_receipt.transactionHash.decode("utf-8")] mocker.patch( - "src.tasks.audius_data.get_audius_data_events_tx", + "src.tasks.entity_manager.get_entity_manager_events_tx", side_effect=get_events_side_effect, autospec=True, ) @@ -299,18 +318,18 @@ def get_events_side_effect(_, tx_receipt): {"user_id": 2, "handle": "user-1", "wallet": "User2Wallet"}, ], "playlists": [ - {"playlist_id": 1, "playlist_owner_id": 1}, + {"playlist_id": PLAYLIST_ID_OFFSET, "playlist_owner_id": 1}, ], } populate_mock_db(db, entities) with db.scoped_session() as session: # index transactions - audius_data_state_update( + entity_manager_update( None, update_task, session, - audius_data_txs, + entity_manager_txs, block_number=0, block_timestamp=1585336422, block_hash=0, diff --git a/discovery-provider/src/app.py b/discovery-provider/src/app.py index a6d137c24e5..daf577ab284 100644 --- a/discovery-provider/src/app.py +++ b/discovery-provider/src/app.py @@ -63,7 +63,7 @@ playlist_factory = None user_library_factory = None user_replica_set_manager = None -audius_data = None +entity_manager = None contract_addresses: Dict[str, Any] = defaultdict() logger = logging.getLogger(__name__) @@ -127,11 +127,11 @@ def init_contracts(): abi=abi_values["UserReplicaSetManager"]["abi"], ) - audius_data_address = web3.toChecksumAddress( + entity_manager_address = web3.toChecksumAddress( shared_config["contracts"]["entity_manager_address"] ) - audius_data_inst = web3.eth.contract( - address=audius_data_address, abi=abi_values["AudiusData"]["abi"] + entity_manager_inst = web3.eth.contract( + address=entity_manager_address, abi=abi_values["EntityManager"]["abi"] ) contract_address_dict = { @@ -142,7 +142,7 @@ def init_contracts(): "playlist_factory": playlist_factory_address, "user_library_factory": user_library_factory_address, "user_replica_set_manager": user_replica_set_manager_address, - "audius_data": audius_data_address, + "entity_manager": entity_manager_address, } return ( @@ -153,7 +153,7 @@ def init_contracts(): playlist_factory_inst, user_library_factory_inst, user_replica_set_manager_inst, - audius_data_inst, + entity_manager_inst, contract_address_dict, ) @@ -186,7 +186,7 @@ def create_celery(test_config=None): global playlist_factory global user_library_factory global user_replica_set_manager - global audius_data + global entity_manager global contract_addresses # pylint: enable=W0603 @@ -198,7 +198,7 @@ def create_celery(test_config=None): playlist_factory, user_library_factory, user_replica_set_manager, - audius_data, + entity_manager, contract_addresses, ) = init_contracts() diff --git a/discovery-provider/src/tasks/audius_data.py b/discovery-provider/src/tasks/entity_manager.py similarity index 94% rename from discovery-provider/src/tasks/audius_data.py rename to discovery-provider/src/tasks/entity_manager.py index 1f1470f8788..ee32c6458e7 100644 --- a/discovery-provider/src/tasks/audius_data.py +++ b/discovery-provider/src/tasks/entity_manager.py @@ -35,11 +35,11 @@ def __str__(self) -> str: MANAGE_ENTITY_EVENT_TYPE = "ManageEntity" -def audius_data_state_update( - self, +def entity_manager_update( + _, update_task: DatabaseTask, session: Session, - audius_data_txs, + entity_manager_txs, block_number, block_timestamp, block_hash, @@ -51,7 +51,7 @@ def audius_data_state_update( changed_entity_ids: Dict[str, Set[(int)]] = defaultdict(set) - if not audius_data_txs: + if not entity_manager_txs: return num_total_changes, changed_entity_ids entities_to_fetch: Dict[EntityType, Set[int]] = defaultdict(set) @@ -59,7 +59,7 @@ def audius_data_state_update( # collect events by entity type and action collect_entities_to_fetch( - update_task, audius_data_txs, entities_to_fetch, users_to_fetch + update_task, entity_manager_txs, entities_to_fetch, users_to_fetch ) # fetch existing playlists @@ -72,10 +72,12 @@ def audius_data_state_update( playlists_to_save: Dict[int, List[Playlist]] = defaultdict(list) # process in tx order and populate playlists_to_save - for tx_receipt in audius_data_txs: + for tx_receipt in entity_manager_txs: txhash = update_task.web3.toHex(tx_receipt.transactionHash) - audius_data_event_tx = get_audius_data_events_tx(update_task, tx_receipt) - for event in audius_data_event_tx: + entity_manager_event_tx = get_entity_manager_events_tx( + update_task, tx_receipt + ) + for event in entity_manager_event_tx: params = ManagePlaylistParameters( event, playlists_to_save, # actions below populate these records @@ -261,13 +263,13 @@ def delete_playlist(params: ManagePlaylistParameters): def collect_entities_to_fetch( update_task, - audius_data_txs, + entity_manager_txs, entities_to_fetch: Dict[EntityType, Set[int]], users_to_fetch: Set[int], ): - for tx_receipt in audius_data_txs: - audius_data_event_tx = get_audius_data_events_tx(update_task, tx_receipt) - for event in audius_data_event_tx: + for tx_receipt in entity_manager_txs: + entity_manager_event_tx = get_entity_manager_events_tx(update_task, tx_receipt) + for event in entity_manager_event_tx: entity_id = helpers.get_tx_arg(event, "_entityId") entity_type = helpers.get_tx_arg(event, "_entityType") user_id = helpers.get_tx_arg(event, "_userId") @@ -332,9 +334,9 @@ def copy_record(old_playlist: Playlist, block_number, event_blockhash, txhash): return new_playlist -def get_audius_data_events_tx(update_task, tx_receipt): +def get_entity_manager_events_tx(update_task, tx_receipt): return getattr( - update_task.audius_data_contract.events, MANAGE_ENTITY_EVENT_TYPE + update_task.entity_manager_contract.events, MANAGE_ENTITY_EVENT_TYPE )().processReceipt(tx_receipt) diff --git a/discovery-provider/src/tasks/index.py b/discovery-provider/src/tasks/index.py index dfc322234a1..0c99e930cf0 100644 --- a/discovery-provider/src/tasks/index.py +++ b/discovery-provider/src/tasks/index.py @@ -30,8 +30,8 @@ set_indexing_error, ) from src.queries.skipped_transactions import add_network_level_skipped_transaction -from src.tasks.audius_data import audius_data_state_update from src.tasks.celery_app import celery +from src.tasks.entity_manager import entity_manager_update from src.tasks.playlists import playlist_state_update from src.tasks.social_features import social_feature_state_update from src.tasks.sort_block_transactions import sort_block_transactions @@ -67,7 +67,7 @@ most_recent_indexed_block_redis_key, ) from src.utils.session_manager import SessionManager -from src.utils.user_event_constants import audius_data_event_types_arr +from src.utils.user_event_constants import entity_manager_event_types_arr USER_FACTORY = CONTRACT_TYPES.USER_FACTORY.value TRACK_FACTORY = CONTRACT_TYPES.TRACK_FACTORY.value @@ -75,7 +75,7 @@ PLAYLIST_FACTORY = CONTRACT_TYPES.PLAYLIST_FACTORY.value USER_LIBRARY_FACTORY = CONTRACT_TYPES.USER_LIBRARY_FACTORY.value USER_REPLICA_SET_MANAGER = CONTRACT_TYPES.USER_REPLICA_SET_MANAGER.value -AUDIUS_DATA = CONTRACT_TYPES.AUDIUS_DATA.value +ENTITY_MANAGER = CONTRACT_TYPES.ENTITY_MANAGER.value USER_FACTORY_CONTRACT_NAME = CONTRACT_NAMES_ON_CHAIN[CONTRACT_TYPES.USER_FACTORY] TRACK_FACTORY_CONTRACT_NAME = CONTRACT_NAMES_ON_CHAIN[CONTRACT_TYPES.TRACK_FACTORY] @@ -91,7 +91,7 @@ USER_REPLICA_SET_MANAGER_CONTRACT_NAME = CONTRACT_NAMES_ON_CHAIN[ CONTRACT_TYPES.USER_REPLICA_SET_MANAGER ] -AUDIUS_DATA_CONTRACT_NAME = CONTRACT_NAMES_ON_CHAIN[CONTRACT_TYPES.AUDIUS_DATA] +ENTITY_MANAGER_CONTRACT_NAME = CONTRACT_NAMES_ON_CHAIN[CONTRACT_TYPES.ENTITY_MANAGER] TX_TYPE_TO_HANDLER_MAP = { USER_FACTORY: user_state_update, @@ -100,7 +100,7 @@ PLAYLIST_FACTORY: playlist_state_update, USER_LIBRARY_FACTORY: user_library_state_update, USER_REPLICA_SET_MANAGER: user_replica_set_state_update, - AUDIUS_DATA: audius_data_state_update, + ENTITY_MANAGER: entity_manager_update, } BLOCKS_PER_DAY = (24 * 60 * 60) / 5 @@ -267,11 +267,11 @@ def fetch_tx_receipts(self, block): return block_tx_with_receipts -def fetch_cid_metadata(db, user_factory_txs, track_factory_txs, audius_data_txs): +def fetch_cid_metadata(db, user_factory_txs, track_factory_txs, entity_manager_txs): start_time = datetime.now() user_contract = update_task.user_contract track_contract = update_task.track_contract - audius_data_contract = update_task.audius_data_contract + entity_manager_contract = update_task.entity_manager_contract cids_txhash_set: Tuple[str, Any] = set() cid_type: Dict[str, str] = {} # cid -> entity type track / user @@ -317,13 +317,13 @@ def fetch_cid_metadata(db, user_factory_txs, track_factory_txs, audius_data_txs) cid_type[cid] = "track" cid_to_user_id[cid] = track_owner_id - for tx_receipt in audius_data_txs: + for tx_receipt in entity_manager_txs: txhash = update_task.web3.toHex(tx_receipt.transactionHash) - for event_type in audius_data_event_types_arr: - audius_data_events_tx = getattr( - audius_data_contract.events, event_type + for event_type in entity_manager_event_types_arr: + entity_manager_events_tx = getattr( + entity_manager_contract.events, event_type )().processReceipt(tx_receipt) - for entry in audius_data_events_tx: + for entry in entity_manager_events_tx: event_args = entry["args"] user_id = event_args._userId entity_type = event_args._entityType @@ -614,7 +614,7 @@ def index_blocks(self, db, blocks_list): PLAYLIST_FACTORY: [], USER_LIBRARY_FACTORY: [], USER_REPLICA_SET_MANAGER: [], - AUDIUS_DATA: [], + ENTITY_MANAGER: [], } try: """ @@ -680,7 +680,7 @@ def index_blocks(self, db, blocks_list): db, txs_grouped_by_type[USER_FACTORY], txs_grouped_by_type[TRACK_FACTORY], - txs_grouped_by_type[AUDIUS_DATA], + txs_grouped_by_type[ENTITY_MANAGER], ) logger.info( f"index.py | index_blocks - fetch_ipfs_metadata in {time.time() - fetch_ipfs_metadata_start_time}s" @@ -1138,9 +1138,12 @@ def update_task(self): abi=user_replica_set_manager_abi, ) - audius_data_contract_abi = update_task.abi_values[AUDIUS_DATA_CONTRACT_NAME]["abi"] - audius_data_contract = update_task.web3.eth.contract( - address=get_contract_addresses()[AUDIUS_DATA], abi=audius_data_contract_abi + entity_manager_contract_abi = update_task.abi_values[ENTITY_MANAGER_CONTRACT_NAME][ + "abi" + ] + entity_manager_contract = update_task.web3.eth.contract( + address=get_contract_addresses()[ENTITY_MANAGER], + abi=entity_manager_contract_abi, ) update_task.track_contract = track_contract @@ -1149,7 +1152,7 @@ def update_task(self): update_task.social_feature_contract = social_feature_contract update_task.user_library_contract = user_library_contract update_task.user_replica_set_manager_contract = user_replica_set_manager_contract - update_task.audius_data_contract = audius_data_contract + update_task.entity_manager_contract = entity_manager_contract # Update redis cache for health check queries update_latest_block_redis() diff --git a/discovery-provider/src/tasks/playlists.py b/discovery-provider/src/tasks/playlists.py index 5ece9c16c69..eb6d70b3376 100644 --- a/discovery-provider/src/tasks/playlists.py +++ b/discovery-provider/src/tasks/playlists.py @@ -7,7 +7,7 @@ from src.database_task import DatabaseTask from src.models.playlists.playlist import Playlist from src.queries.skipped_transactions import add_node_level_skipped_transaction -from src.tasks.audius_data import PLAYLIST_ID_OFFSET +from src.tasks.entity_manager import PLAYLIST_ID_OFFSET from src.utils import helpers from src.utils.indexing_errors import EntityMissingRequiredFieldError, IndexingError from src.utils.model_nullable_validator import all_required_fields_present diff --git a/discovery-provider/src/utils/constants.py b/discovery-provider/src/utils/constants.py index d7221a593df..29e55f45c2b 100644 --- a/discovery-provider/src/utils/constants.py +++ b/discovery-provider/src/utils/constants.py @@ -1034,7 +1034,7 @@ class CONTRACT_TYPES(Enum): PLAYLIST_FACTORY = "playlist_factory" USER_LIBRARY_FACTORY = "user_library_factory" USER_REPLICA_SET_MANAGER = "user_replica_set_manager" - AUDIUS_DATA = "audius_data" + ENTITY_MANAGER = "entity_manager" CONTRACT_NAMES_ON_CHAIN = { @@ -1044,5 +1044,5 @@ class CONTRACT_TYPES(Enum): CONTRACT_TYPES.PLAYLIST_FACTORY: "PlaylistFactory", CONTRACT_TYPES.USER_LIBRARY_FACTORY: "UserLibraryFactory", CONTRACT_TYPES.USER_REPLICA_SET_MANAGER: "UserReplicaSetManager", - CONTRACT_TYPES.AUDIUS_DATA: "AudiusData", + CONTRACT_TYPES.ENTITY_MANAGER: "EntityManager", } diff --git a/discovery-provider/src/utils/user_event_constants.py b/discovery-provider/src/utils/user_event_constants.py index 4503e9ed039..5705bc34727 100644 --- a/discovery-provider/src/utils/user_event_constants.py +++ b/discovery-provider/src/utils/user_event_constants.py @@ -36,6 +36,6 @@ user_replica_set_manager_event_types_lookup["add_or_update_content_node"], ] -audius_data_event_types_lookup = {"manage_entity": "ManageEntity"} +entity_manager_event_types_lookup = {"manage_entity": "ManageEntity"} -audius_data_event_types_arr = [audius_data_event_types_lookup["manage_entity"]] +entity_manager_event_types_arr = [entity_manager_event_types_lookup["manage_entity"]] From 15d21e01eb2005367ee80d219387eb489d9607d3 Mon Sep 17 00:00:00 2001 From: Isaac Date: Tue, 2 Aug 2022 23:23:01 +0000 Subject: [PATCH 089/101] working playlist updates --- discovery-provider/src/api/v1/helpers.py | 6 +- .../src/api/v1/models/playlists.py | 1 + .../src/tasks/entity_manager.py | 73 ++++++-- discovery-provider/src/tasks/index.py | 1 - libs/data-contracts/ABIs/AudiusData.json | 174 ++++++++++++++++++ libs/src/api/Track.ts | 2 + libs/src/api/entityManager.ts | 24 ++- libs/src/libs.js | 1 + libs/src/services/creatorNode/CreatorNode.ts | 2 +- 9 files changed, 255 insertions(+), 29 deletions(-) create mode 100644 libs/data-contracts/ABIs/AudiusData.json diff --git a/discovery-provider/src/api/v1/helpers.py b/discovery-provider/src/api/v1/helpers.py index 9235c757448..c06a6c1bf98 100644 --- a/discovery-provider/src/api/v1/helpers.py +++ b/discovery-provider/src/api/v1/helpers.py @@ -67,7 +67,11 @@ def add_playlist_added_timestamps(playlist): added_timestamps = [] for track in playlist["playlist_contents"]["track_ids"]: added_timestamps.append( - {"track_id": encode_int_id(track["track"]), "timestamp": track["time"]} + { + "track_id": encode_int_id(track["track"]), + "timestamp": track["time"], + "metadata_timestamp": track["metadata_time"], + } ) return added_timestamps diff --git a/discovery-provider/src/api/v1/models/playlists.py b/discovery-provider/src/api/v1/models/playlists.py index 8d7927e02c4..32809a06c69 100644 --- a/discovery-provider/src/api/v1/models/playlists.py +++ b/discovery-provider/src/api/v1/models/playlists.py @@ -16,6 +16,7 @@ playlist_added_timestamp = ns.model( "playlist_added_timestamp", { + "metadata_timestamp": fields.Integer(required=True), "timestamp": fields.Integer(required=True), "track_id": fields.String(required=True), }, diff --git a/discovery-provider/src/tasks/entity_manager.py b/discovery-provider/src/tasks/entity_manager.py index ee32c6458e7..26a1ed3d82e 100644 --- a/discovery-provider/src/tasks/entity_manager.py +++ b/discovery-provider/src/tasks/entity_manager.py @@ -190,13 +190,19 @@ def create_playlist(params: ManagePlaylistParameters): return metadata = params.ipfs_metadata[params.metadata_cid] - - track_ids = metadata["playlist_contents"] - playlist_contents = [] - for track_id in track_ids: - playlist_contents.append({"track": track_id, "time": params.block_integer_time}) + tracks = metadata["playlist_contents"].get("track_ids", []) + tracks_with_index_time = [] + for track in tracks: + tracks_with_index_time.append( + { + "track": track["track"], + "metadata_time": track["time"], + "time": params.block_integer_time, + } + ) create_playlist_record = Playlist( playlist_id=params.entity_id, + metadata_multihash=params.metadata_cid, playlist_owner_id=params.user_id, is_album=metadata.get("is_album", False), description=metadata["description"], @@ -204,7 +210,7 @@ def create_playlist(params: ManagePlaylistParameters): playlist_image_sizes_multihash=metadata["playlist_image_sizes_multihash"], playlist_name=metadata["playlist_name"], is_private=metadata.get("is_private", False), - playlist_contents={"track_ids": playlist_contents}, + playlist_contents={"track_ids": tracks_with_index_time}, created_at=params.block_datetime, updated_at=params.block_datetime, blocknumber=params.block_number, @@ -234,13 +240,15 @@ def update_playlist(params: ManagePlaylistParameters): updated_playlist = copy_record( existing_playlist, params.block_number, params.event_blockhash, params.txhash ) - parse_playlist_data_event( + process_playlist_data_event( updated_playlist, metadata, params.block_integer_time, params.block_datetime, + params.metadata_cid, ) params.playlists_to_save[params.entity_id].append(updated_playlist) + params.existing_playlist_id_to_playlist[params.entity_id] = updated_playlist def delete_playlist(params: ManagePlaylistParameters): @@ -340,11 +348,44 @@ def get_entity_manager_events_tx(update_task, tx_receipt): )().processReceipt(tx_receipt) -def parse_playlist_data_event( +def process_playlist_contents(playlist_record, playlist_metadata, block_integer_time): + # mapping of track's metadata time to index time + metadata_index_time_dict: Dict[int, Dict[int, int]] = defaultdict(dict) + for track in playlist_record.playlist_contents["track_ids"]: + track_id = track["track"] + metadata_time = track["metadata_time"] + + metadata_index_time_dict[track_id][metadata_time] = track["time"] + + updated_tracks = [] + for track in playlist_metadata["playlist_contents"]["track_ids"]: + track_id = track["track"] + metadata_time = track["time"] + index_time = block_integer_time # default to current block for new tracks + + if ( + track_id in metadata_index_time_dict + and metadata_time in metadata_index_time_dict[track_id] + ): + # track exists in prev record (reorder / delete) + index_time = metadata_index_time_dict[track_id][metadata_time] + + updated_tracks.append( + { + "track": track_id, + "time": index_time, + "metadata_time": metadata_time, + } + ) + return {"track_ids": updated_tracks} + + +def process_playlist_data_event( playlist_record: Playlist, playlist_metadata, block_integer_time, block_datetime, + metadata_cid, ): playlist_record.is_album = ( playlist_metadata["is_album"] if "is_album" in playlist_metadata else False @@ -360,17 +401,11 @@ def parse_playlist_data_event( playlist_record.is_private = ( playlist_metadata["is_private"] if "is_private" in playlist_metadata else False ) - playlist_content_array = [] - track_ids = playlist_metadata["playlist_contents"] - if track_ids: - for track_id in track_ids: - playlist_content_array.append( - {"track": track_id, "time": block_integer_time} - ) - playlist_record.playlist_contents = {"track_ids": playlist_content_array} + playlist_record.playlist_contents = process_playlist_contents( + playlist_record, playlist_metadata, block_integer_time + ) playlist_record.created_at = block_datetime playlist_record.updated_at = block_datetime + playlist_record.metadata_multihash = metadata_cid - logger.info(f"index.py | AudiusData | Created playlist record {playlist_record}") - # TODO: All required fields validation - return playlist_record + logger.info(f"index.py | AudiusData | Updated playlist record {playlist_record}") diff --git a/discovery-provider/src/tasks/index.py b/discovery-provider/src/tasks/index.py index 0c99e930cf0..6fc1510963d 100644 --- a/discovery-provider/src/tasks/index.py +++ b/discovery-provider/src/tasks/index.py @@ -720,7 +720,6 @@ def index_blocks(self, db, blocks_list): Add state changes in block to db (users, tracks, etc.) """ process_state_changes_start_time = time.time() - logger.info(f"index.py | processing {txs_grouped_by_type}") # bulk process operations once all tx's for block have been parsed # and get changed entity IDs for cache clearing # after session commit diff --git a/libs/data-contracts/ABIs/AudiusData.json b/libs/data-contracts/ABIs/AudiusData.json new file mode 100644 index 00000000000..02f5b9651d1 --- /dev/null +++ b/libs/data-contracts/ABIs/AudiusData.json @@ -0,0 +1,174 @@ +{ + "contractName": "AudiusData", + "abi": [ + { + "constant": true, + "inputs": [ + { + "name": "", + "type": "bytes32" + } + ], + "name": "usedSignatures", + "outputs": [ + { + "name": "", + "type": "bool" + } + ], + "payable": false, + "stateMutability": "view", + "type": "function" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": false, + "name": "_userId", + "type": "uint256" + }, + { + "indexed": false, + "name": "_signer", + "type": "address" + }, + { + "indexed": false, + "name": "_entityType", + "type": "string" + }, + { + "indexed": false, + "name": "_entityId", + "type": "uint256" + }, + { + "indexed": false, + "name": "_metadata", + "type": "string" + }, + { + "indexed": false, + "name": "_action", + "type": "string" + } + ], + "name": "ManageEntity", + "type": "event" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": false, + "name": "_userId", + "type": "uint256" + }, + { + "indexed": false, + "name": "_isVerified", + "type": "bool" + } + ], + "name": "ManageIsVerified", + "type": "event" + }, + { + "constant": false, + "inputs": [ + { + "name": "name", + "type": "string" + }, + { + "name": "version", + "type": "string" + }, + { + "name": "chainId", + "type": "uint256" + } + ], + "name": "initialize", + "outputs": [], + "payable": false, + "stateMutability": "nonpayable", + "type": "function" + }, + { + "constant": false, + "inputs": [ + { + "name": "_verifierAddress", + "type": "address" + }, + { + "name": "_networkId", + "type": "uint256" + } + ], + "name": "initialize", + "outputs": [], + "payable": false, + "stateMutability": "nonpayable", + "type": "function" + }, + { + "constant": false, + "inputs": [ + { + "name": "_userId", + "type": "uint256" + }, + { + "name": "_entityType", + "type": "string" + }, + { + "name": "_entityId", + "type": "uint256" + }, + { + "name": "_action", + "type": "string" + }, + { + "name": "_metadata", + "type": "string" + }, + { + "name": "_nonce", + "type": "bytes32" + }, + { + "name": "_subjectSig", + "type": "bytes" + } + ], + "name": "manageEntity", + "outputs": [], + "payable": false, + "stateMutability": "nonpayable", + "type": "function" + }, + { + "constant": false, + "inputs": [ + { + "name": "_userId", + "type": "uint256" + }, + { + "name": "_isVerified", + "type": "bool" + } + ], + "name": "manageIsVerified", + "outputs": [], + "payable": false, + "stateMutability": "nonpayable", + "type": "function" + } + ] +} \ No newline at end of file diff --git a/libs/src/api/Track.ts b/libs/src/api/Track.ts index 4b4c6a58d05..e937cf2cbba 100644 --- a/libs/src/api/Track.ts +++ b/libs/src/api/Track.ts @@ -380,6 +380,7 @@ export class Track extends Base { phase = phases.UPLOADING_TRACK_CONTENT // Upload metadata + console.log('asdf uploadTrackContent to CN', metadata) const { metadataMultihash, metadataFileUUID, @@ -408,6 +409,7 @@ export class Track extends Base { } } ) + console.log('asdf uploadTrackContent to CN metadataMultihash', metadataMultihash) phase = phases.ADDING_TRACK diff --git a/libs/src/api/entityManager.ts b/libs/src/api/entityManager.ts index 01298eb2707..f3812874f99 100644 --- a/libs/src/api/entityManager.ts +++ b/libs/src/api/entityManager.ts @@ -1,6 +1,7 @@ import { Base, Services } from './base' import type { PlaylistMetadata } from '../services/creatorNode' import type { Nullable } from '../utils' +import web3 from '../web3' export enum Action { CREATE = 'Create', @@ -123,10 +124,14 @@ export class EntityManager extends Base { coverArt, true // square ) + const web3 = this.ethWeb3Manager.getWeb3() + const currentBlockNumber = await web3.eth.getBlockNumber() + const currentBlock = await web3.eth.getBlock(currentBlockNumber) + const tracks = trackIds.map(trackId => ({ track: trackId, time: currentBlock.timestamp })) const dirCID = updatedPlaylistImage.dirCID const metadata: PlaylistMetadata = { playlist_id: entityId, - playlist_contents: trackIds, + playlist_contents: { track_ids: tracks }, playlist_name: playlistName, playlist_image_sizes_multihash: dirCID, description, @@ -199,7 +204,7 @@ export class EntityManager extends Base { async updatePlaylist({ playlistId, playlistName, - trackIds, + playlistContents, description, isAlbum, isPrivate, @@ -207,8 +212,8 @@ export class EntityManager extends Base { logger = console }: { playlistId: number + playlistContents: any playlistName: Nullable - trackIds: Nullable description: Nullable isAlbum: Nullable isPrivate: Nullable @@ -217,7 +222,9 @@ export class EntityManager extends Base { }): Promise { const responseValues: PlaylistOperationResponse = this.getDefaultPlaylistReponseValues() + try { + const currentUserId: string | null = this.userStateManager.getCurrentUserId() if (!playlistId || playlistId === undefined) { @@ -245,9 +252,13 @@ export class EntityManager extends Base { await this.discoveryProvider.getPlaylists(1, 0, [playlistId], userId) )[0] + if (playlistContents) { + playlistContents.track_ids = playlistContents.track_ids.map((track_obj: { track: any; time: any }) => ({ track: track_obj.track, time: track_obj.time })) + } + const metadata: PlaylistMetadata = { playlist_id: playlistId, - playlist_contents: trackIds, + playlist_contents: playlistContents ?? playlist.playlist_contents, playlist_name: playlistName ?? playlist.playlist_name, playlist_image_sizes_multihash: dirCID || playlist.playlist_image_sizes_multihash, @@ -257,7 +268,6 @@ export class EntityManager extends Base { } const { metadataMultihash } = await this.creatorNode.uploadPlaylistMetadata(metadata) - const resp = await this.manageEntity({ userId, entityType, @@ -276,7 +286,7 @@ export class EntityManager extends Base { return responseValues } } - + /** * Manage an entity with the updated data contract flow * Leveraged to manipulate User/Track/Playlist/+ other entities @@ -312,4 +322,4 @@ export class EntityManager extends Base { return { txReceipt: null, error } } } -} +} \ No newline at end of file diff --git a/libs/src/libs.js b/libs/src/libs.js index 5e470834ab0..6d2cdb05c5c 100644 --- a/libs/src/libs.js +++ b/libs/src/libs.js @@ -34,6 +34,7 @@ const Web3 = require('./web3') const { Keypair } = require('@solana/web3.js') const { PublicKey } = require('@solana/web3.js') +const { EntityManager } = require('./api/entityManager') const { getPlatformLocalStorage } = require('./utils/localStorage') class AudiusLibs { diff --git a/libs/src/services/creatorNode/CreatorNode.ts b/libs/src/services/creatorNode/CreatorNode.ts index 48448219750..8b90930a578 100644 --- a/libs/src/services/creatorNode/CreatorNode.ts +++ b/libs/src/services/creatorNode/CreatorNode.ts @@ -28,7 +28,7 @@ type Metadata = { } export type PlaylistMetadata = { - playlist_contents: Nullable + playlist_contents: any playlist_id: number playlist_name: string playlist_image_sizes_multihash: string From f55df6e369360dc4eea7709ab72f79c59bfc5aae Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 4 Aug 2022 19:55:11 +0000 Subject: [PATCH 090/101] metadata time default to index time --- discovery-provider/src/api/v1/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/discovery-provider/src/api/v1/helpers.py b/discovery-provider/src/api/v1/helpers.py index c06a6c1bf98..db6745b6aa5 100644 --- a/discovery-provider/src/api/v1/helpers.py +++ b/discovery-provider/src/api/v1/helpers.py @@ -70,7 +70,7 @@ def add_playlist_added_timestamps(playlist): { "track_id": encode_int_id(track["track"]), "timestamp": track["time"], - "metadata_timestamp": track["metadata_time"], + "metadata_timestamp": track.get("metadata_time", track["time"]), } ) return added_timestamps From 7c738134c0f0bfb8014c9b3721cbc074c2adad2b Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 4 Aug 2022 20:59:55 +0000 Subject: [PATCH 091/101] clean up --- .../src/tasks/entity_manager.py | 19 +++++++++++-------- libs/src/api/Track.ts | 3 --- libs/src/api/entityManager.ts | 8 +++++++- libs/src/services/creatorNode/CreatorNode.ts | 8 +++++++- 4 files changed, 25 insertions(+), 13 deletions(-) diff --git a/discovery-provider/src/tasks/entity_manager.py b/discovery-provider/src/tasks/entity_manager.py index 26a1ed3d82e..1b52efb4f38 100644 --- a/discovery-provider/src/tasks/entity_manager.py +++ b/discovery-provider/src/tasks/entity_manager.py @@ -70,7 +70,7 @@ def entity_manager_update( # fetch users existing_user_id_to_user: Dict[int, User] = fetch_users(session, users_to_fetch) - playlists_to_save: Dict[int, List[Playlist]] = defaultdict(list) + new_playlist_records: Dict[int, List[Playlist]] = defaultdict(list) # process in tx order and populate playlists_to_save for tx_receipt in entity_manager_txs: txhash = update_task.web3.toHex(tx_receipt.transactionHash) @@ -80,7 +80,7 @@ def entity_manager_update( for event in entity_manager_event_tx: params = ManagePlaylistParameters( event, - playlists_to_save, # actions below populate these records + new_playlist_records, # actions below populate these records existing_playlist_id_to_playlist, existing_user_id_to_user, ipfs_metadata, @@ -107,15 +107,15 @@ def entity_manager_update( delete_playlist(params) # compile records_to_save - records_to_save = [] - for _, playlist_records in playlists_to_save.items(): + new_records = [] + for playlist_records in new_playlist_records.values(): # flip is_current to true for the last tx in each playlist playlist_records[-1].is_current = True - records_to_save.extend(playlist_records) + new_records.extend(playlist_records) # insert/update all playlist records in this block - session.bulk_save_objects(records_to_save) - num_total_changes += len(records_to_save) + session.bulk_save_objects(new_records) + num_total_changes += len(new_records) except Exception as e: logger.error(f"Exception occurred {e}", exc_info=True) @@ -164,6 +164,10 @@ def is_valid_playlist_tx(params: ManagePlaylistParameters): if wallet and wallet.lower() != params.signer.lower(): # user does not match signer return False + + if params.entity_type != EntityType.PLAYLIST: + return False + if params.action == Action.CREATE: if params.entity_id in params.existing_playlist_id_to_playlist: # playlist already exists @@ -404,7 +408,6 @@ def process_playlist_data_event( playlist_record.playlist_contents = process_playlist_contents( playlist_record, playlist_metadata, block_integer_time ) - playlist_record.created_at = block_datetime playlist_record.updated_at = block_datetime playlist_record.metadata_multihash = metadata_cid diff --git a/libs/src/api/Track.ts b/libs/src/api/Track.ts index e937cf2cbba..35233284298 100644 --- a/libs/src/api/Track.ts +++ b/libs/src/api/Track.ts @@ -380,7 +380,6 @@ export class Track extends Base { phase = phases.UPLOADING_TRACK_CONTENT // Upload metadata - console.log('asdf uploadTrackContent to CN', metadata) const { metadataMultihash, metadataFileUUID, @@ -409,8 +408,6 @@ export class Track extends Base { } } ) - console.log('asdf uploadTrackContent to CN metadataMultihash', metadataMultihash) - phase = phases.ADDING_TRACK // Write metadata to chain diff --git a/libs/src/api/entityManager.ts b/libs/src/api/entityManager.ts index f3812874f99..5a5546af1da 100644 --- a/libs/src/api/entityManager.ts +++ b/libs/src/api/entityManager.ts @@ -37,6 +37,12 @@ const MIN_PLAYLIST_ID = 400000 // Maximum playlist ID, reflects postgres max integer value const MAX_PLAYLIST_ID = 2147483647 +type PlaylistTrackId = { time: number; track: number; metadata_time?: number } + +type PlaylistContents = { + track_ids: Array +} + /* API surface for updated data contract interactions. Provides simplified entity management in a generic fashion @@ -212,7 +218,7 @@ export class EntityManager extends Base { logger = console }: { playlistId: number - playlistContents: any + playlistContents: Nullable playlistName: Nullable description: Nullable isAlbum: Nullable diff --git a/libs/src/services/creatorNode/CreatorNode.ts b/libs/src/services/creatorNode/CreatorNode.ts index 8b90930a578..c5a07f41df1 100644 --- a/libs/src/services/creatorNode/CreatorNode.ts +++ b/libs/src/services/creatorNode/CreatorNode.ts @@ -27,8 +27,14 @@ type Metadata = { cover_art_sizes: string } +type PlaylistTrackId = { time: number; track: number; } + +type PlaylistContents = { + track_ids: Array +} + export type PlaylistMetadata = { - playlist_contents: any + playlist_contents: PlaylistContents playlist_id: number playlist_name: string playlist_image_sizes_multihash: string From 9baca2f8cca814d21afeaaadc53fb858c5d1ab8d Mon Sep 17 00:00:00 2001 From: Isaac Solo Date: Fri, 5 Aug 2022 14:25:16 -0700 Subject: [PATCH 092/101] available playlist ID endpoint (#3651) * Add is playlist id occupied to dn * add get playlist occupied test * add libs discovery endpoint Co-authored-by: jowlee --- .../queries/test_get_playlist_id_occupied.py | 33 +++++++++++++++++++ .../src/api/v1/models/common.py | 7 ++++ discovery-provider/src/api/v1/playlists.py | 24 ++++++++++++++ .../src/queries/get_playlist_is_occupied.py | 31 +++++++++++++++++ .../discoveryProvider/DiscoveryProvider.ts | 5 +++ .../services/discoveryProvider/requests.ts | 6 ++++ 6 files changed, 106 insertions(+) create mode 100644 discovery-provider/integration_tests/queries/test_get_playlist_id_occupied.py create mode 100644 discovery-provider/src/queries/get_playlist_is_occupied.py diff --git a/discovery-provider/integration_tests/queries/test_get_playlist_id_occupied.py b/discovery-provider/integration_tests/queries/test_get_playlist_id_occupied.py new file mode 100644 index 00000000000..125c467ba8d --- /dev/null +++ b/discovery-provider/integration_tests/queries/test_get_playlist_id_occupied.py @@ -0,0 +1,33 @@ +from integration_tests.utils import populate_mock_db +from src.queries.get_playlist_is_occupied import _get_playlist_is_occupied +from src.utils.db_session import get_db + + +def populate_playlist(db): + test_entities = { + "playlists": [ + {"playlist_id": 1, "is_delete": True, "is_private": True, "is_album": True} + ], + "users": [ + {"user_id": 1, "handle": "user1"}, + ], + } + + populate_mock_db(db, test_entities) + + +def test_get_playlist_is_occupied(app): + """Test getting playlist is occupied""" + + with app.app_context(): + db = get_db() + + populate_playlist(db) + + with db.scoped_session() as session: + is_occupied = _get_playlist_is_occupied(session, 1) + + assert is_occupied == True + + is_occupied = _get_playlist_is_occupied(session, 2) + assert is_occupied == False diff --git a/discovery-provider/src/api/v1/models/common.py b/discovery-provider/src/api/v1/models/common.py index 61d44054f96..ad73186c527 100644 --- a/discovery-provider/src/api/v1/models/common.py +++ b/discovery-provider/src/api/v1/models/common.py @@ -38,3 +38,10 @@ "version": fields.Nested(version_metadata, required=True), }, ) + +id_occupied = ns.model( + "occupied", + { + "is_occupied": fields.Boolean(required=True) + }, +) diff --git a/discovery-provider/src/api/v1/playlists.py b/discovery-provider/src/api/v1/playlists.py index 6d0f517028f..368cad97388 100644 --- a/discovery-provider/src/api/v1/playlists.py +++ b/discovery-provider/src/api/v1/playlists.py @@ -21,8 +21,10 @@ success_response, trending_parser, ) +from src.api.v1.models.common import id_occupied from src.api.v1.models.playlists import full_playlist_model, playlist_model from src.api.v1.models.users import user_model_full +from src.queries.get_playlist_is_occupied import get_playlist_is_occupied from src.queries.get_playlist_tracks import get_playlist_tracks from src.queries.get_playlists import get_playlists from src.queries.get_reposters_for_playlist import get_reposters_for_playlist @@ -408,3 +410,25 @@ def get(self, version): ) playlists = get_full_trending_playlists(request, args, strategy) return success_response(playlists) + + +occupied_response = make_response("occupied_response", ns, fields.Nested(id_occupied)) +PLAYLIST_OCCUPIED_ROUTE = "//occupied" + + +@ns.route(PLAYLIST_OCCUPIED_ROUTE) +class Playlist(Resource): + @record_metrics + @ns.doc( + id="""Check if Playlist ID is occupied""", + description="""Check if Playlist ID is occupied""", + params={"playlist_id": "A Playlist ID"}, + responses={200: "Success", 400: "Bad request", 500: "Server error"}, + ) + @ns.marshal_with(occupied_response) + @cache(ttl_sec=5) + def get(self, playlist_id): + playlist_id = decode_with_abort(playlist_id, ns) + is_occupied = get_playlist_is_occupied(playlist_id) + response = success_response({"is_occupied": is_occupied}) + return response diff --git a/discovery-provider/src/queries/get_playlist_is_occupied.py b/discovery-provider/src/queries/get_playlist_is_occupied.py new file mode 100644 index 00000000000..8d7a655a7d8 --- /dev/null +++ b/discovery-provider/src/queries/get_playlist_is_occupied.py @@ -0,0 +1,31 @@ +import logging # pylint: disable=C0302 + +from sqlalchemy.orm import Session +from src.models.playlists.playlist import Playlist +from src.utils.db_session import get_db_read_replica + +logger = logging.getLogger(__name__) + + +def get_playlist_is_occupied(playlist_id: int): + db = get_db_read_replica() + with db.scoped_session() as session: + return _get_playlist_is_occupied(session, playlist_id) + + +def _get_playlist_is_occupied(session: Session, playlist_id: int): + x = session.query( + session.query(Playlist) + .filter(Playlist.is_current == True, Playlist.playlist_id == playlist_id) + .exists() + ) + logger.info(x) + logger.info(x) + logger.info(x) + + playlist_exists = session.query( + session.query(Playlist) + .filter(Playlist.is_current == True, Playlist.playlist_id == playlist_id) + .exists() + ).scalar() + return playlist_exists diff --git a/libs/src/services/discoveryProvider/DiscoveryProvider.ts b/libs/src/services/discoveryProvider/DiscoveryProvider.ts index 1dcda8978ad..9442bbe8f71 100644 --- a/libs/src/services/discoveryProvider/DiscoveryProvider.ts +++ b/libs/src/services/discoveryProvider/DiscoveryProvider.ts @@ -869,6 +869,11 @@ export class DiscoveryProvider { return res.map((r) => ({ ...r, amount: parseInt(r.amount) })) } + async getIsPlaylistIdOccupied(playlistId: string) { + const req = Requests.getIsPlaylistIdOccupied(playlistId) + return await this._makeRequest<{ is_occupied: boolean }>(req) + } + /* ------- INTERNAL FUNCTIONS ------- */ /** diff --git a/libs/src/services/discoveryProvider/requests.ts b/libs/src/services/discoveryProvider/requests.ts index 15e0eb3d37f..e49693374c3 100644 --- a/libs/src/services/discoveryProvider/requests.ts +++ b/libs/src/services/discoveryProvider/requests.ts @@ -689,3 +689,9 @@ export const verifyToken = (token: string) => { } } } + +export const getIsPlaylistIdOccupied = (playlistId: string) => { + return { + endpoint: `/v1/playlists/${playlistId}/occupied` + } +} From 79ae88da86cf331cfe379eefa94140925deeccd3 Mon Sep 17 00:00:00 2001 From: Isaac Date: Mon, 8 Aug 2022 23:38:07 +0000 Subject: [PATCH 093/101] integrate playlist is occupied --- discovery-provider/src/api/v1/playlists.py | 2 +- .../src/tasks/entity_manager.py | 11 +- discovery-provider/src/tasks/index.py | 5 +- .../src/utils/cid_metadata_client.py | 7 - libs/data-contracts/ABIs/AudiusData.json | 174 ------------------ libs/src/AudiusLibs.ts | 6 +- libs/src/api/entityManager.ts | 32 ++-- .../dataContracts/EntityManagerClient.ts | 5 +- .../discoveryProvider/DiscoveryProvider.ts | 11 ++ .../services/discoveryProvider/requests.ts | 15 +- 10 files changed, 54 insertions(+), 214 deletions(-) delete mode 100644 libs/data-contracts/ABIs/AudiusData.json diff --git a/discovery-provider/src/api/v1/playlists.py b/discovery-provider/src/api/v1/playlists.py index 368cad97388..d58ffe27725 100644 --- a/discovery-provider/src/api/v1/playlists.py +++ b/discovery-provider/src/api/v1/playlists.py @@ -417,7 +417,7 @@ def get(self, version): @ns.route(PLAYLIST_OCCUPIED_ROUTE) -class Playlist(Resource): +class GetPlaylistIsOccupied(Resource): @record_metrics @ns.doc( id="""Check if Playlist ID is occupied""", diff --git a/discovery-provider/src/tasks/entity_manager.py b/discovery-provider/src/tasks/entity_manager.py index 1b52efb4f38..a7690068bad 100644 --- a/discovery-provider/src/tasks/entity_manager.py +++ b/discovery-provider/src/tasks/entity_manager.py @@ -58,8 +58,8 @@ def entity_manager_update( users_to_fetch: Set[int] = set() # collect events by entity type and action - collect_entities_to_fetch( - update_task, entity_manager_txs, entities_to_fetch, users_to_fetch + entities_to_fetch = collect_entities_to_fetch( + update_task, entity_manager_txs, users_to_fetch ) # fetch existing playlists @@ -276,9 +276,9 @@ def delete_playlist(params: ManagePlaylistParameters): def collect_entities_to_fetch( update_task, entity_manager_txs, - entities_to_fetch: Dict[EntityType, Set[int]], users_to_fetch: Set[int], ): + entities_to_fetch: Dict[EntityType, Set[int]] = {} for tx_receipt in entity_manager_txs: entity_manager_event_tx = get_entity_manager_events_tx(update_task, tx_receipt) for event in entity_manager_event_tx: @@ -288,6 +288,7 @@ def collect_entities_to_fetch( entities_to_fetch[entity_type].add(entity_id) users_to_fetch.add(user_id) + return entities_to_fetch def fetch_existing_entities( @@ -352,7 +353,9 @@ def get_entity_manager_events_tx(update_task, tx_receipt): )().processReceipt(tx_receipt) -def process_playlist_contents(playlist_record, playlist_metadata, block_integer_time): +def process_playlist_contents( + playlist_record: Playlist, playlist_metadata, block_integer_time +): # mapping of track's metadata time to index time metadata_index_time_dict: Dict[int, Dict[int, int]] = defaultdict(dict) for track in playlist_record.playlist_contents["track_ids"]: diff --git a/discovery-provider/src/tasks/index.py b/discovery-provider/src/tasks/index.py index 6fc1510963d..34926e53a9c 100644 --- a/discovery-provider/src/tasks/index.py +++ b/discovery-provider/src/tasks/index.py @@ -326,13 +326,10 @@ def fetch_cid_metadata(db, user_factory_txs, track_factory_txs, entity_manager_t for entry in entity_manager_events_tx: event_args = entry["args"] user_id = event_args._userId - entity_type = event_args._entityType cid = event_args._metadata if not cid: continue - logger.info( - f"index.py | newcontract {txhash}, {event_args}, {entity_type}, {cid}" - ) + cids_txhash_set.add((cid, txhash)) cid_to_user_id[cid] = user_id cid_type[cid] = "playlist_data" diff --git a/discovery-provider/src/utils/cid_metadata_client.py b/discovery-provider/src/utils/cid_metadata_client.py index e50a238ee0d..814ffe5dfd7 100644 --- a/discovery-provider/src/utils/cid_metadata_client.py +++ b/discovery-provider/src/utils/cid_metadata_client.py @@ -73,7 +73,6 @@ async def _get_metadata_async(self, async_session, multihash, gateway_endpoint): f"CIDMetadataClient | Invalid URL from provided gateway addr - {url}" ) - logger.info(f"newcontract | fetching {multihash} from {url}") async with async_session.get( url, timeout=GET_METADATA_TIMEOUT_SECONDS ) as resp: @@ -161,12 +160,6 @@ async def _fetch_metadata_from_gateway_endpoints( cid, metadata_json = future_result - # # TODO add playlist type - # metadata_format = ( - # track_metadata_format - # if cid_type[cid] == "track" - # else user_metadata_format - # ) metadata_format = None if cid_type[cid] == "track": metadata_format = track_metadata_format diff --git a/libs/data-contracts/ABIs/AudiusData.json b/libs/data-contracts/ABIs/AudiusData.json deleted file mode 100644 index 02f5b9651d1..00000000000 --- a/libs/data-contracts/ABIs/AudiusData.json +++ /dev/null @@ -1,174 +0,0 @@ -{ - "contractName": "AudiusData", - "abi": [ - { - "constant": true, - "inputs": [ - { - "name": "", - "type": "bytes32" - } - ], - "name": "usedSignatures", - "outputs": [ - { - "name": "", - "type": "bool" - } - ], - "payable": false, - "stateMutability": "view", - "type": "function" - }, - { - "anonymous": false, - "inputs": [ - { - "indexed": false, - "name": "_userId", - "type": "uint256" - }, - { - "indexed": false, - "name": "_signer", - "type": "address" - }, - { - "indexed": false, - "name": "_entityType", - "type": "string" - }, - { - "indexed": false, - "name": "_entityId", - "type": "uint256" - }, - { - "indexed": false, - "name": "_metadata", - "type": "string" - }, - { - "indexed": false, - "name": "_action", - "type": "string" - } - ], - "name": "ManageEntity", - "type": "event" - }, - { - "anonymous": false, - "inputs": [ - { - "indexed": false, - "name": "_userId", - "type": "uint256" - }, - { - "indexed": false, - "name": "_isVerified", - "type": "bool" - } - ], - "name": "ManageIsVerified", - "type": "event" - }, - { - "constant": false, - "inputs": [ - { - "name": "name", - "type": "string" - }, - { - "name": "version", - "type": "string" - }, - { - "name": "chainId", - "type": "uint256" - } - ], - "name": "initialize", - "outputs": [], - "payable": false, - "stateMutability": "nonpayable", - "type": "function" - }, - { - "constant": false, - "inputs": [ - { - "name": "_verifierAddress", - "type": "address" - }, - { - "name": "_networkId", - "type": "uint256" - } - ], - "name": "initialize", - "outputs": [], - "payable": false, - "stateMutability": "nonpayable", - "type": "function" - }, - { - "constant": false, - "inputs": [ - { - "name": "_userId", - "type": "uint256" - }, - { - "name": "_entityType", - "type": "string" - }, - { - "name": "_entityId", - "type": "uint256" - }, - { - "name": "_action", - "type": "string" - }, - { - "name": "_metadata", - "type": "string" - }, - { - "name": "_nonce", - "type": "bytes32" - }, - { - "name": "_subjectSig", - "type": "bytes" - } - ], - "name": "manageEntity", - "outputs": [], - "payable": false, - "stateMutability": "nonpayable", - "type": "function" - }, - { - "constant": false, - "inputs": [ - { - "name": "_userId", - "type": "uint256" - }, - { - "name": "_isVerified", - "type": "bool" - } - ], - "name": "manageIsVerified", - "outputs": [], - "payable": false, - "stateMutability": "nonpayable", - "type": "function" - } - ] -} \ No newline at end of file diff --git a/libs/src/AudiusLibs.ts b/libs/src/AudiusLibs.ts index e38e2f7bfb0..1139c04bacc 100644 --- a/libs/src/AudiusLibs.ts +++ b/libs/src/AudiusLibs.ts @@ -45,7 +45,7 @@ import { Keypair, PublicKey } from '@solana/web3.js' import { getPlatformLocalStorage, LocalStorage } from './utils/localStorage' import type { BaseConstructorArgs } from './api/base' import type { MonitoringCallbacks } from './services/types' -import type { EntityManager } from './api/entityManager' +import { EntityManager } from './api/entityManager' type LibsIdentityServiceConfig = { @@ -147,7 +147,7 @@ export class AudiusLibs { // network chain id networkId: string, // wallet address to force use instead of the first wallet on the provided web3 - walletOverride: Nullable = null + walletOverride: Nullable = null, // entity manager address entityManagerAddress: Nullable = null ) { @@ -369,6 +369,7 @@ export class AudiusLibs { File: Nullable Rewards: Nullable Reactions: Nullable + EntityManager: Nullable preferHigherPatchForPrimary: boolean preferHigherPatchForSecondaries: boolean @@ -450,6 +451,7 @@ export class AudiusLibs { this.File = null this.Rewards = null this.Reactions = null + this.EntityManager = null this.preferHigherPatchForPrimary = preferHigherPatchForPrimary this.preferHigherPatchForSecondaries = preferHigherPatchForSecondaries diff --git a/libs/src/api/entityManager.ts b/libs/src/api/entityManager.ts index 5a5546af1da..e6bfed4f88b 100644 --- a/libs/src/api/entityManager.ts +++ b/libs/src/api/entityManager.ts @@ -1,7 +1,6 @@ import { Base, Services } from './base' import type { PlaylistMetadata } from '../services/creatorNode' -import type { Nullable } from '../utils' -import web3 from '../web3' +import { Nullable, Utils } from '../utils' export enum Action { CREATE = 'Create', @@ -32,6 +31,8 @@ export interface PlaylistOperationResponse { error: Nullable } +const { encodeHashId } = Utils + // Minimum playlist ID, intentionally higher than legacy playlist ID range const MIN_PLAYLIST_ID = 400000 // Maximum playlist ID, reflects postgres max integer value @@ -64,20 +65,23 @@ export class EntityManager extends Base { * Minimum value is artificially set to 400000 */ async getValidPlaylistId(): Promise { - // TODO: Confirm collision of ID with disc prov endpoint to account for hidden / private - let playlistId: number = this.getRandomInt(MIN_PLAYLIST_ID, MAX_PLAYLIST_ID) - let validIdFound: boolean = false - while (!validIdFound) { - const resp: any = await this.discoveryProvider.getPlaylists(1, 0, [ - playlistId - ]) - if (resp.length !== 0) { - playlistId = this.getRandomInt(MIN_PLAYLIST_ID, MAX_PLAYLIST_ID) - } else { - validIdFound = true + while (true) { + const playlistId: number = this.getRandomInt( + MIN_PLAYLIST_ID, + MAX_PLAYLIST_ID + ) + const encodedPlaylistId = encodeHashId(playlistId) + + if (encodedPlaylistId) { + const resp: any = await this.discoveryProvider.getPlaylistIsOccupied( + encodedPlaylistId + ) + + if (!resp.is_occupied) { + return playlistId + } } } - return playlistId } /** diff --git a/libs/src/services/dataContracts/EntityManagerClient.ts b/libs/src/services/dataContracts/EntityManagerClient.ts index dbb8765c79b..fdc10f91ca2 100644 --- a/libs/src/services/dataContracts/EntityManagerClient.ts +++ b/libs/src/services/dataContracts/EntityManagerClient.ts @@ -1,7 +1,10 @@ import { ContractClient } from '../contracts/ContractClient' -import * as signatureSchemas from '../../../data-contracts/signatureSchemas' +import * as signatureSchemas from '../../data-contracts/signatureSchemas' import type { Web3Manager } from '../web3Manager' +/** + * Generic management of Audius Data entities + **/ export class EntityManagerClient extends ContractClient { override web3Manager!: Web3Manager diff --git a/libs/src/services/discoveryProvider/DiscoveryProvider.ts b/libs/src/services/discoveryProvider/DiscoveryProvider.ts index 9442bbe8f71..365ae393778 100644 --- a/libs/src/services/discoveryProvider/DiscoveryProvider.ts +++ b/libs/src/services/discoveryProvider/DiscoveryProvider.ts @@ -391,6 +391,17 @@ export class DiscoveryProvider { return await this._makeRequest(req) } + /** + * Check if playlist ID is occupied + * @param playlistId encoded string of playlist ID + * @returns true if occupied + */ + async getPlaylistIsOccupied(playlistId: string): Promise { + const req = Requests.getPlaylistIsOccupied(playlistId) + const resp = await this._makeRequest(req) + return resp + } + /** * Return social feed for current user * @param filter - filter by "all", "original", or "repost" diff --git a/libs/src/services/discoveryProvider/requests.ts b/libs/src/services/discoveryProvider/requests.ts index e49693374c3..0032b66d452 100644 --- a/libs/src/services/discoveryProvider/requests.ts +++ b/libs/src/services/discoveryProvider/requests.ts @@ -241,6 +241,13 @@ export const getPlaylists = ( } } +export const getPlaylistIsOccupied = (playlistId: string) => { + return { + endpoint: 'v1/playlists', + urlParams: '/' + playlistId + '/occupied' + } +} + export const getSocialFeed = ( filter: string, limit = 100, @@ -688,10 +695,4 @@ export const verifyToken = (token: string) => { token: token } } -} - -export const getIsPlaylistIdOccupied = (playlistId: string) => { - return { - endpoint: `/v1/playlists/${playlistId}/occupied` - } -} +} \ No newline at end of file From ec993cf1be329288984df74202c7dd60c697a11b Mon Sep 17 00:00:00 2001 From: Isaac Date: Wed, 10 Aug 2022 23:44:55 +0000 Subject: [PATCH 094/101] optimistcally use random playlist ID --- discovery-provider/src/api/v1/helpers.py | 2 +- .../src/tasks/entity_manager.py | 87 ++++-- libs/src/api/entityManager.ts | 274 ++++++++++++++++-- libs/src/data-contracts/ABIs/AudiusData.json | 174 +++++++++++ .../data-contracts/ABIs/EntityManager.json | 174 +++++++++++ libs/src/utils/types.ts | 7 +- 6 files changed, 669 insertions(+), 49 deletions(-) create mode 100644 libs/src/data-contracts/ABIs/AudiusData.json create mode 100644 libs/src/data-contracts/ABIs/EntityManager.json diff --git a/discovery-provider/src/api/v1/helpers.py b/discovery-provider/src/api/v1/helpers.py index db6745b6aa5..9f50f6305f5 100644 --- a/discovery-provider/src/api/v1/helpers.py +++ b/discovery-provider/src/api/v1/helpers.py @@ -70,7 +70,7 @@ def add_playlist_added_timestamps(playlist): { "track_id": encode_int_id(track["track"]), "timestamp": track["time"], - "metadata_timestamp": track.get("metadata_time", track["time"]), + "metadata_timestamp": track.get("metadata_time"), } ) return added_timestamps diff --git a/discovery-provider/src/tasks/entity_manager.py b/discovery-provider/src/tasks/entity_manager.py index a7690068bad..7351337e831 100644 --- a/discovery-provider/src/tasks/entity_manager.py +++ b/discovery-provider/src/tasks/entity_manager.py @@ -278,7 +278,7 @@ def collect_entities_to_fetch( entity_manager_txs, users_to_fetch: Set[int], ): - entities_to_fetch: Dict[EntityType, Set[int]] = {} + entities_to_fetch: Dict[EntityType, Set[int]] = defaultdict(set) for tx_receipt in entity_manager_txs: entity_manager_event_tx = get_entity_manager_events_tx(update_task, tx_receipt) for event in entity_manager_event_tx: @@ -343,6 +343,7 @@ def copy_record(old_playlist: Playlist, block_number, event_blockhash, txhash): txhash=txhash, is_current=False, is_delete=old_playlist.is_delete, + metadata_multihash=old_playlist.metadata_multihash, ) return new_playlist @@ -356,34 +357,62 @@ def get_entity_manager_events_tx(update_task, tx_receipt): def process_playlist_contents( playlist_record: Playlist, playlist_metadata, block_integer_time ): - # mapping of track's metadata time to index time - metadata_index_time_dict: Dict[int, Dict[int, int]] = defaultdict(dict) - for track in playlist_record.playlist_contents["track_ids"]: - track_id = track["track"] - metadata_time = track["metadata_time"] - - metadata_index_time_dict[track_id][metadata_time] = track["time"] - - updated_tracks = [] - for track in playlist_metadata["playlist_contents"]["track_ids"]: - track_id = track["track"] - metadata_time = track["time"] - index_time = block_integer_time # default to current block for new tracks - - if ( - track_id in metadata_index_time_dict - and metadata_time in metadata_index_time_dict[track_id] - ): - # track exists in prev record (reorder / delete) - index_time = metadata_index_time_dict[track_id][metadata_time] - - updated_tracks.append( - { - "track": track_id, - "time": index_time, - "metadata_time": metadata_time, - } - ) + if playlist_record.metadata_multihash: + # playlist already has metadata + metadata_index_time_dict: Dict[int, Dict[int, int]] = defaultdict(dict) + for track in playlist_record.playlist_contents["track_ids"]: + track_id = track["track"] + metadata_time = track["metadata_time"] + metadata_index_time_dict[track_id][metadata_time] = track["time"] + + updated_tracks = [] + for track in playlist_metadata["playlist_contents"]["track_ids"]: + track_id = track["track"] + metadata_time = track["time"] + index_time = block_integer_time # default to current block for new tracks + + if ( + track_id in metadata_index_time_dict + and metadata_time in metadata_index_time_dict[track_id] + ): + # track exists in prev record (reorder / delete) + index_time = metadata_index_time_dict[track_id][metadata_time] + + updated_tracks.append( + { + "track": track_id, + "time": index_time, + "metadata_time": metadata_time, + } + ) + else: + # upgrade legacy playlist to include metadata + # assume metadata and indexing timestamp is the same + track_id_index_times: Set = set() + for track in playlist_record.playlist_contents["track_ids"]: + track_id = track["track"] + index_time = track["time"] + track_id_index_times.add((track_id, index_time)) + + updated_tracks = [] + for track in playlist_metadata["playlist_contents"]["track_ids"]: + track_id = track["track"] + metadata_time = track["time"] + + # use track["time"] if present in previous record else this is a new track + index_time = ( + track["time"] + if (track_id, metadata_time) in track_id_index_times + else block_integer_time + ) + updated_tracks.append( + { + "track": track_id, + "time": index_time, + "metadata_time": metadata_time, + } + ) + return {"track_ids": updated_tracks} diff --git a/libs/src/api/entityManager.ts b/libs/src/api/entityManager.ts index e6bfed4f88b..2e6d2dc9ac4 100644 --- a/libs/src/api/entityManager.ts +++ b/libs/src/api/entityManager.ts @@ -41,7 +41,7 @@ const MAX_PLAYLIST_ID = 2147483647 type PlaylistTrackId = { time: number; track: number; metadata_time?: number } type PlaylistContents = { - track_ids: Array + track_ids: PlaylistTrackId[] } /* @@ -100,6 +100,7 @@ export class EntityManager extends Base { * Create a playlist using updated data contracts flow */ async createPlaylist({ + playlistId, playlistName, trackIds, description, @@ -108,6 +109,7 @@ export class EntityManager extends Base { coverArt, logger = console }: { + playlistId: number playlistName: string trackIds: number[] description: string @@ -128,19 +130,21 @@ export class EntityManager extends Base { const userId: number = parseInt(currentUserId) const createAction = Action.CREATE const entityType = EntityType.PLAYLIST - const entityId = await this.getValidPlaylistId() this.REQUIRES(Services.CREATOR_NODE) const updatedPlaylistImage = await this.creatorNode.uploadImage( coverArt, true // square ) - const web3 = this.ethWeb3Manager.getWeb3() + const web3 = this.web3Manager.getWeb3() const currentBlockNumber = await web3.eth.getBlockNumber() const currentBlock = await web3.eth.getBlock(currentBlockNumber) - const tracks = trackIds.map(trackId => ({ track: trackId, time: currentBlock.timestamp })) + const tracks = trackIds.map((trackId) => ({ + track: trackId, + time: currentBlock.timestamp as number + })) const dirCID = updatedPlaylistImage.dirCID const metadata: PlaylistMetadata = { - playlist_id: entityId, + playlist_id: playlistId, playlist_contents: { track_ids: tracks }, playlist_name: playlistName, playlist_image_sizes_multihash: dirCID, @@ -153,14 +157,14 @@ export class EntityManager extends Base { const manageEntityResponse = await this.manageEntity({ userId: userId, entityType, - entityId, + entityId: playlistId, action: createAction, metadataMultihash }) const txReceipt = manageEntityResponse.txReceipt responseValues.blockHash = txReceipt.blockHash responseValues.blockNumber = txReceipt.blockNumber - responseValues.playlistId = entityId + responseValues.playlistId = playlistId return responseValues } catch (e) { const error = (e as Error).message @@ -211,10 +215,9 @@ export class EntityManager extends Base { /** * Update a playlist using updated data contracts flow **/ - async updatePlaylist({ + async editPlaylist({ playlistId, playlistName, - playlistContents, description, isAlbum, isPrivate, @@ -222,7 +225,6 @@ export class EntityManager extends Base { logger = console }: { playlistId: number - playlistContents: Nullable playlistName: Nullable description: Nullable isAlbum: Nullable @@ -234,7 +236,6 @@ export class EntityManager extends Base { this.getDefaultPlaylistReponseValues() try { - const currentUserId: string | null = this.userStateManager.getCurrentUserId() if (!playlistId || playlistId === undefined) { @@ -262,16 +263,22 @@ export class EntityManager extends Base { await this.discoveryProvider.getPlaylists(1, 0, [playlistId], userId) )[0] - if (playlistContents) { - playlistContents.track_ids = playlistContents.track_ids.map((track_obj: { track: any; time: any }) => ({ track: track_obj.track, time: track_obj.time })) - } - + const trackIds = playlist.playlist_contents.track_ids.map( + (trackObj: { + track: number + metadata_time?: number + time: number + }) => ({ + track: trackObj.track, + time: trackObj.metadata_time ?? trackObj.time + }) + ) const metadata: PlaylistMetadata = { playlist_id: playlistId, - playlist_contents: playlistContents ?? playlist.playlist_contents, + playlist_contents: { track_ids: trackIds }, playlist_name: playlistName ?? playlist.playlist_name, playlist_image_sizes_multihash: - dirCID || playlist.playlist_image_sizes_multihash, + dirCID ?? playlist.playlist_image_sizes_multihash, description: description ?? playlist.description, is_album: isAlbum ?? playlist.is_album, is_private: isPrivate ?? playlist.is_private @@ -296,7 +303,238 @@ export class EntityManager extends Base { return responseValues } } - + + async addPlaylistTrack({ + playlistId, + trackId, + logger = console + }: { + playlistId: number + trackId: number + logger: Console + }): Promise { + const responseValues: PlaylistOperationResponse = + this.getDefaultPlaylistReponseValues() + + try { + const currentUserId: string | null = + this.userStateManager.getCurrentUserId() + if (!playlistId || playlistId === undefined) { + responseValues.error = 'Missing current playlistId' + return responseValues + } + if (!currentUserId) { + responseValues.error = 'Missing current user ID' + return responseValues + } + const userId: number = parseInt(currentUserId) + const updateAction = Action.UPDATE + const entityType = EntityType.PLAYLIST + this.REQUIRES(Services.CREATOR_NODE) + const playlist: any = ( + await this.discoveryProvider.getPlaylists(1, 0, [playlistId], userId) + )[0] + + const updatedTrackIds = playlist.playlist_contents.track_ids.map( + (trackObj: { + track: number + metadata_time?: number + time?: number + }) => ({ + track: trackObj.track, + time: trackObj.metadata_time ?? trackObj.time + }) + ) + + const web3 = this.web3Manager.getWeb3() + const currentBlockNumber = await web3.eth.getBlockNumber() + const currentBlock = await web3.eth.getBlock(currentBlockNumber) + updatedTrackIds.push({ track: trackId, time: currentBlock.timestamp }) + + const metadata: PlaylistMetadata = { + playlist_id: playlistId, + playlist_contents: { track_ids: updatedTrackIds }, + playlist_name: playlist.playlist_name, + playlist_image_sizes_multihash: playlist.playlist_image_sizes_multihash, + description: playlist.description, + is_album: playlist.is_album, + is_private: playlist.is_private + } + const { metadataMultihash } = + await this.creatorNode.uploadPlaylistMetadata(metadata) + const resp = await this.manageEntity({ + userId, + entityType, + entityId: playlistId, + action: updateAction, + metadataMultihash + }) + const txReceipt = resp.txReceipt + responseValues.blockHash = txReceipt.blockHash + responseValues.blockNumber = txReceipt.blockNumber + responseValues.playlistId = playlistId + return responseValues + } catch (e) { + const error = (e as Error).message + responseValues.error = error + return responseValues + } + } + + async deletePlaylistTrack({ + playlistId, + trackId, + timestamp, + logger = console + }: { + playlistId: number + trackId: number + timestamp: number + logger: Console + }): Promise { + const responseValues: PlaylistOperationResponse = + this.getDefaultPlaylistReponseValues() + + try { + const currentUserId: string | null = + this.userStateManager.getCurrentUserId() + if (!playlistId || playlistId === undefined) { + responseValues.error = 'Missing current playlistId' + return responseValues + } + if (!currentUserId) { + responseValues.error = 'Missing current user ID' + return responseValues + } + const userId: number = parseInt(currentUserId) + const updateAction = Action.UPDATE + const entityType = EntityType.PLAYLIST + this.REQUIRES(Services.CREATOR_NODE) + const playlist: any = ( + await this.discoveryProvider.getPlaylists(1, 0, [playlistId], userId) + )[0] + + const updatedTrackIds = playlist.playlist_contents.track_ids.filter( + (trackObj: { track: number; metadata_time?: number; time: number }) => + (trackObj.track !== trackId && + timestamp !== trackObj.metadata_time) ?? + trackObj.time + ) + + const metadata: PlaylistMetadata = { + playlist_id: playlistId, + playlist_contents: { track_ids: updatedTrackIds }, + playlist_name: playlist.playlist_name, + playlist_image_sizes_multihash: playlist.playlist_image_sizes_multihash, + description: playlist.description, + is_album: playlist.is_album, + is_private: playlist.is_private + } + const { metadataMultihash } = + await this.creatorNode.uploadPlaylistMetadata(metadata) + const resp = await this.manageEntity({ + userId, + entityType, + entityId: playlistId, + action: updateAction, + metadataMultihash + }) + const txReceipt = resp.txReceipt + responseValues.blockHash = txReceipt.blockHash + responseValues.blockNumber = txReceipt.blockNumber + responseValues.playlistId = playlistId + return responseValues + } catch (e) { + const error = (e as Error).message + responseValues.error = error + return responseValues + } + } + + /** + * Update a playlist using updated data contracts flow + **/ + async orderPlaylist({ + playlistId, + trackIds, + logger = console + }: { + playlistId: number + trackIds: number[] + logger: Console + }): Promise { + const responseValues: PlaylistOperationResponse = + this.getDefaultPlaylistReponseValues() + + try { + const currentUserId: string | null = + this.userStateManager.getCurrentUserId() + if (!playlistId || playlistId === undefined) { + responseValues.error = 'Missing current playlistId' + return responseValues + } + if (!currentUserId) { + responseValues.error = 'Missing current user ID' + return responseValues + } + const userId: number = parseInt(currentUserId) + const updateAction = Action.UPDATE + const entityType = EntityType.PLAYLIST + this.REQUIRES(Services.CREATOR_NODE) + const playlist: any = ( + await this.discoveryProvider.getPlaylists(1, 0, [playlistId], userId) + )[0] + + let trackIdsWithTimes = [] + const trackIdTimes = {} + playlist.playlist_contents.track_ids.forEach( + (trackObj: { track: number; metadata_time?: number; time: number }) => { + const trackId = trackObj.track + const timestamp = trackObj.metadata_time ?? trackObj.time + if (trackId in trackIdTimes) { + trackIdTimes[trackId].push(timestamp) + } else { + trackIdTimes[trackId] = [timestamp] + } + } + ) + + // new tracks default to currentBlock timestamp + trackIdsWithTimes = trackIds.map((trackId: number) => ({ + track: trackId, + time: trackIdTimes[trackId].pop() + })) + const metadata: PlaylistMetadata = { + playlist_id: playlistId, + playlist_contents: { track_ids: trackIdsWithTimes }, + playlist_name: playlist.playlist_name, + playlist_image_sizes_multihash: playlist.playlist_image_sizes_multihash, + description: playlist.description, + is_album: playlist.is_album, + is_private: playlist.is_private + } + const { metadataMultihash } = + await this.creatorNode.uploadPlaylistMetadata(metadata) + + const resp = await this.manageEntity({ + userId, + entityType, + entityId: playlistId, + action: updateAction, + metadataMultihash + }) + const txReceipt = resp.txReceipt + responseValues.blockHash = txReceipt.blockHash + responseValues.blockNumber = txReceipt.blockNumber + responseValues.playlistId = playlistId + return responseValues + } catch (e) { + const error = (e as Error).message + responseValues.error = error + return responseValues + } + } + /** * Manage an entity with the updated data contract flow * Leveraged to manipulate User/Track/Playlist/+ other entities diff --git a/libs/src/data-contracts/ABIs/AudiusData.json b/libs/src/data-contracts/ABIs/AudiusData.json new file mode 100644 index 00000000000..02f5b9651d1 --- /dev/null +++ b/libs/src/data-contracts/ABIs/AudiusData.json @@ -0,0 +1,174 @@ +{ + "contractName": "AudiusData", + "abi": [ + { + "constant": true, + "inputs": [ + { + "name": "", + "type": "bytes32" + } + ], + "name": "usedSignatures", + "outputs": [ + { + "name": "", + "type": "bool" + } + ], + "payable": false, + "stateMutability": "view", + "type": "function" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": false, + "name": "_userId", + "type": "uint256" + }, + { + "indexed": false, + "name": "_signer", + "type": "address" + }, + { + "indexed": false, + "name": "_entityType", + "type": "string" + }, + { + "indexed": false, + "name": "_entityId", + "type": "uint256" + }, + { + "indexed": false, + "name": "_metadata", + "type": "string" + }, + { + "indexed": false, + "name": "_action", + "type": "string" + } + ], + "name": "ManageEntity", + "type": "event" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": false, + "name": "_userId", + "type": "uint256" + }, + { + "indexed": false, + "name": "_isVerified", + "type": "bool" + } + ], + "name": "ManageIsVerified", + "type": "event" + }, + { + "constant": false, + "inputs": [ + { + "name": "name", + "type": "string" + }, + { + "name": "version", + "type": "string" + }, + { + "name": "chainId", + "type": "uint256" + } + ], + "name": "initialize", + "outputs": [], + "payable": false, + "stateMutability": "nonpayable", + "type": "function" + }, + { + "constant": false, + "inputs": [ + { + "name": "_verifierAddress", + "type": "address" + }, + { + "name": "_networkId", + "type": "uint256" + } + ], + "name": "initialize", + "outputs": [], + "payable": false, + "stateMutability": "nonpayable", + "type": "function" + }, + { + "constant": false, + "inputs": [ + { + "name": "_userId", + "type": "uint256" + }, + { + "name": "_entityType", + "type": "string" + }, + { + "name": "_entityId", + "type": "uint256" + }, + { + "name": "_action", + "type": "string" + }, + { + "name": "_metadata", + "type": "string" + }, + { + "name": "_nonce", + "type": "bytes32" + }, + { + "name": "_subjectSig", + "type": "bytes" + } + ], + "name": "manageEntity", + "outputs": [], + "payable": false, + "stateMutability": "nonpayable", + "type": "function" + }, + { + "constant": false, + "inputs": [ + { + "name": "_userId", + "type": "uint256" + }, + { + "name": "_isVerified", + "type": "bool" + } + ], + "name": "manageIsVerified", + "outputs": [], + "payable": false, + "stateMutability": "nonpayable", + "type": "function" + } + ] +} \ No newline at end of file diff --git a/libs/src/data-contracts/ABIs/EntityManager.json b/libs/src/data-contracts/ABIs/EntityManager.json new file mode 100644 index 00000000000..f7398fb21e8 --- /dev/null +++ b/libs/src/data-contracts/ABIs/EntityManager.json @@ -0,0 +1,174 @@ +{ + "contractName": "EntityManager", + "abi": [ + { + "constant": true, + "inputs": [ + { + "name": "", + "type": "bytes32" + } + ], + "name": "usedSignatures", + "outputs": [ + { + "name": "", + "type": "bool" + } + ], + "payable": false, + "stateMutability": "view", + "type": "function" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": false, + "name": "_userId", + "type": "uint256" + }, + { + "indexed": false, + "name": "_signer", + "type": "address" + }, + { + "indexed": false, + "name": "_entityType", + "type": "string" + }, + { + "indexed": false, + "name": "_entityId", + "type": "uint256" + }, + { + "indexed": false, + "name": "_metadata", + "type": "string" + }, + { + "indexed": false, + "name": "_action", + "type": "string" + } + ], + "name": "ManageEntity", + "type": "event" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": false, + "name": "_userId", + "type": "uint256" + }, + { + "indexed": false, + "name": "_isVerified", + "type": "bool" + } + ], + "name": "ManageIsVerified", + "type": "event" + }, + { + "constant": false, + "inputs": [ + { + "name": "name", + "type": "string" + }, + { + "name": "version", + "type": "string" + }, + { + "name": "chainId", + "type": "uint256" + } + ], + "name": "initialize", + "outputs": [], + "payable": false, + "stateMutability": "nonpayable", + "type": "function" + }, + { + "constant": false, + "inputs": [ + { + "name": "_verifierAddress", + "type": "address" + }, + { + "name": "_networkId", + "type": "uint256" + } + ], + "name": "initialize", + "outputs": [], + "payable": false, + "stateMutability": "nonpayable", + "type": "function" + }, + { + "constant": false, + "inputs": [ + { + "name": "_userId", + "type": "uint256" + }, + { + "name": "_entityType", + "type": "string" + }, + { + "name": "_entityId", + "type": "uint256" + }, + { + "name": "_action", + "type": "string" + }, + { + "name": "_metadata", + "type": "string" + }, + { + "name": "_nonce", + "type": "bytes32" + }, + { + "name": "_subjectSig", + "type": "bytes" + } + ], + "name": "manageEntity", + "outputs": [], + "payable": false, + "stateMutability": "nonpayable", + "type": "function" + }, + { + "constant": false, + "inputs": [ + { + "name": "_userId", + "type": "uint256" + }, + { + "name": "_isVerified", + "type": "bool" + } + ], + "name": "manageIsVerified", + "outputs": [], + "payable": false, + "stateMutability": "nonpayable", + "type": "function" + } + ] +} \ No newline at end of file diff --git a/libs/src/utils/types.ts b/libs/src/utils/types.ts index fa3645e22de..532f3a11f23 100644 --- a/libs/src/utils/types.ts +++ b/libs/src/utils/types.ts @@ -133,7 +133,12 @@ export type CollectionMetadata = { is_delete: boolean is_private: boolean playlist_contents: { - track_ids: Array<{ time: number; track: ID; uid?: UID }> + track_ids: Array<{ + metadata_time: number + time: number + track: ID + uid?: UID + }> } tracks?: TrackMetadata[] track_count: number From e864f2c21cc43ad9384fb278a8f714b63e1d0e7b Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 11 Aug 2022 21:51:22 +0000 Subject: [PATCH 095/101] wip --- discovery-provider/src/api/v1/playlists.py | 2 +- .../src/queries/get_playlists.py | 7 +- .../src/tasks/entity_manager.py | 4 +- discovery-provider/src/tasks/index.py | 2 +- discovery-provider/src/utils/redis_cache.py | 5 + libs/src/api/entityManager.ts | 139 +++++++++--------- .../discoveryProvider/DiscoveryProvider.ts | 17 +-- .../services/discoveryProvider/requests.ts | 14 +- 8 files changed, 97 insertions(+), 93 deletions(-) diff --git a/discovery-provider/src/api/v1/playlists.py b/discovery-provider/src/api/v1/playlists.py index d58ffe27725..0073f6bcb04 100644 --- a/discovery-provider/src/api/v1/playlists.py +++ b/discovery-provider/src/api/v1/playlists.py @@ -142,8 +142,8 @@ def get(self, playlist_id): playlist_id = decode_with_abort(playlist_id, full_ns) args = current_user_parser.parse_args() current_user_id = get_current_user_id(args) - playlist = get_playlist(playlist_id, current_user_id) + logger.info(f"asdf playlist {playlist}") if playlist: tracks = get_tracks_for_playlist(playlist_id, current_user_id) playlist["tracks"] = tracks diff --git a/discovery-provider/src/queries/get_playlists.py b/discovery-provider/src/queries/get_playlists.py index e3fce4a6f0b..2fa68a20bf3 100644 --- a/discovery-provider/src/queries/get_playlists.py +++ b/discovery-provider/src/queries/get_playlists.py @@ -91,7 +91,6 @@ def get_unpopulated_playlists(): playlist_ids = list( map(lambda playlist: playlist["playlist_id"], playlists) ) - return (playlists, playlist_ids) try: @@ -99,10 +98,8 @@ def get_unpopulated_playlists(): # redis cache or via get_unpopulated_playlists key = make_cache_key(args) - (playlists, playlist_ids) = use_redis_cache( - key, UNPOPULATED_PLAYLIST_CACHE_DURATION_SEC, get_unpopulated_playlists - ) - + (playlists, playlist_ids) = get_unpopulated_playlists() + logger.info(f"asdf unpopulated playlists {playlists}") # bundle peripheral info into playlist results playlists = populate_playlist_metadata( session, diff --git a/discovery-provider/src/tasks/entity_manager.py b/discovery-provider/src/tasks/entity_manager.py index 7351337e831..31f6bd0e457 100644 --- a/discovery-provider/src/tasks/entity_manager.py +++ b/discovery-provider/src/tasks/entity_manager.py @@ -443,4 +443,6 @@ def process_playlist_data_event( playlist_record.updated_at = block_datetime playlist_record.metadata_multihash = metadata_cid - logger.info(f"index.py | AudiusData | Updated playlist record {playlist_record}") + logger.info( + f"asdf index.py | AudiusData | Updated playlist record {playlist_record}" + ) diff --git a/discovery-provider/src/tasks/index.py b/discovery-provider/src/tasks/index.py index 34926e53a9c..22871cda05d 100644 --- a/discovery-provider/src/tasks/index.py +++ b/discovery-provider/src/tasks/index.py @@ -748,7 +748,7 @@ def index_blocks(self, db, blocks_list): session.commit() metric.save_time({"scope": "commit_time"}, start_time=commit_start_time) logger.info( - f"index.py | session committed to db for block={block_number} in {time.time() - commit_start_time}s" + f"asdf index.py | session committed to db for block={block_number} in {time.time() - commit_start_time}s" ) except Exception as e: # Use 'commit' as the tx hash here. diff --git a/discovery-provider/src/utils/redis_cache.py b/discovery-provider/src/utils/redis_cache.py index 8735c0ba359..1d0f95e2ee1 100644 --- a/discovery-provider/src/utils/redis_cache.py +++ b/discovery-provider/src/utils/redis_cache.py @@ -31,6 +31,7 @@ def use_redis_cache(key, ttl_sec, work_func): redis = redis_connection.get_redis() cached_value = get_json_cached_key(redis, key) if cached_value: + logger.info(f"asdf use redis cache {cached_value}") return cached_value to_cache = work_func() set_json_cached_key(redis, key, to_cache, ttl_sec) @@ -144,6 +145,8 @@ def inner_wrap(*args, **kwargs): if cached_resp: if transform is not None: return transform(cached_resp) + logger.info(f"asdf get cached response {cached_resp}") + return cached_resp, 200 response = func(*args, **kwargs) @@ -152,6 +155,8 @@ def inner_wrap(*args, **kwargs): resp, status_code = response if status_code < 400: set_json_cached_key(redis, key, resp, ttl_sec) + logger.info(f"asdf get non cached response {resp}") + return resp, status_code set_json_cached_key(redis, key, response, ttl_sec) return transform(response) diff --git a/libs/src/api/entityManager.ts b/libs/src/api/entityManager.ts index 2e6d2dc9ac4..4d5070c096a 100644 --- a/libs/src/api/entityManager.ts +++ b/libs/src/api/entityManager.ts @@ -31,7 +31,7 @@ export interface PlaylistOperationResponse { error: Nullable } -const { encodeHashId } = Utils +const { encodeHashId, decodeHashId } = Utils // Minimum playlist ID, intentionally higher than legacy playlist ID range const MIN_PLAYLIST_ID = 400000 @@ -59,29 +59,32 @@ export class EntityManager extends Base { return Math.floor(Math.random() * (max - min) + min) } - /** - * Calculate an unoccupied playlist ID - * Maximum value is postgres integer max (2147483647) - * Minimum value is artificially set to 400000 - */ - async getValidPlaylistId(): Promise { - while (true) { - const playlistId: number = this.getRandomInt( - MIN_PLAYLIST_ID, - MAX_PLAYLIST_ID + async getFullPlaylist(playlistId: number, userId: number) { + const encodedPlaylistId = encodeHashId(playlistId) as string + const encodedUserId = encodeHashId(userId) as string + + const playlist: any = ( + await this.discoveryProvider.getFullPlaylist( + encodedPlaylistId, + encodedUserId ) - const encodedPlaylistId = encodeHashId(playlistId) + )[0] + return playlist + } - if (encodedPlaylistId) { - const resp: any = await this.discoveryProvider.getPlaylistIsOccupied( - encodedPlaylistId - ) + mapAddedTimestamps(added_timestamps: any) { + const trackIds = added_timestamps.map( + (trackObj: { + track_id: string + metadata_timestamp?: number + timestamp: number + }) => ({ + track: decodeHashId(trackObj.track_id), + time: trackObj.metadata_timestamp ?? trackObj.timestamp + }) + ) - if (!resp.is_occupied) { - return playlistId - } - } - } + return trackIds } /** @@ -121,6 +124,7 @@ export class EntityManager extends Base { const responseValues: PlaylistOperationResponse = this.getDefaultPlaylistReponseValues() try { + const currentUserId: string | null = this.userStateManager.getCurrentUserId() if (!currentUserId) { @@ -259,26 +263,15 @@ export class EntityManager extends Base { ) dirCID = updatedPlaylistImage.dirCID } - const playlist: any = ( - await this.discoveryProvider.getPlaylists(1, 0, [playlistId], userId) - )[0] - - const trackIds = playlist.playlist_contents.track_ids.map( - (trackObj: { - track: number - metadata_time?: number - time: number - }) => ({ - track: trackObj.track, - time: trackObj.metadata_time ?? trackObj.time - }) + const playlist = await this.getFullPlaylist(playlistId, userId) + const existingPlaylistTracks = this.mapAddedTimestamps( + playlist.added_timestamps ) const metadata: PlaylistMetadata = { playlist_id: playlistId, - playlist_contents: { track_ids: trackIds }, + playlist_contents: { track_ids: existingPlaylistTracks }, playlist_name: playlistName ?? playlist.playlist_name, - playlist_image_sizes_multihash: - dirCID ?? playlist.playlist_image_sizes_multihash, + playlist_image_sizes_multihash: dirCID ?? playlist.cover_art, description: description ?? playlist.description, is_album: isAlbum ?? playlist.is_album, is_private: isPrivate ?? playlist.is_private @@ -307,10 +300,12 @@ export class EntityManager extends Base { async addPlaylistTrack({ playlistId, trackId, + timestamp, logger = console }: { playlistId: number trackId: number + timestamp: number logger: Console }): Promise { const responseValues: PlaylistOperationResponse = @@ -331,31 +326,24 @@ export class EntityManager extends Base { const updateAction = Action.UPDATE const entityType = EntityType.PLAYLIST this.REQUIRES(Services.CREATOR_NODE) - const playlist: any = ( - await this.discoveryProvider.getPlaylists(1, 0, [playlistId], userId) - )[0] - - const updatedTrackIds = playlist.playlist_contents.track_ids.map( - (trackObj: { - track: number - metadata_time?: number - time?: number - }) => ({ - track: trackObj.track, - time: trackObj.metadata_time ?? trackObj.time - }) + + const playlist = await this.getFullPlaylist(playlistId, userId) + console.log('asdf existing playlist', playlist) + + const updatedPlaylistTracks = this.mapAddedTimestamps( + playlist.added_timestamps ) - const web3 = this.web3Manager.getWeb3() - const currentBlockNumber = await web3.eth.getBlockNumber() - const currentBlock = await web3.eth.getBlock(currentBlockNumber) - updatedTrackIds.push({ track: trackId, time: currentBlock.timestamp }) + updatedPlaylistTracks.push({ + track: trackId, + time: timestamp + }) const metadata: PlaylistMetadata = { playlist_id: playlistId, - playlist_contents: { track_ids: updatedTrackIds }, + playlist_contents: { track_ids: updatedPlaylistTracks }, playlist_name: playlist.playlist_name, - playlist_image_sizes_multihash: playlist.playlist_image_sizes_multihash, + playlist_image_sizes_multihash: playlist.cover_art, description: playlist.description, is_album: playlist.is_album, is_private: playlist.is_private @@ -410,11 +398,14 @@ export class EntityManager extends Base { const updateAction = Action.UPDATE const entityType = EntityType.PLAYLIST this.REQUIRES(Services.CREATOR_NODE) - const playlist: any = ( - await this.discoveryProvider.getPlaylists(1, 0, [playlistId], userId) - )[0] + const playlist = await this.getFullPlaylist(playlistId, userId) + console.log('asdf existing playlist', playlist) + + const existingPlaylistTracks = this.mapAddedTimestamps( + playlist.added_timestamps + ) - const updatedTrackIds = playlist.playlist_contents.track_ids.filter( + const updatedTrackIds = existingPlaylistTracks.filter( (trackObj: { track: number; metadata_time?: number; time: number }) => (trackObj.track !== trackId && timestamp !== trackObj.metadata_time) ?? @@ -425,7 +416,7 @@ export class EntityManager extends Base { playlist_id: playlistId, playlist_contents: { track_ids: updatedTrackIds }, playlist_name: playlist.playlist_name, - playlist_image_sizes_multihash: playlist.playlist_image_sizes_multihash, + playlist_image_sizes_multihash: playlist.cover_art, description: playlist.description, is_album: playlist.is_album, is_private: playlist.is_private @@ -481,13 +472,17 @@ export class EntityManager extends Base { const updateAction = Action.UPDATE const entityType = EntityType.PLAYLIST this.REQUIRES(Services.CREATOR_NODE) - const playlist: any = ( - await this.discoveryProvider.getPlaylists(1, 0, [playlistId], userId) - )[0] + console.log('asdf get full playlist ', { playlistId, userId }) + const playlist = await this.getFullPlaylist(playlistId, userId) + console.log('asdf existing playlist', playlist) + + const existingPlaylistTracks = this.mapAddedTimestamps( + playlist.added_timestamps + ) let trackIdsWithTimes = [] const trackIdTimes = {} - playlist.playlist_contents.track_ids.forEach( + existingPlaylistTracks.forEach( (trackObj: { track: number; metadata_time?: number; time: number }) => { const trackId = trackObj.track const timestamp = trackObj.metadata_time ?? trackObj.time @@ -508,7 +503,7 @@ export class EntityManager extends Base { playlist_id: playlistId, playlist_contents: { track_ids: trackIdsWithTimes }, playlist_name: playlist.playlist_name, - playlist_image_sizes_multihash: playlist.playlist_image_sizes_multihash, + playlist_image_sizes_multihash: playlist.cover_art, description: playlist.description, is_album: playlist.is_album, is_private: playlist.is_private @@ -557,6 +552,14 @@ export class EntityManager extends Base { let error = null let resp: any try { + console.log('asdf managing entity', { + userId, + entityType, + entityId, + action, + metadataMultihash + }) + resp = await this.contracts.EntityManagerClient?.manageEntity( userId, entityType, @@ -564,10 +567,12 @@ export class EntityManager extends Base { action, metadataMultihash ) + return { txReceipt: resp.txReceipt, error } } catch (e) { error = (e as Error).message + console.log('asdf manageEntity error', error) return { txReceipt: null, error } } } -} \ No newline at end of file +} diff --git a/libs/src/services/discoveryProvider/DiscoveryProvider.ts b/libs/src/services/discoveryProvider/DiscoveryProvider.ts index 365ae393778..adad64e8718 100644 --- a/libs/src/services/discoveryProvider/DiscoveryProvider.ts +++ b/libs/src/services/discoveryProvider/DiscoveryProvider.ts @@ -391,15 +391,9 @@ export class DiscoveryProvider { return await this._makeRequest(req) } - /** - * Check if playlist ID is occupied - * @param playlistId encoded string of playlist ID - * @returns true if occupied - */ - async getPlaylistIsOccupied(playlistId: string): Promise { - const req = Requests.getPlaylistIsOccupied(playlistId) - const resp = await this._makeRequest(req) - return resp + async getFullPlaylist(encodedPlaylistId: string, encodedUserId: string) { + const req = Requests.getFullPlaylist(encodedPlaylistId, encodedUserId) + return await this._makeRequest(req) } /** @@ -880,11 +874,6 @@ export class DiscoveryProvider { return res.map((r) => ({ ...r, amount: parseInt(r.amount) })) } - async getIsPlaylistIdOccupied(playlistId: string) { - const req = Requests.getIsPlaylistIdOccupied(playlistId) - return await this._makeRequest<{ is_occupied: boolean }>(req) - } - /* ------- INTERNAL FUNCTIONS ------- */ /** diff --git a/libs/src/services/discoveryProvider/requests.ts b/libs/src/services/discoveryProvider/requests.ts index 0032b66d452..84e11bdad1b 100644 --- a/libs/src/services/discoveryProvider/requests.ts +++ b/libs/src/services/discoveryProvider/requests.ts @@ -241,10 +241,16 @@ export const getPlaylists = ( } } -export const getPlaylistIsOccupied = (playlistId: string) => { +export const getFullPlaylist = ( + encodedPlaylistId: string, + encodedUserId: string +) => { return { - endpoint: 'v1/playlists', - urlParams: '/' + playlistId + '/occupied' + endpoint: 'v1/full/playlists', + urlParams: '/' + encodedPlaylistId, + queryParams: { + user_id: encodedUserId + } } } @@ -695,4 +701,4 @@ export const verifyToken = (token: string) => { token: token } } -} \ No newline at end of file +} From 66a6adabd5d74581796b46f33eb653bcc0445e67 Mon Sep 17 00:00:00 2001 From: isaac Date: Fri, 12 Aug 2022 23:22:32 +0000 Subject: [PATCH 096/101] edit json schema --- discovery-provider/src/api/v1/playlists.py | 1 - .../src/queries/get_playlists.py | 1 - discovery-provider/src/tasks/entity_manager.py | 2 +- discovery-provider/src/tasks/index.py | 2 +- discovery-provider/src/utils/redis_cache.py | 3 --- libs/src/api/entityManager.ts | 18 ------------------ .../schemas/playlistSchema.json | 18 ++++++++++++++++-- 7 files changed, 18 insertions(+), 27 deletions(-) diff --git a/discovery-provider/src/api/v1/playlists.py b/discovery-provider/src/api/v1/playlists.py index 0073f6bcb04..ed4688b8274 100644 --- a/discovery-provider/src/api/v1/playlists.py +++ b/discovery-provider/src/api/v1/playlists.py @@ -143,7 +143,6 @@ def get(self, playlist_id): args = current_user_parser.parse_args() current_user_id = get_current_user_id(args) playlist = get_playlist(playlist_id, current_user_id) - logger.info(f"asdf playlist {playlist}") if playlist: tracks = get_tracks_for_playlist(playlist_id, current_user_id) playlist["tracks"] = tracks diff --git a/discovery-provider/src/queries/get_playlists.py b/discovery-provider/src/queries/get_playlists.py index 2fa68a20bf3..c63974ac046 100644 --- a/discovery-provider/src/queries/get_playlists.py +++ b/discovery-provider/src/queries/get_playlists.py @@ -99,7 +99,6 @@ def get_unpopulated_playlists(): key = make_cache_key(args) (playlists, playlist_ids) = get_unpopulated_playlists() - logger.info(f"asdf unpopulated playlists {playlists}") # bundle peripheral info into playlist results playlists = populate_playlist_metadata( session, diff --git a/discovery-provider/src/tasks/entity_manager.py b/discovery-provider/src/tasks/entity_manager.py index 31f6bd0e457..6798f528a46 100644 --- a/discovery-provider/src/tasks/entity_manager.py +++ b/discovery-provider/src/tasks/entity_manager.py @@ -444,5 +444,5 @@ def process_playlist_data_event( playlist_record.metadata_multihash = metadata_cid logger.info( - f"asdf index.py | AudiusData | Updated playlist record {playlist_record}" + f"index.py | AudiusData | Updated playlist record {playlist_record}" ) diff --git a/discovery-provider/src/tasks/index.py b/discovery-provider/src/tasks/index.py index 22871cda05d..34926e53a9c 100644 --- a/discovery-provider/src/tasks/index.py +++ b/discovery-provider/src/tasks/index.py @@ -748,7 +748,7 @@ def index_blocks(self, db, blocks_list): session.commit() metric.save_time({"scope": "commit_time"}, start_time=commit_start_time) logger.info( - f"asdf index.py | session committed to db for block={block_number} in {time.time() - commit_start_time}s" + f"index.py | session committed to db for block={block_number} in {time.time() - commit_start_time}s" ) except Exception as e: # Use 'commit' as the tx hash here. diff --git a/discovery-provider/src/utils/redis_cache.py b/discovery-provider/src/utils/redis_cache.py index 1d0f95e2ee1..bd52b410ae6 100644 --- a/discovery-provider/src/utils/redis_cache.py +++ b/discovery-provider/src/utils/redis_cache.py @@ -31,7 +31,6 @@ def use_redis_cache(key, ttl_sec, work_func): redis = redis_connection.get_redis() cached_value = get_json_cached_key(redis, key) if cached_value: - logger.info(f"asdf use redis cache {cached_value}") return cached_value to_cache = work_func() set_json_cached_key(redis, key, to_cache, ttl_sec) @@ -145,7 +144,6 @@ def inner_wrap(*args, **kwargs): if cached_resp: if transform is not None: return transform(cached_resp) - logger.info(f"asdf get cached response {cached_resp}") return cached_resp, 200 @@ -155,7 +153,6 @@ def inner_wrap(*args, **kwargs): resp, status_code = response if status_code < 400: set_json_cached_key(redis, key, resp, ttl_sec) - logger.info(f"asdf get non cached response {resp}") return resp, status_code set_json_cached_key(redis, key, response, ttl_sec) diff --git a/libs/src/api/entityManager.ts b/libs/src/api/entityManager.ts index 4d5070c096a..67d5b6cc252 100644 --- a/libs/src/api/entityManager.ts +++ b/libs/src/api/entityManager.ts @@ -33,11 +33,6 @@ export interface PlaylistOperationResponse { const { encodeHashId, decodeHashId } = Utils -// Minimum playlist ID, intentionally higher than legacy playlist ID range -const MIN_PLAYLIST_ID = 400000 -// Maximum playlist ID, reflects postgres max integer value -const MAX_PLAYLIST_ID = 2147483647 - type PlaylistTrackId = { time: number; track: number; metadata_time?: number } type PlaylistContents = { @@ -328,7 +323,6 @@ export class EntityManager extends Base { this.REQUIRES(Services.CREATOR_NODE) const playlist = await this.getFullPlaylist(playlistId, userId) - console.log('asdf existing playlist', playlist) const updatedPlaylistTracks = this.mapAddedTimestamps( playlist.added_timestamps @@ -399,7 +393,6 @@ export class EntityManager extends Base { const entityType = EntityType.PLAYLIST this.REQUIRES(Services.CREATOR_NODE) const playlist = await this.getFullPlaylist(playlistId, userId) - console.log('asdf existing playlist', playlist) const existingPlaylistTracks = this.mapAddedTimestamps( playlist.added_timestamps @@ -472,9 +465,7 @@ export class EntityManager extends Base { const updateAction = Action.UPDATE const entityType = EntityType.PLAYLIST this.REQUIRES(Services.CREATOR_NODE) - console.log('asdf get full playlist ', { playlistId, userId }) const playlist = await this.getFullPlaylist(playlistId, userId) - console.log('asdf existing playlist', playlist) const existingPlaylistTracks = this.mapAddedTimestamps( playlist.added_timestamps @@ -552,14 +543,6 @@ export class EntityManager extends Base { let error = null let resp: any try { - console.log('asdf managing entity', { - userId, - entityType, - entityId, - action, - metadataMultihash - }) - resp = await this.contracts.EntityManagerClient?.manageEntity( userId, entityType, @@ -571,7 +554,6 @@ export class EntityManager extends Base { return { txReceipt: resp.txReceipt, error } } catch (e) { error = (e as Error).message - console.log('asdf manageEntity error', error) return { txReceipt: null, error } } } diff --git a/libs/src/services/schemaValidator/schemas/playlistSchema.json b/libs/src/services/schemaValidator/schemas/playlistSchema.json index 34ee4fee23b..261828673c8 100644 --- a/libs/src/services/schemaValidator/schemas/playlistSchema.json +++ b/libs/src/services/schemaValidator/schemas/playlistSchema.json @@ -11,8 +11,11 @@ "default": null }, "playlist_contents": { - "type": ["array", "null"], - "default": [] + "type": "object", + "$ref": "#/definitions/PlaylistContents", + "default": { + "track_ids": [] + } }, "playlist_name": { "type": ["string", "null"], @@ -51,6 +54,17 @@ "maxLength": 46, "pattern": "^Qm[a-zA-Z0-9]{44}$", "title": "CID" + }, + "PlaylistContents": { + "type": "object", + "additionalProperties": false, + "properties": { + "track_ids": { + "type": "array", + "default": [] + } + }, + "title": "PlaylistContents" } } } From b4b5911231ab5c7713b9af6fef363c35f7522aee Mon Sep 17 00:00:00 2001 From: isaacsolo Date: Mon, 15 Aug 2022 18:08:34 +0000 Subject: [PATCH 097/101] fix lint and nullable address --- .../src/api/v1/models/common.py | 4 +--- discovery-provider/src/app.py | 15 +++++++++------ .../src/queries/get_playlists.py | 19 ------------------- .../src/tasks/entity_manager.py | 8 ++------ 4 files changed, 12 insertions(+), 34 deletions(-) diff --git a/discovery-provider/src/api/v1/models/common.py b/discovery-provider/src/api/v1/models/common.py index ad73186c527..1cf2a3da5db 100644 --- a/discovery-provider/src/api/v1/models/common.py +++ b/discovery-provider/src/api/v1/models/common.py @@ -41,7 +41,5 @@ id_occupied = ns.model( "occupied", - { - "is_occupied": fields.Boolean(required=True) - }, + {"is_occupied": fields.Boolean(required=True)}, ) diff --git a/discovery-provider/src/app.py b/discovery-provider/src/app.py index daf577ab284..b1d4ec0d1c9 100644 --- a/discovery-provider/src/app.py +++ b/discovery-provider/src/app.py @@ -127,12 +127,15 @@ def init_contracts(): abi=abi_values["UserReplicaSetManager"]["abi"], ) - entity_manager_address = web3.toChecksumAddress( - shared_config["contracts"]["entity_manager_address"] - ) - entity_manager_inst = web3.eth.contract( - address=entity_manager_address, abi=abi_values["EntityManager"]["abi"] - ) + entity_manager_address = None + entity_manager_inst = None + if shared_config["contracts"]["entity_manager_address"]: + entity_manager_address = web3.toChecksumAddress( + shared_config["contracts"]["entity_manager_address"] + ) + entity_manager_inst = web3.eth.contract( + address=entity_manager_address, abi=abi_values["EntityManager"]["abi"] + ) contract_address_dict = { "registry": registry_address, diff --git a/discovery-provider/src/queries/get_playlists.py b/discovery-provider/src/queries/get_playlists.py index c63974ac046..c0de38bde9c 100644 --- a/discovery-provider/src/queries/get_playlists.py +++ b/discovery-provider/src/queries/get_playlists.py @@ -1,7 +1,6 @@ import logging # pylint: disable=C0302 import sqlalchemy -from flask.globals import request from sqlalchemy import desc from src import exceptions from src.models.playlists.playlist import Playlist @@ -15,26 +14,12 @@ ) from src.utils import helpers from src.utils.db_session import get_db_read_replica -from src.utils.redis_cache import extract_key, use_redis_cache logger = logging.getLogger(__name__) UNPOPULATED_PLAYLIST_CACHE_DURATION_SEC = 10 -def make_cache_key(args): - cache_keys = {"user_id": args.get("user_id"), "with_users": args.get("with_users")} - - if args.get("playlist_id"): - ids = args.get("playlist_id") - ids = map(str, ids) - ids = ",".join(ids) - cache_keys["playlist_id"] = ids - - key = extract_key(f"unpopulated-playlist:{request.path}", cache_keys.items()) - return key - - def get_playlists(args): playlists = [] current_user_id = args.get("current_user_id") @@ -94,10 +79,6 @@ def get_unpopulated_playlists(): return (playlists, playlist_ids) try: - # Get unpopulated playlists, either via - # redis cache or via get_unpopulated_playlists - key = make_cache_key(args) - (playlists, playlist_ids) = get_unpopulated_playlists() # bundle peripheral info into playlist results playlists = populate_playlist_metadata( diff --git a/discovery-provider/src/tasks/entity_manager.py b/discovery-provider/src/tasks/entity_manager.py index 6798f528a46..7be78516153 100644 --- a/discovery-provider/src/tasks/entity_manager.py +++ b/discovery-provider/src/tasks/entity_manager.py @@ -354,9 +354,7 @@ def get_entity_manager_events_tx(update_task, tx_receipt): )().processReceipt(tx_receipt) -def process_playlist_contents( - playlist_record: Playlist, playlist_metadata, block_integer_time -): +def process_playlist_contents(playlist_record, playlist_metadata, block_integer_time): if playlist_record.metadata_multihash: # playlist already has metadata metadata_index_time_dict: Dict[int, Dict[int, int]] = defaultdict(dict) @@ -443,6 +441,4 @@ def process_playlist_data_event( playlist_record.updated_at = block_datetime playlist_record.metadata_multihash = metadata_cid - logger.info( - f"index.py | AudiusData | Updated playlist record {playlist_record}" - ) + logger.info(f"index.py | AudiusData | Updated playlist record {playlist_record}") From 4f11bc38fc8c792fa28e321b919a894d93279d7d Mon Sep 17 00:00:00 2001 From: isaacsolo Date: Mon, 15 Aug 2022 20:02:42 +0000 Subject: [PATCH 098/101] add track back --- .../tasks/test_entity_manager.py | 7 +- .../tasks/test_track_entity_manager.py | 486 ++++++++++++++++++ .../src/tasks/entity_manager.py | 444 ---------------- .../tasks/entity_manager/entity_manager.py | 202 ++++++++ .../src/tasks/entity_manager/playlist.py | 249 +++++++++ .../src/tasks/entity_manager/track.py | 199 +++++++ .../src/tasks/entity_manager/types.py | 91 ++++ discovery-provider/src/tasks/index.py | 2 +- discovery-provider/src/tasks/playlists.py | 2 +- discovery-provider/src/tasks/tracks.py | 2 +- libs/src/api/entityManager.ts | 4 +- 11 files changed, 1236 insertions(+), 452 deletions(-) create mode 100644 discovery-provider/integration_tests/tasks/test_track_entity_manager.py delete mode 100644 discovery-provider/src/tasks/entity_manager.py create mode 100644 discovery-provider/src/tasks/entity_manager/entity_manager.py create mode 100644 discovery-provider/src/tasks/entity_manager/playlist.py create mode 100644 discovery-provider/src/tasks/entity_manager/track.py create mode 100644 discovery-provider/src/tasks/entity_manager/types.py diff --git a/discovery-provider/integration_tests/tasks/test_entity_manager.py b/discovery-provider/integration_tests/tasks/test_entity_manager.py index b0c2351e98e..9749bd717cb 100644 --- a/discovery-provider/integration_tests/tasks/test_entity_manager.py +++ b/discovery-provider/integration_tests/tasks/test_entity_manager.py @@ -3,7 +3,8 @@ from integration_tests.challenges.index_helpers import UpdateTask from integration_tests.utils import populate_mock_db from src.models.playlists.playlist import Playlist -from src.tasks.entity_manager import PLAYLIST_ID_OFFSET, entity_manager_update +from src.tasks.entity_manager.entity_manager import entity_manager_update +from src.tasks.entity_manager.types import PLAYLIST_ID_OFFSET from src.utils.db_session import get_db from web3 import Web3 from web3.datastructures import AttributeDict @@ -86,7 +87,7 @@ def get_events_side_effect(_, tx_receipt): return tx_receipts[tx_receipt.transactionHash.decode("utf-8")] mocker.patch( - "src.tasks.entity_manager.get_entity_manager_events_tx", + "src.tasks.entity_manager.entity_manager.get_entity_manager_events_tx", side_effect=get_events_side_effect, autospec=True, ) @@ -307,7 +308,7 @@ def get_events_side_effect(_, tx_receipt): return tx_receipts[tx_receipt.transactionHash.decode("utf-8")] mocker.patch( - "src.tasks.entity_manager.get_entity_manager_events_tx", + "src.tasks.entity_manager.entity_manager.get_entity_manager_events_tx", side_effect=get_events_side_effect, autospec=True, ) diff --git a/discovery-provider/integration_tests/tasks/test_track_entity_manager.py b/discovery-provider/integration_tests/tasks/test_track_entity_manager.py new file mode 100644 index 00000000000..f1e25bd5ab8 --- /dev/null +++ b/discovery-provider/integration_tests/tasks/test_track_entity_manager.py @@ -0,0 +1,486 @@ +from typing import List + +from integration_tests.challenges.index_helpers import UpdateTask +from integration_tests.utils import populate_mock_db +from src.challenges.challenge_event_bus import ChallengeEventBus, setup_challenge_bus +from src.models.tracks.track import Track +from src.models.tracks.track_route import TrackRoute +from src.tasks.entity_manager.entity_manager import entity_manager_update +from src.tasks.entity_manager.types import TRACK_ID_OFFSET +from src.utils.db_session import get_db +from web3 import Web3 +from web3.datastructures import AttributeDict + + +def test_index_valid_track(app, mocker): + "Tests valid batch of tracks create/update/delete actions" + + # setup db and mocked txs + with app.app_context(): + db = get_db() + web3 = Web3() + challenge_event_bus: ChallengeEventBus = setup_challenge_bus() + update_task = UpdateTask(None, web3, challenge_event_bus) + + tx_receipts = { + "CreateTrack1Tx": [ + { + "args": AttributeDict( + { + "_entityId": TRACK_ID_OFFSET, + "_entityType": "Track", + "_userId": 1, + "_action": "Create", + "_metadata": "QmCreateTrack1", + "_signer": "user1wallet", + } + ) + }, + ], + "UpdateTrack1Tx": [ + { + "args": AttributeDict( + { + "_entityId": TRACK_ID_OFFSET, + "_entityType": "Track", + "_userId": 1, + "_action": "Update", + "_metadata": "QmUpdateTrack1", + "_signer": "user1wallet", + } + ) + }, + ], + "DeleteTrack1Tx": [ + { + "args": AttributeDict( + { + "_entityId": TRACK_ID_OFFSET, + "_entityType": "Track", + "_userId": 1, + "_action": "Delete", + "_metadata": "", + "_signer": "user1wallet", + } + ) + }, + ], + "CreateTrack2Tx": [ + { + "args": AttributeDict( + { + "_entityId": TRACK_ID_OFFSET + 1, + "_entityType": "Track", + "_userId": 1, + "_action": "Create", + "_metadata": "QmCreateTrack2", + "_signer": "user1wallet", + } + ) + }, + ], + } + + entity_manager_txs = [ + AttributeDict({"transactionHash": update_task.web3.toBytes(text=tx_receipt)}) + for tx_receipt in tx_receipts + ] + + def get_events_side_effect(_, tx_receipt): + return tx_receipts[tx_receipt.transactionHash.decode("utf-8")] + + mocker.patch( + "src.tasks.entity_manager.entity_manager.get_entity_manager_events_tx", + side_effect=get_events_side_effect, + autospec=True, + ) + test_metadata = { + "QmCreateTrack1": { + "owner_id": 1, + "title": "track 1", + "length": None, + "cover_art": None, + "cover_art_sizes": "QmdxhDiRUC3zQEKqwnqksaSsSSeHiRghjwKzwoRvm77yaZ", + "tags": "realmagic,rickyreed,theroom", + "genre": "R&B/Soul", + "mood": "Empowering", + "credits_splits": None, + "created_at": "2020-07-11 08:22:15", + "create_date": None, + "updated_at": "2020-07-11 08:22:15", + "release_date": "Sat Jul 11 2020 01:19:58 GMT-0700", + "file_type": None, + "track_segments": [ + { + "duration": 6.016, + "multihash": "QmabM5svgDgcRdQZaEKSMBCpSZrrYy2y87L8Dx8EQ3T2jp", + } + ], + "has_current_user_reposted": False, + "is_current": True, + "is_unlisted": False, + "field_visibility": { + "mood": True, + "tags": True, + "genre": True, + "share": True, + "play_count": True, + "remixes": True, + }, + "remix_of": {"tracks": [{"parent_track_id": 75808}]}, + "repost_count": 12, + "save_count": 21, + "description": None, + "license": "All rights reserved", + "isrc": None, + "iswc": None, + "download": { + "cid": None, + "is_downloadable": False, + "requires_follow": False, + }, + "track_id": 77955, + "stem_of": None, + }, + "QmCreateTrack2": { + "owner_id": 1, + "title": "track 2", + "length": None, + "cover_art": None, + "cover_art_sizes": "QmQKXkVxGBbCFjcnhgxftzYDhph1CT8PJCuPEsRpffjjGC", + "tags": None, + "genre": "Electronic", + "mood": None, + "credits_splits": None, + "created_at": None, + "create_date": None, + "updated_at": None, + "release_date": None, + "file_type": None, + "track_segments": [], + "has_current_user_reposted": False, + "is_current": True, + "is_unlisted": False, + "field_visibility": { + "genre": True, + "mood": True, + "tags": True, + "share": True, + "play_count": True, + "remixes": True, + }, + "remix_of": None, + "repost_count": 0, + "save_count": 0, + "description": "", + "license": "", + "isrc": "", + "iswc": "", + }, + "QmUpdateTrack1": { + "owner_id": 1, + "title": "track 1 2", + "length": None, + "cover_art": None, + "cover_art_sizes": "QmdxhDiRUC3zQEKqwnqksaSsSSeHiRghjwKzwoRvm77yaZ", + "tags": "realmagic,rickyreed,theroom", + "genre": "R&B/Soul", + "mood": "Empowering", + "credits_splits": None, + "created_at": "2020-07-11 08:22:15", + "create_date": None, + "updated_at": "2020-07-11 08:22:15", + "release_date": "Sat Jul 11 2020 01:19:58 GMT-0700", + "file_type": None, + "track_segments": [ + { + "duration": 6.016, + "multihash": "QmabM5svgDgcRdQZaEKSMBCpSZrrYy2y87L8Dx8EQ3T2jp", + } + ], + "has_current_user_reposted": False, + "is_current": True, + "is_unlisted": False, + "field_visibility": { + "mood": True, + "tags": True, + "genre": True, + "share": True, + "play_count": True, + "remixes": True, + }, + "remix_of": {"tracks": [{"parent_track_id": 75808}]}, + "repost_count": 12, + "save_count": 21, + "description": "updated description", + "license": "All rights reserved", + "isrc": None, + "iswc": None, + "download": { + "cid": None, + "is_downloadable": False, + "requires_follow": False, + }, + "track_id": 77955, + "stem_of": None, + }, + } + + entities = { + "users": [ + {"user_id": 1, "handle": "user-1", "wallet": "user1wallet"}, + ] + } + populate_mock_db(db, entities) + + with db.scoped_session() as session: + # index transactions + entity_manager_update( + None, + update_task, + session, + entity_manager_txs, + block_number=0, + block_timestamp=1585336422, + block_hash=0, + ipfs_metadata=test_metadata, + ) + + # validate db records + all_tracks: List[Track] = session.query(Track).all() + assert len(all_tracks) == 4 + + track_1: Track = ( + session.query(Track) + .filter(Track.is_current == True, Track.track_id == TRACK_ID_OFFSET) + .first() + ) + assert track_1.description == "updated description" + assert track_1.is_delete == True + + track_2: Track = ( + session.query(Track) + .filter( + Track.is_current == True, + Track.track_id == TRACK_ID_OFFSET + 1, + ) + .first() + ) + assert track_2.title == "track 2" + assert track_2.is_delete == False + + # Check that track routes are updated appropriately + track_routes = ( + session.query(TrackRoute) + .filter(TrackRoute.track_id == TRACK_ID_OFFSET) + .all() + ) + # Should have the two routes created on track creation as well as two more for the update + assert len(track_routes) == 2, "Has two total routes after a track name update" + assert ( + len( + [ + route + for route in track_routes + if route.is_current is True and route.slug == "track-1-2" + ] + ) + == 1 + ), "The current route is 'track-1-2'" + assert ( + len([route for route in track_routes if route.is_current is False]) == 1 + ), "One route is marked non-current" + assert ( + len( + [ + route + for route in track_routes + if route.slug in ("track-1-2", "track-1") + ] + ) + == 2 + ), "Has both of the 'new-style' routes" + + +def test_index_invalid_tracks(app, mocker): + "Tests invalid batch of playlists create/update/delete actions" + + # setup db and mocked txs + with app.app_context(): + db = get_db() + web3 = Web3() + update_task = UpdateTask(None, web3, None) + + tx_receipts = { + # invalid create + "CreateTrackBelowOffset": [ + { + "args": AttributeDict( + { + "_entityId": 1, + "_entityType": "Track", + "_userId": 1, + "_action": "Create", + "_metadata": "", + "_signer": "user1wallet", + } + ) + }, + ], + "CreateTrackUserDoesNotExist": [ + { + "args": AttributeDict( + { + "_entityId": TRACK_ID_OFFSET + 1, + "_entityType": "Track", + "_userId": 2, + "_action": "Create", + "_metadata": "", + "_signer": "user1wallet", + } + ) + }, + ], + "CreateTrackUserDoesNotMatchSigner": [ + { + "args": AttributeDict( + { + "_entityId": TRACK_ID_OFFSET + 1, + "_entityType": "Track", + "_userId": 1, + "_action": "Create", + "_metadata": "", + "_signer": "InvalidWallet", + } + ) + }, + ], + "CreateTrackAlreadyExists": [ + { + "args": AttributeDict( + { + "_entityId": TRACK_ID_OFFSET, + "_entityType": "Track", + "_userId": 1, + "_action": "Create", + "_metadata": "", + "_signer": "user1wallet", + } + ) + }, + ], + # invalid updates + "UpdateTrackInvalidSigner": [ + { + "args": AttributeDict( + { + "_entityId": TRACK_ID_OFFSET, + "_entityType": "Track", + "_userId": 1, + "_action": "Update", + "_metadata": "", + "_signer": "InvalidWallet", + } + ) + }, + ], + "UpdateTrackInvalidOwner": [ + { + "args": AttributeDict( + { + "_entityId": TRACK_ID_OFFSET, + "_entityType": "Track", + "_userId": 2, + "_action": "Update", + "_metadata": "", + "_signer": "User2Wallet", + } + ) + }, + ], + # invalid deletes + "DeleteTrackInvalidSigner": [ + { + "args": AttributeDict( + { + "_entityId": TRACK_ID_OFFSET, + "_entityType": "Track", + "_userId": 1, + "_action": "Delete", + "_metadata": "", + "_signer": "InvalidWallet", + } + ) + }, + ], + "DeleteTrackDoesNotExist": [ + { + "args": AttributeDict( + { + "_entityId": TRACK_ID_OFFSET + 1, + "_entityType": "Track", + "_userId": 1, + "_action": "Update", + "_metadata": "", + "_signer": "user1wallet", + } + ) + }, + ], + "DeleteTrackInvalidOwner": [ + { + "args": AttributeDict( + { + "_entityId": TRACK_ID_OFFSET + 1, + "_entityType": "Track", + "_userId": 2, + "_action": "Update", + "_metadata": "", + "_signer": "User2Wallet", + } + ) + }, + ], + } + + entity_manager_txs = [ + AttributeDict({"transactionHash": update_task.web3.toBytes(text=tx_receipt)}) + for tx_receipt in tx_receipts + ] + + def get_events_side_effect(_, tx_receipt): + return tx_receipts[tx_receipt.transactionHash.decode("utf-8")] + + mocker.patch( + "src.tasks.entity_manager.entity_manager.get_entity_manager_events_tx", + side_effect=get_events_side_effect, + autospec=True, + ) + + entities = { + "users": [ + {"user_id": 1, "handle": "user-1", "wallet": "user1wallet"}, + {"user_id": 2, "handle": "user-1", "wallet": "User2Wallet"}, + ], + "tracks": [ + {"track_id": TRACK_ID_OFFSET, "owner_id": 1}, + ], + } + populate_mock_db(db, entities) + + with db.scoped_session() as session: + # index transactions + entity_manager_update( + None, + update_task, + session, + entity_manager_txs, + block_number=0, + block_timestamp=1585336422, + block_hash=0, + ipfs_metadata={}, + ) + + # validate db records + all_tracks: List[Track] = session.query(Track).all() + assert len(all_tracks) == 1 # no new playlists indexed diff --git a/discovery-provider/src/tasks/entity_manager.py b/discovery-provider/src/tasks/entity_manager.py deleted file mode 100644 index 7be78516153..00000000000 --- a/discovery-provider/src/tasks/entity_manager.py +++ /dev/null @@ -1,444 +0,0 @@ -import logging -from collections import defaultdict -from datetime import datetime -from enum import Enum -from typing import Any, Dict, List, Set, Tuple - -from sqlalchemy.orm.session import Session -from src.database_task import DatabaseTask -from src.models.playlists.playlist import Playlist -from src.models.users.user import User -from src.utils import helpers -from web3.datastructures import AttributeDict - -logger = logging.getLogger(__name__) - -PLAYLIST_ID_OFFSET = 400000 - - -class Action(str, Enum): - CREATE = "Create" - UPDATE = "Update" - DELETE = "Delete" - - def __str__(self) -> str: - return str.__str__(self) - - -class EntityType(str, Enum): - PLAYLIST = "Playlist" - - def __str__(self) -> str: - return str.__str__(self) - - -MANAGE_ENTITY_EVENT_TYPE = "ManageEntity" - - -def entity_manager_update( - _, - update_task: DatabaseTask, - session: Session, - entity_manager_txs, - block_number, - block_timestamp, - block_hash, - ipfs_metadata, -) -> Tuple[int, Dict[str, Set[(int)]]]: - try: - num_total_changes = 0 - event_blockhash = update_task.web3.toHex(block_hash) - - changed_entity_ids: Dict[str, Set[(int)]] = defaultdict(set) - - if not entity_manager_txs: - return num_total_changes, changed_entity_ids - - entities_to_fetch: Dict[EntityType, Set[int]] = defaultdict(set) - users_to_fetch: Set[int] = set() - - # collect events by entity type and action - entities_to_fetch = collect_entities_to_fetch( - update_task, entity_manager_txs, users_to_fetch - ) - - # fetch existing playlists - existing_playlist_id_to_playlist: Dict[int, Playlist] = fetch_existing_entities( - session, entities_to_fetch - ) - - # fetch users - existing_user_id_to_user: Dict[int, User] = fetch_users(session, users_to_fetch) - - new_playlist_records: Dict[int, List[Playlist]] = defaultdict(list) - # process in tx order and populate playlists_to_save - for tx_receipt in entity_manager_txs: - txhash = update_task.web3.toHex(tx_receipt.transactionHash) - entity_manager_event_tx = get_entity_manager_events_tx( - update_task, tx_receipt - ) - for event in entity_manager_event_tx: - params = ManagePlaylistParameters( - event, - new_playlist_records, # actions below populate these records - existing_playlist_id_to_playlist, - existing_user_id_to_user, - ipfs_metadata, - block_timestamp, - block_number, - event_blockhash, - txhash, - ) - if ( - params.action == Action.CREATE - and params.entity_type == EntityType.PLAYLIST - ): - create_playlist(params) - elif ( - params.action == Action.UPDATE - and params.entity_type == EntityType.PLAYLIST - ): - update_playlist(params) - - elif ( - params.action == Action.DELETE - and params.entity_type == EntityType.PLAYLIST - ): - delete_playlist(params) - - # compile records_to_save - new_records = [] - for playlist_records in new_playlist_records.values(): - # flip is_current to true for the last tx in each playlist - playlist_records[-1].is_current = True - new_records.extend(playlist_records) - - # insert/update all playlist records in this block - session.bulk_save_objects(new_records) - num_total_changes += len(new_records) - - except Exception as e: - logger.error(f"Exception occurred {e}", exc_info=True) - raise e - return num_total_changes, changed_entity_ids - - -class ManagePlaylistParameters: - def __init__( - self, - event: AttributeDict, - playlists_to_save: Dict[int, List[Playlist]], - existing_playlist_id_to_playlist: Dict[int, Playlist], - existing_user_id_to_user: Dict[int, User], - ipfs_metadata: Dict[str, Dict[str, Any]], - block_timestamp: int, - block_number: int, - event_blockhash: str, - txhash: str, - ): - self.user_id = helpers.get_tx_arg(event, "_userId") - self.entity_id = helpers.get_tx_arg(event, "_entityId") - self.entity_type = helpers.get_tx_arg(event, "_entityType") - self.action = helpers.get_tx_arg(event, "_action") - self.metadata_cid = helpers.get_tx_arg(event, "_metadata") - self.signer = helpers.get_tx_arg(event, "_signer") - self.block_datetime = datetime.utcfromtimestamp(block_timestamp) - self.block_integer_time = int(block_timestamp) - - self.event = event - self.ipfs_metadata = ipfs_metadata - self.existing_playlist_id_to_playlist = existing_playlist_id_to_playlist - self.block_number = block_number - self.event_blockhash = event_blockhash - self.txhash = txhash - self.existing_user_id_to_user = existing_user_id_to_user - self.playlists_to_save = playlists_to_save - - -def is_valid_playlist_tx(params: ManagePlaylistParameters): - if params.user_id not in params.existing_user_id_to_user: - # user does not exist - return False - - wallet = params.existing_user_id_to_user[params.user_id].wallet - if wallet and wallet.lower() != params.signer.lower(): - # user does not match signer - return False - - if params.entity_type != EntityType.PLAYLIST: - return False - - if params.action == Action.CREATE: - if params.entity_id in params.existing_playlist_id_to_playlist: - # playlist already exists - return False - if params.entity_id < PLAYLIST_ID_OFFSET: - return False - else: - # update / delete specific validations - if params.entity_id not in params.existing_playlist_id_to_playlist: - # playlist does not exist - return False - existing_playlist: Playlist = params.existing_playlist_id_to_playlist[ - params.entity_id - ] - if existing_playlist.playlist_owner_id != params.user_id: - # existing playlist does not match user - return False - - return True - - -def create_playlist(params: ManagePlaylistParameters): - if not is_valid_playlist_tx(params): - return - - metadata = params.ipfs_metadata[params.metadata_cid] - tracks = metadata["playlist_contents"].get("track_ids", []) - tracks_with_index_time = [] - for track in tracks: - tracks_with_index_time.append( - { - "track": track["track"], - "metadata_time": track["time"], - "time": params.block_integer_time, - } - ) - create_playlist_record = Playlist( - playlist_id=params.entity_id, - metadata_multihash=params.metadata_cid, - playlist_owner_id=params.user_id, - is_album=metadata.get("is_album", False), - description=metadata["description"], - playlist_image_multihash=metadata["playlist_image_sizes_multihash"], - playlist_image_sizes_multihash=metadata["playlist_image_sizes_multihash"], - playlist_name=metadata["playlist_name"], - is_private=metadata.get("is_private", False), - playlist_contents={"track_ids": tracks_with_index_time}, - created_at=params.block_datetime, - updated_at=params.block_datetime, - blocknumber=params.block_number, - blockhash=params.event_blockhash, - txhash=params.txhash, - is_current=False, - is_delete=False, - ) - - params.playlists_to_save[params.entity_id].append(create_playlist_record) - params.existing_playlist_id_to_playlist[params.entity_id] = create_playlist_record - - -def update_playlist(params: ManagePlaylistParameters): - if not is_valid_playlist_tx(params): - return - # TODO ignore updates on deleted playlists? - - metadata = params.ipfs_metadata[params.metadata_cid] - existing_playlist = params.existing_playlist_id_to_playlist[params.entity_id] - existing_playlist.is_current = False # invalidate - if ( - params.entity_id in params.playlists_to_save - ): # override with last updated playlist is in this block - existing_playlist = params.playlists_to_save[params.entity_id][-1] - - updated_playlist = copy_record( - existing_playlist, params.block_number, params.event_blockhash, params.txhash - ) - process_playlist_data_event( - updated_playlist, - metadata, - params.block_integer_time, - params.block_datetime, - params.metadata_cid, - ) - params.playlists_to_save[params.entity_id].append(updated_playlist) - params.existing_playlist_id_to_playlist[params.entity_id] = updated_playlist - - -def delete_playlist(params: ManagePlaylistParameters): - if not is_valid_playlist_tx(params): - return - - existing_playlist = params.existing_playlist_id_to_playlist[params.entity_id] - existing_playlist.is_current = False # invalidate old playlist - if params.entity_id in params.playlists_to_save: - # override with last updated playlist is in this block - existing_playlist = params.playlists_to_save[params.entity_id][-1] - - deleted_playlist = copy_record( - existing_playlist, params.block_number, params.event_blockhash, params.txhash - ) - deleted_playlist.is_delete = True - - params.playlists_to_save[params.entity_id].append(deleted_playlist) - - -def collect_entities_to_fetch( - update_task, - entity_manager_txs, - users_to_fetch: Set[int], -): - entities_to_fetch: Dict[EntityType, Set[int]] = defaultdict(set) - for tx_receipt in entity_manager_txs: - entity_manager_event_tx = get_entity_manager_events_tx(update_task, tx_receipt) - for event in entity_manager_event_tx: - entity_id = helpers.get_tx_arg(event, "_entityId") - entity_type = helpers.get_tx_arg(event, "_entityType") - user_id = helpers.get_tx_arg(event, "_userId") - - entities_to_fetch[entity_type].add(entity_id) - users_to_fetch.add(user_id) - return entities_to_fetch - - -def fetch_existing_entities( - session: Session, entities_to_fetch: Dict[EntityType, Set[int]] -): - existing_playlist_id_to_playlist: Dict[int, Playlist] = {} - existing_playlists_query = ( - session.query(Playlist) - .filter( - Playlist.playlist_id.in_(entities_to_fetch[EntityType.PLAYLIST]), - Playlist.is_current == True, - ) - .all() - ) - for existing_playlist in existing_playlists_query: - existing_playlist_id_to_playlist[ - existing_playlist.playlist_id - ] = existing_playlist - return existing_playlist_id_to_playlist - - -def fetch_users(session: Session, users_to_fetch: Set[int]): - existing_user_id_to_user: Dict[int, User] = {} - existing_users_query = ( - session.query(User) - .filter( - User.user_id.in_(users_to_fetch), - User.is_current == True, - ) - .all() - ) - for user in existing_users_query: - existing_user_id_to_user[user.user_id] = user - return existing_user_id_to_user - - -def copy_record(old_playlist: Playlist, block_number, event_blockhash, txhash): - new_playlist = Playlist( - playlist_id=old_playlist.playlist_id, - playlist_owner_id=old_playlist.playlist_owner_id, - is_album=old_playlist.is_album, - description=old_playlist.description, - playlist_image_multihash=old_playlist.playlist_image_multihash, - playlist_image_sizes_multihash=old_playlist.playlist_image_sizes_multihash, - playlist_name=old_playlist.playlist_name, - is_private=old_playlist.is_private, - playlist_contents=old_playlist.playlist_contents, - created_at=old_playlist.created_at, - updated_at=old_playlist.updated_at, - blocknumber=block_number, - blockhash=event_blockhash, - txhash=txhash, - is_current=False, - is_delete=old_playlist.is_delete, - metadata_multihash=old_playlist.metadata_multihash, - ) - return new_playlist - - -def get_entity_manager_events_tx(update_task, tx_receipt): - return getattr( - update_task.entity_manager_contract.events, MANAGE_ENTITY_EVENT_TYPE - )().processReceipt(tx_receipt) - - -def process_playlist_contents(playlist_record, playlist_metadata, block_integer_time): - if playlist_record.metadata_multihash: - # playlist already has metadata - metadata_index_time_dict: Dict[int, Dict[int, int]] = defaultdict(dict) - for track in playlist_record.playlist_contents["track_ids"]: - track_id = track["track"] - metadata_time = track["metadata_time"] - metadata_index_time_dict[track_id][metadata_time] = track["time"] - - updated_tracks = [] - for track in playlist_metadata["playlist_contents"]["track_ids"]: - track_id = track["track"] - metadata_time = track["time"] - index_time = block_integer_time # default to current block for new tracks - - if ( - track_id in metadata_index_time_dict - and metadata_time in metadata_index_time_dict[track_id] - ): - # track exists in prev record (reorder / delete) - index_time = metadata_index_time_dict[track_id][metadata_time] - - updated_tracks.append( - { - "track": track_id, - "time": index_time, - "metadata_time": metadata_time, - } - ) - else: - # upgrade legacy playlist to include metadata - # assume metadata and indexing timestamp is the same - track_id_index_times: Set = set() - for track in playlist_record.playlist_contents["track_ids"]: - track_id = track["track"] - index_time = track["time"] - track_id_index_times.add((track_id, index_time)) - - updated_tracks = [] - for track in playlist_metadata["playlist_contents"]["track_ids"]: - track_id = track["track"] - metadata_time = track["time"] - - # use track["time"] if present in previous record else this is a new track - index_time = ( - track["time"] - if (track_id, metadata_time) in track_id_index_times - else block_integer_time - ) - updated_tracks.append( - { - "track": track_id, - "time": index_time, - "metadata_time": metadata_time, - } - ) - - return {"track_ids": updated_tracks} - - -def process_playlist_data_event( - playlist_record: Playlist, - playlist_metadata, - block_integer_time, - block_datetime, - metadata_cid, -): - playlist_record.is_album = ( - playlist_metadata["is_album"] if "is_album" in playlist_metadata else False - ) - playlist_record.description = playlist_metadata["description"] - playlist_record.playlist_image_multihash = playlist_metadata[ - "playlist_image_sizes_multihash" - ] - playlist_record.playlist_image_sizes_multihash = playlist_metadata[ - "playlist_image_sizes_multihash" - ] - playlist_record.playlist_name = playlist_metadata["playlist_name"] - playlist_record.is_private = ( - playlist_metadata["is_private"] if "is_private" in playlist_metadata else False - ) - playlist_record.playlist_contents = process_playlist_contents( - playlist_record, playlist_metadata, block_integer_time - ) - playlist_record.updated_at = block_datetime - playlist_record.metadata_multihash = metadata_cid - - logger.info(f"index.py | AudiusData | Updated playlist record {playlist_record}") diff --git a/discovery-provider/src/tasks/entity_manager/entity_manager.py b/discovery-provider/src/tasks/entity_manager/entity_manager.py new file mode 100644 index 00000000000..00ace42cd9d --- /dev/null +++ b/discovery-provider/src/tasks/entity_manager/entity_manager.py @@ -0,0 +1,202 @@ +import logging +from collections import defaultdict +from typing import Any, Dict, List, Set, Tuple + +from sqlalchemy.orm.session import Session +from src.challenges.challenge_event_bus import ChallengeEventBus +from src.database_task import DatabaseTask +from src.models.playlists.playlist import Playlist +from src.models.tracks.track import Track +from src.models.tracks.track_route import TrackRoute +from src.models.users.user import User +from src.tasks.entity_manager.playlist import ( + create_playlist, + delete_playlist, + update_playlist, +) +from src.tasks.entity_manager.track import create_track, delete_track, update_track +from src.tasks.entity_manager.types import ( + MANAGE_ENTITY_EVENT_TYPE, + Action, + EntityType, + ExistingRecordDict, + ManageEntityParameters, + RecordDict, +) +from src.utils import helpers + +logger = logging.getLogger(__name__) + + +def entity_manager_update( + _, # main indexing task + update_task: DatabaseTask, + session: Session, + entity_manager_txs: List[Any], + block_number: int, + block_timestamp, + block_hash: str, + ipfs_metadata: Dict, +) -> Tuple[int, Dict[str, Set[(int)]]]: + try: + challenge_bus: ChallengeEventBus = update_task.challenge_event_bus + + num_total_changes = 0 + event_blockhash = update_task.web3.toHex(block_hash) + + changed_entity_ids: Dict[str, Set[(int)]] = defaultdict(set) + + if not entity_manager_txs: + return num_total_changes, changed_entity_ids + + # collect events by entity type and action + entities_to_fetch = collect_entities_to_fetch(update_task, entity_manager_txs) + + # fetch existing playlists + existing_records: ExistingRecordDict = fetch_existing_entities( + session, entities_to_fetch + ) + + new_records: RecordDict = { + "playlists": defaultdict(list), + "tracks": defaultdict(list), + } + + pending_track_routes: List[TrackRoute] = [] + + # process in tx order and populate playlists_to_save + for tx_receipt in entity_manager_txs: + txhash = update_task.web3.toHex(tx_receipt.transactionHash) + entity_manager_event_tx = get_entity_manager_events_tx( + update_task, tx_receipt + ) + for event in entity_manager_event_tx: + params = ManageEntityParameters( + session, + challenge_bus, + event, + new_records, # actions below populate these records + existing_records, + pending_track_routes, + ipfs_metadata, + block_timestamp, + block_number, + event_blockhash, + txhash, + ) + if ( + params.action == Action.CREATE + and params.entity_type == EntityType.PLAYLIST + ): + create_playlist(params) + elif ( + params.action == Action.UPDATE + and params.entity_type == EntityType.PLAYLIST + ): + update_playlist(params) + elif ( + params.action == Action.DELETE + and params.entity_type == EntityType.PLAYLIST + ): + delete_playlist(params) + elif ( + params.action == Action.CREATE + and params.entity_type == EntityType.TRACK + ): + create_track(params) + elif ( + params.action == Action.UPDATE + and params.entity_type == EntityType.TRACK + ): + update_track(params) + + elif ( + params.action == Action.DELETE + and params.entity_type == EntityType.TRACK + ): + delete_track(params) + + logger.info(new_records) + + # compile records_to_save + records_to_save = [] + for playlist_records in new_records["playlists"].values(): + # flip is_current to true for the last tx in each playlist + playlist_records[-1].is_current = True + records_to_save.extend(playlist_records) + + for track_records in new_records["tracks"].values(): + # flip is_current to true for the last tx in each playlist + track_records[-1].is_current = True + records_to_save.extend(track_records) + + # insert/update all tracks, playlist records in this block + session.bulk_save_objects(records_to_save) + num_total_changes += len(records_to_save) + + except Exception as e: + logger.error(f"Exception occurred {e}", exc_info=True) + raise e + return num_total_changes, changed_entity_ids + + +def collect_entities_to_fetch( + update_task, + entity_manager_txs, +): + entities_to_fetch: Dict[EntityType, Set[int]] = defaultdict(set) + for tx_receipt in entity_manager_txs: + entity_manager_event_tx = get_entity_manager_events_tx(update_task, tx_receipt) + for event in entity_manager_event_tx: + entity_id = helpers.get_tx_arg(event, "_entityId") + entity_type = helpers.get_tx_arg(event, "_entityType") + user_id = helpers.get_tx_arg(event, "_userId") + + entities_to_fetch[entity_type].add(entity_id) + entities_to_fetch[EntityType.USER].add(user_id) + return entities_to_fetch + + +def fetch_existing_entities( + session: Session, entities_to_fetch: Dict[EntityType, Set[int]] +): + existing_entities = {} + playlists: List[Playlist] = ( + session.query(Playlist) + .filter( + Playlist.playlist_id.in_(entities_to_fetch[EntityType.PLAYLIST]), + Playlist.is_current == True, + ) + .all() + ) + existing_entities["playlists"] = { + playlist.playlist_id: playlist for playlist in playlists + } + + tracks: List[Track] = ( + session.query(Track) + .filter( + Track.track_id.in_(entities_to_fetch[EntityType.TRACK]), + Track.is_current == True, + ) + .all() + ) + existing_entities["tracks"] = {track.track_id: track for track in tracks} + + users: List[User] = ( + session.query(User) + .filter( + User.user_id.in_(entities_to_fetch[EntityType.USER]), + User.is_current == True, + ) + .all() + ) + existing_entities["users"] = {user.user_id: user for user in users} + + return existing_entities + + +def get_entity_manager_events_tx(update_task, tx_receipt): + return getattr( + update_task.entity_manager_contract.events, MANAGE_ENTITY_EVENT_TYPE + )().processReceipt(tx_receipt) diff --git a/discovery-provider/src/tasks/entity_manager/playlist.py b/discovery-provider/src/tasks/entity_manager/playlist.py new file mode 100644 index 00000000000..1b2e595b51a --- /dev/null +++ b/discovery-provider/src/tasks/entity_manager/playlist.py @@ -0,0 +1,249 @@ +import logging +from collections import defaultdict +from typing import Dict, Set + +from src.models.playlists.playlist import Playlist +from src.tasks.entity_manager.types import ( + PLAYLIST_ID_OFFSET, + Action, + EntityType, + ManageEntityParameters, +) + +logger = logging.getLogger(__name__) + + +def is_valid_playlist_tx(params: ManageEntityParameters): + user_id = params.user_id + playlist_id = params.entity_id + if user_id not in params.existing_records["users"]: + # user does not exist + return False + + wallet = params.existing_records["users"][user_id].wallet + if wallet and wallet.lower() != params.signer.lower(): + # user does not match signer + return False + + if params.entity_type != EntityType.PLAYLIST: + return False + + if params.action == Action.CREATE: + if playlist_id in params.existing_records["playlists"]: + # playlist already exists + return False + if playlist_id < PLAYLIST_ID_OFFSET: + return False + else: + # update / delete specific validations + if playlist_id not in params.existing_records["playlists"]: + # playlist does not exist + return False + existing_playlist: Playlist = params.existing_records["playlists"][playlist_id] + if existing_playlist.playlist_owner_id != user_id: + # existing playlist does not match user + return False + + return True + + +def create_playlist(params: ManageEntityParameters): + if not is_valid_playlist_tx(params): + return + + playlist_id = params.entity_id + metadata = params.ipfs_metadata[params.metadata_cid] + tracks = metadata["playlist_contents"].get("track_ids", []) + tracks_with_index_time = [] + for track in tracks: + tracks_with_index_time.append( + { + "track": track["track"], + "metadata_time": track["time"], + "time": params.block_integer_time, + } + ) + logger.info("making model") + create_playlist_record = Playlist( + playlist_id=playlist_id, + metadata_multihash=params.metadata_cid, + playlist_owner_id=params.user_id, + is_album=metadata.get("is_album", False), + description=metadata["description"], + playlist_image_multihash=metadata["playlist_image_sizes_multihash"], + playlist_image_sizes_multihash=metadata["playlist_image_sizes_multihash"], + playlist_name=metadata["playlist_name"], + is_private=metadata.get("is_private", False), + playlist_contents={"track_ids": tracks_with_index_time}, + created_at=params.block_datetime, + updated_at=params.block_datetime, + blocknumber=params.block_number, + blockhash=params.event_blockhash, + txhash=params.txhash, + is_current=False, + is_delete=False, + ) + logger.info("adding playlist info") + logger.info("adding playlist info") + logger.info("adding playlist info") + params.add_playlist_record(playlist_id, create_playlist_record) + + +def update_playlist(params: ManageEntityParameters): + if not is_valid_playlist_tx(params): + return + # TODO ignore updates on deleted playlists? + + playlist_id = params.entity_id + metadata = params.ipfs_metadata[params.metadata_cid] + existing_playlist = params.existing_records["playlists"][playlist_id] + existing_playlist.is_current = False # invalidate + if ( + playlist_id in params.new_records["playlists"] + ): # override with last updated playlist is in this block + existing_playlist = params.new_records["playlists"][playlist_id][-1] + + updated_playlist = copy_record( + existing_playlist, params.block_number, params.event_blockhash, params.txhash + ) + process_playlist_data_event( + updated_playlist, + metadata, + params.block_integer_time, + params.block_datetime, + params.metadata_cid, + ) + params.add_playlist_record(playlist_id, updated_playlist) + + +def delete_playlist(params: ManageEntityParameters): + if not is_valid_playlist_tx(params): + return + + existing_playlist = params.existing_records["playlists"][params.entity_id] + existing_playlist.is_current = False # invalidate old playlist + if params.entity_id in params.new_records["playlists"]: + # override with last updated playlist is in this block + existing_playlist = params.new_records["playlists"][params.entity_id][-1] + + deleted_playlist = copy_record( + existing_playlist, params.block_number, params.event_blockhash, params.txhash + ) + deleted_playlist.is_delete = True + + params.new_records["playlists"][params.entity_id].append(deleted_playlist) + + +def copy_record(old_playlist: Playlist, block_number, event_blockhash, txhash): + new_playlist = Playlist( + playlist_id=old_playlist.playlist_id, + playlist_owner_id=old_playlist.playlist_owner_id, + is_album=old_playlist.is_album, + description=old_playlist.description, + playlist_image_multihash=old_playlist.playlist_image_multihash, + playlist_image_sizes_multihash=old_playlist.playlist_image_sizes_multihash, + playlist_name=old_playlist.playlist_name, + is_private=old_playlist.is_private, + playlist_contents=old_playlist.playlist_contents, + created_at=old_playlist.created_at, + updated_at=old_playlist.updated_at, + blocknumber=block_number, + blockhash=event_blockhash, + txhash=txhash, + is_current=False, + is_delete=old_playlist.is_delete, + metadata_multihash=old_playlist.metadata_multihash, + ) + return new_playlist + + +def process_playlist_contents( + playlist_record: Playlist, playlist_metadata, block_integer_time +): + if playlist_record.metadata_multihash: + # playlist already has metadata + metadata_index_time_dict: Dict[int, Dict[int, int]] = defaultdict(dict) + for track in playlist_record.playlist_contents["track_ids"]: + track_id = track["track"] + metadata_time = track["metadata_time"] + metadata_index_time_dict[track_id][metadata_time] = track["time"] + + updated_tracks = [] + for track in playlist_metadata["playlist_contents"]["track_ids"]: + track_id = track["track"] + metadata_time = track["time"] + index_time = block_integer_time # default to current block for new tracks + + if ( + track_id in metadata_index_time_dict + and metadata_time in metadata_index_time_dict[track_id] + ): + # track exists in prev record (reorder / delete) + index_time = metadata_index_time_dict[track_id][metadata_time] + + updated_tracks.append( + { + "track": track_id, + "time": index_time, + "metadata_time": metadata_time, + } + ) + else: + # upgrade legacy playlist to include metadata + # assume metadata and indexing timestamp is the same + track_id_index_times: Set = set() + for track in playlist_record.playlist_contents["track_ids"]: + track_id = track["track"] + index_time = track["time"] + track_id_index_times.add((track_id, index_time)) + + updated_tracks = [] + for track in playlist_metadata["playlist_contents"]["track_ids"]: + track_id = track["track"] + metadata_time = track["time"] + + # use track["time"] if present in previous record else this is a new track + index_time = ( + track["time"] + if (track_id, metadata_time) in track_id_index_times + else block_integer_time + ) + updated_tracks.append( + { + "track": track_id, + "time": index_time, + "metadata_time": metadata_time, + } + ) + + return {"track_ids": updated_tracks} + + +def process_playlist_data_event( + playlist_record: Playlist, + playlist_metadata, + block_integer_time, + block_datetime, + metadata_cid, +): + playlist_record.is_album = ( + playlist_metadata["is_album"] if "is_album" in playlist_metadata else False + ) + playlist_record.description = playlist_metadata["description"] + playlist_record.playlist_image_multihash = playlist_metadata[ + "playlist_image_sizes_multihash" + ] + playlist_record.playlist_image_sizes_multihash = playlist_metadata[ + "playlist_image_sizes_multihash" + ] + playlist_record.playlist_name = playlist_metadata["playlist_name"] + playlist_record.is_private = ( + playlist_metadata["is_private"] if "is_private" in playlist_metadata else False + ) + playlist_record.playlist_contents = process_playlist_contents( + playlist_record, playlist_metadata, block_integer_time + ) + playlist_record.updated_at = block_datetime + playlist_record.metadata_multihash = metadata_cid + + logger.info(f"index.py | AudiusData | Updated playlist record {playlist_record}") diff --git a/discovery-provider/src/tasks/entity_manager/track.py b/discovery-provider/src/tasks/entity_manager/track.py new file mode 100644 index 00000000000..c1939405ef4 --- /dev/null +++ b/discovery-provider/src/tasks/entity_manager/track.py @@ -0,0 +1,199 @@ +import logging +from typing import Dict + +from src.models.tracks.track import Track +from src.models.users.user import User +from src.tasks.entity_manager.types import ( + TRACK_ID_OFFSET, + Action, + EntityType, + ManageEntityParameters, +) +from src.tasks.tracks import ( + dispatch_challenge_track_upload, + populate_track_record_metadata, + update_remixes_table, + update_stems_table, + update_track_routes_table, +) + +logger = logging.getLogger(__name__) + + +def is_valid_track_tx(params: ManageEntityParameters): + user_id = params.user_id + track_id = params.entity_id + if user_id not in params.existing_records["users"]: + # user does not exist + return False + + wallet = params.existing_records["users"][user_id].wallet + if wallet and wallet.lower() != params.signer.lower(): + # user does not match signer + return False + + if params.entity_type != EntityType.TRACK: + return False + + if params.action == Action.CREATE: + if track_id in params.existing_records["tracks"]: + # playlist already exists + return False + if track_id < TRACK_ID_OFFSET: + return False + else: + # update / delete specific validations + if track_id not in params.existing_records["tracks"]: + # playlist does not exist + return False + existing_track: Track = params.existing_records["tracks"][track_id] + if existing_track.owner_id != params.user_id: + # existing playlist does not match user + return False + + return True + + +def copy_track_record( + old_track: Track, block_number: int, event_blockhash: str, txhash: str +): + return Track( + track_id=old_track.track_id, + owner_id=old_track.owner_id, + title=old_track.title, + length=old_track.length, + cover_art=old_track.cover_art, + tags=old_track.tags, + genre=old_track.genre, + mood=old_track.mood, + credits_splits=old_track.credits_splits, + create_date=old_track.create_date, + release_date=old_track.release_date, + file_type=old_track.file_type, + metadata_multihash=old_track.metadata_multihash, + track_segments=old_track.track_segments, + description=old_track.description, + isrc=old_track.isrc, + iswc=old_track.iswc, + license=old_track.license, + cover_art_sizes=old_track.cover_art_sizes, + download=old_track.download, + is_unlisted=old_track.is_unlisted, + field_visibility=old_track.field_visibility, + route_id=old_track.route_id, + stem_of=old_track.stem_of, + remix_of=old_track.remix_of, + is_available=old_track.is_available, + is_delete=old_track.is_delete, + created_at=old_track.created_at, + updated_at=old_track.updated_at, + blocknumber=block_number, + blockhash=event_blockhash, + txhash=txhash, + is_current=False, + ) + + +def get_handle(params: ManageEntityParameters): + # TODO: get the track owner user handle + handle = ( + params.session.query(User.handle) + .filter(User.user_id == params.user_id, User.is_current == True) + .first() + )[0] + if not handle: + logger.error("missing track user in entity manager handle track") + return handle + + +def update_track_record(params: ManageEntityParameters, track: Track, metadata: Dict): + handle = get_handle(params) + populate_track_record_metadata(track, metadata, handle) + track.metadata_multihash = params.metadata_cid + # if cover_art CID is of a dir, store under _sizes field instead + if track.cover_art: + logger.info( + f"index.py | tracks.py | Processing track cover art {track.cover_art}" + ) + track.cover_art_sizes = track.cover_art + track.cover_art = None + + +def create_track(params: ManageEntityParameters): + if not is_valid_track_tx(params): + return + + track_id = params.entity_id + owner_id = params.user_id + track_metadata = params.ipfs_metadata[params.metadata_cid] + + track_record = Track( + track_id=track_id, + owner_id=owner_id, + txhash=params.txhash, + blockhash=params.event_blockhash, + blocknumber=params.block_number, + created_at=params.block_datetime, + updated_at=params.block_datetime, + is_delete=False, + ) + + update_track_routes_table( + params.session, track_record, track_metadata, params.pending_track_routes + ) + + update_track_record(params, track_record, track_metadata) + + update_stems_table(params.session, track_record, track_metadata) + update_remixes_table(params.session, track_record, track_metadata) + dispatch_challenge_track_upload( + params.challenge_bus, params.block_number, track_record + ) + + params.add_track_record(track_id, track_record) + + +def update_track(params: ManageEntityParameters): + if not is_valid_track_tx(params): + return + # TODO ignore updates on deleted playlists? + + track_metadata = params.ipfs_metadata[params.metadata_cid] + track_id = params.entity_id + existing_track = params.existing_records["tracks"][track_id] + existing_track.is_current = False # invalidate + if ( + track_id in params.new_records["tracks"] + ): # override with last updated playlist is in this block + existing_track = params.new_records["tracks"][track_id][-1] + + updated_track = copy_track_record( + existing_track, params.block_number, params.event_blockhash, params.txhash + ) + + update_track_routes_table( + params.session, updated_track, track_metadata, params.pending_track_routes + ) + update_track_record(params, updated_track, track_metadata) + update_remixes_table(params.session, updated_track, track_metadata) + + params.add_track_record(track_id, updated_track) + + +def delete_track(params: ManageEntityParameters): + if not is_valid_track_tx(params): + return + + track_id = params.entity_id + existing_track = params.existing_records["tracks"][track_id] + existing_track.is_current = False # invalidate old playlist + if params.entity_id in params.new_records["tracks"]: + # override with last updated playlist is in this block + existing_track = params.new_records["tracks"][params.entity_id][-1] + + deleted_track = copy_track_record( + existing_track, params.block_number, params.event_blockhash, params.txhash + ) + deleted_track.is_delete = True + + params.add_track_record(track_id, deleted_track) diff --git a/discovery-provider/src/tasks/entity_manager/types.py b/discovery-provider/src/tasks/entity_manager/types.py new file mode 100644 index 00000000000..410990ef010 --- /dev/null +++ b/discovery-provider/src/tasks/entity_manager/types.py @@ -0,0 +1,91 @@ +from datetime import datetime +from enum import Enum +from typing import Dict, List, TypedDict + +from src.challenges.challenge_event_bus import ChallengeEventBus +from src.models.playlists.playlist import Playlist +from src.models.tracks.track import Track +from src.models.tracks.track_route import TrackRoute +from src.models.users.user import User +from src.utils import helpers +from web3.datastructures import AttributeDict + +PLAYLIST_ID_OFFSET = 400_000 +TRACK_ID_OFFSET = 1_000_000 + + +class Action(str, Enum): + CREATE = "Create" + UPDATE = "Update" + DELETE = "Delete" + + def __str__(self) -> str: + return str.__str__(self) + + +class EntityType(str, Enum): + PLAYLIST = "Playlist" + TRACK = "Track" + USER = "User" + + def __str__(self) -> str: + return str.__str__(self) + + +class RecordDict(TypedDict): + playlists: Dict[int, List[Playlist]] + tracks: Dict[int, List[Track]] + + +class ExistingRecordDict(TypedDict): + playlists: Dict[int, Playlist] + tracks: Dict[int, Track] + users: Dict[int, User] + + +MANAGE_ENTITY_EVENT_TYPE = "ManageEntity" + + +class ManageEntityParameters: + def __init__( + self, + session, + challenge_bus: ChallengeEventBus, + event: AttributeDict, + new_records: RecordDict, + existing_records: ExistingRecordDict, + pending_track_routes: List[TrackRoute], + ipfs_metadata: Dict[str, Dict[str, Dict]], + block_timestamp: int, + block_number: int, + event_blockhash: str, + txhash: str, + ): + self.user_id = helpers.get_tx_arg(event, "_userId") + self.entity_id = helpers.get_tx_arg(event, "_entityId") + self.entity_type = helpers.get_tx_arg(event, "_entityType") + self.action = helpers.get_tx_arg(event, "_action") + self.metadata_cid = helpers.get_tx_arg(event, "_metadata") + self.signer = helpers.get_tx_arg(event, "_signer") + self.block_datetime = datetime.utcfromtimestamp(block_timestamp) + self.block_integer_time = int(block_timestamp) + + self.session = session + self.challenge_bus = challenge_bus + self.pending_track_routes = pending_track_routes + + self.event = event + self.ipfs_metadata = ipfs_metadata + self.block_number = block_number + self.event_blockhash = event_blockhash + self.txhash = txhash + self.new_records = new_records + self.existing_records = existing_records + + def add_playlist_record(self, playlist_id: int, playlist: Playlist): + self.new_records["playlists"][playlist_id].append(playlist) + self.existing_records["playlists"][playlist_id] = playlist + + def add_track_record(self, track_id: int, track: Track): + self.new_records["tracks"][track_id].append(track) + self.existing_records["tracks"][track_id] = track diff --git a/discovery-provider/src/tasks/index.py b/discovery-provider/src/tasks/index.py index 34926e53a9c..20049c46782 100644 --- a/discovery-provider/src/tasks/index.py +++ b/discovery-provider/src/tasks/index.py @@ -31,7 +31,7 @@ ) from src.queries.skipped_transactions import add_network_level_skipped_transaction from src.tasks.celery_app import celery -from src.tasks.entity_manager import entity_manager_update +from src.tasks.entity_manager.entity_manager import entity_manager_update from src.tasks.playlists import playlist_state_update from src.tasks.social_features import social_feature_state_update from src.tasks.sort_block_transactions import sort_block_transactions diff --git a/discovery-provider/src/tasks/playlists.py b/discovery-provider/src/tasks/playlists.py index eb6d70b3376..9fb77f8aaf0 100644 --- a/discovery-provider/src/tasks/playlists.py +++ b/discovery-provider/src/tasks/playlists.py @@ -7,7 +7,7 @@ from src.database_task import DatabaseTask from src.models.playlists.playlist import Playlist from src.queries.skipped_transactions import add_node_level_skipped_transaction -from src.tasks.entity_manager import PLAYLIST_ID_OFFSET +from src.tasks.entity_manager.types import PLAYLIST_ID_OFFSET from src.utils import helpers from src.utils.indexing_errors import EntityMissingRequiredFieldError, IndexingError from src.utils.model_nullable_validator import all_required_fields_present diff --git a/discovery-provider/src/tasks/tracks.py b/discovery-provider/src/tasks/tracks.py index 4850583df8a..fdbd93f01a0 100644 --- a/discovery-provider/src/tasks/tracks.py +++ b/discovery-provider/src/tasks/tracks.py @@ -558,7 +558,7 @@ def is_valid_json_field(metadata, field): def populate_track_record_metadata(track_record, track_metadata, handle): track_record.title = track_metadata["title"] - track_record.length = track_metadata["length"] or 0 + track_record.length = track_metadata.get("length", 0) track_record.cover_art = track_metadata["cover_art"] if track_metadata["cover_art_sizes"]: track_record.cover_art = track_metadata["cover_art_sizes"] diff --git a/libs/src/api/entityManager.ts b/libs/src/api/entityManager.ts index 67d5b6cc252..b03182566f1 100644 --- a/libs/src/api/entityManager.ts +++ b/libs/src/api/entityManager.ts @@ -67,8 +67,8 @@ export class EntityManager extends Base { return playlist } - mapAddedTimestamps(added_timestamps: any) { - const trackIds = added_timestamps.map( + mapAddedTimestamps(addedTimestamps: any) { + const trackIds = addedTimestamps.map( (trackObj: { track_id: string metadata_timestamp?: number From 9d920d34087a79e4ba6c7a02b2ebfe1d13bedf96 Mon Sep 17 00:00:00 2001 From: isaacsolo Date: Mon, 15 Aug 2022 21:06:27 +0000 Subject: [PATCH 099/101] resolve type error --- .../integration_tests/tasks/test_entity_manager.py | 2 +- .../integration_tests/tasks/test_track_entity_manager.py | 2 +- .../src/tasks/entity_manager/entity_manager.py | 4 ++-- discovery-provider/src/tasks/entity_manager/playlist.py | 6 ++---- discovery-provider/src/tasks/entity_manager/track.py | 2 +- .../src/tasks/entity_manager/{types.py => utils.py} | 0 discovery-provider/src/tasks/playlists.py | 2 +- 7 files changed, 8 insertions(+), 10 deletions(-) rename discovery-provider/src/tasks/entity_manager/{types.py => utils.py} (100%) diff --git a/discovery-provider/integration_tests/tasks/test_entity_manager.py b/discovery-provider/integration_tests/tasks/test_entity_manager.py index 9749bd717cb..43725391260 100644 --- a/discovery-provider/integration_tests/tasks/test_entity_manager.py +++ b/discovery-provider/integration_tests/tasks/test_entity_manager.py @@ -4,7 +4,7 @@ from integration_tests.utils import populate_mock_db from src.models.playlists.playlist import Playlist from src.tasks.entity_manager.entity_manager import entity_manager_update -from src.tasks.entity_manager.types import PLAYLIST_ID_OFFSET +from src.tasks.entity_manager.utils import PLAYLIST_ID_OFFSET from src.utils.db_session import get_db from web3 import Web3 from web3.datastructures import AttributeDict diff --git a/discovery-provider/integration_tests/tasks/test_track_entity_manager.py b/discovery-provider/integration_tests/tasks/test_track_entity_manager.py index f1e25bd5ab8..2d05abcc2cf 100644 --- a/discovery-provider/integration_tests/tasks/test_track_entity_manager.py +++ b/discovery-provider/integration_tests/tasks/test_track_entity_manager.py @@ -6,7 +6,7 @@ from src.models.tracks.track import Track from src.models.tracks.track_route import TrackRoute from src.tasks.entity_manager.entity_manager import entity_manager_update -from src.tasks.entity_manager.types import TRACK_ID_OFFSET +from src.tasks.entity_manager.utils import TRACK_ID_OFFSET from src.utils.db_session import get_db from web3 import Web3 from web3.datastructures import AttributeDict diff --git a/discovery-provider/src/tasks/entity_manager/entity_manager.py b/discovery-provider/src/tasks/entity_manager/entity_manager.py index 00ace42cd9d..34b9642a91c 100644 --- a/discovery-provider/src/tasks/entity_manager/entity_manager.py +++ b/discovery-provider/src/tasks/entity_manager/entity_manager.py @@ -15,7 +15,7 @@ update_playlist, ) from src.tasks.entity_manager.track import create_track, delete_track, update_track -from src.tasks.entity_manager.types import ( +from src.tasks.entity_manager.utils import ( MANAGE_ENTITY_EVENT_TYPE, Action, EntityType, @@ -160,7 +160,7 @@ def collect_entities_to_fetch( def fetch_existing_entities( session: Session, entities_to_fetch: Dict[EntityType, Set[int]] ): - existing_entities = {} + existing_entities: ExistingRecordDict = {} playlists: List[Playlist] = ( session.query(Playlist) .filter( diff --git a/discovery-provider/src/tasks/entity_manager/playlist.py b/discovery-provider/src/tasks/entity_manager/playlist.py index 1b2e595b51a..656178466a5 100644 --- a/discovery-provider/src/tasks/entity_manager/playlist.py +++ b/discovery-provider/src/tasks/entity_manager/playlist.py @@ -3,7 +3,7 @@ from typing import Dict, Set from src.models.playlists.playlist import Playlist -from src.tasks.entity_manager.types import ( +from src.tasks.entity_manager.utils import ( PLAYLIST_ID_OFFSET, Action, EntityType, @@ -157,9 +157,7 @@ def copy_record(old_playlist: Playlist, block_number, event_blockhash, txhash): return new_playlist -def process_playlist_contents( - playlist_record: Playlist, playlist_metadata, block_integer_time -): +def process_playlist_contents(playlist_record, playlist_metadata, block_integer_time): if playlist_record.metadata_multihash: # playlist already has metadata metadata_index_time_dict: Dict[int, Dict[int, int]] = defaultdict(dict) diff --git a/discovery-provider/src/tasks/entity_manager/track.py b/discovery-provider/src/tasks/entity_manager/track.py index c1939405ef4..b44efb2c508 100644 --- a/discovery-provider/src/tasks/entity_manager/track.py +++ b/discovery-provider/src/tasks/entity_manager/track.py @@ -3,7 +3,7 @@ from src.models.tracks.track import Track from src.models.users.user import User -from src.tasks.entity_manager.types import ( +from src.tasks.entity_manager.utils import ( TRACK_ID_OFFSET, Action, EntityType, diff --git a/discovery-provider/src/tasks/entity_manager/types.py b/discovery-provider/src/tasks/entity_manager/utils.py similarity index 100% rename from discovery-provider/src/tasks/entity_manager/types.py rename to discovery-provider/src/tasks/entity_manager/utils.py diff --git a/discovery-provider/src/tasks/playlists.py b/discovery-provider/src/tasks/playlists.py index 9fb77f8aaf0..10b7d4fa7fc 100644 --- a/discovery-provider/src/tasks/playlists.py +++ b/discovery-provider/src/tasks/playlists.py @@ -7,7 +7,7 @@ from src.database_task import DatabaseTask from src.models.playlists.playlist import Playlist from src.queries.skipped_transactions import add_node_level_skipped_transaction -from src.tasks.entity_manager.types import PLAYLIST_ID_OFFSET +from src.tasks.entity_manager.utils import PLAYLIST_ID_OFFSET from src.utils import helpers from src.utils.indexing_errors import EntityMissingRequiredFieldError, IndexingError from src.utils.model_nullable_validator import all_required_fields_present From 873ed7d9a7a1267e5fcb2ed96b9f38fd3ed90cc4 Mon Sep 17 00:00:00 2001 From: isaacsolo Date: Mon, 15 Aug 2022 21:19:43 +0000 Subject: [PATCH 100/101] lint libs --- libs/src/AudiusLibs.ts | 5 ++--- libs/src/api/entityManager.ts | 7 ------ libs/src/services/creatorNode/CreatorNode.ts | 23 ++++++-------------- 3 files changed, 9 insertions(+), 26 deletions(-) diff --git a/libs/src/AudiusLibs.ts b/libs/src/AudiusLibs.ts index 1139c04bacc..6d55e232268 100644 --- a/libs/src/AudiusLibs.ts +++ b/libs/src/AudiusLibs.ts @@ -47,7 +47,6 @@ import type { BaseConstructorArgs } from './api/base' import type { MonitoringCallbacks } from './services/types' import { EntityManager } from './api/entityManager' - type LibsIdentityServiceConfig = { url: string useHedgehogLocalStorage: boolean @@ -203,7 +202,7 @@ export class AudiusLibs { /** * Configures an eth web3 */ - static configEthWeb3 ( + static configEthWeb3( tokenAddress, registryAddress, providers, @@ -603,7 +602,7 @@ export class AudiusLibs { if (this.creatorNodeConfig) { const currentUser = this.userStateManager.getCurrentUser() const creatorNodeEndpoint = currentUser - ? CreatorNode.getPrimary(currentUser.creator_node_endpoint) || + ? CreatorNode.getPrimary(currentUser.creator_node_endpoint) ?? this.creatorNodeConfig.fallbackUrl : this.creatorNodeConfig.fallbackUrl diff --git a/libs/src/api/entityManager.ts b/libs/src/api/entityManager.ts index b03182566f1..c271b990b4c 100644 --- a/libs/src/api/entityManager.ts +++ b/libs/src/api/entityManager.ts @@ -33,12 +33,6 @@ export interface PlaylistOperationResponse { const { encodeHashId, decodeHashId } = Utils -type PlaylistTrackId = { time: number; track: number; metadata_time?: number } - -type PlaylistContents = { - track_ids: PlaylistTrackId[] -} - /* API surface for updated data contract interactions. Provides simplified entity management in a generic fashion @@ -119,7 +113,6 @@ export class EntityManager extends Base { const responseValues: PlaylistOperationResponse = this.getDefaultPlaylistReponseValues() try { - const currentUserId: string | null = this.userStateManager.getCurrentUserId() if (!currentUserId) { diff --git a/libs/src/services/creatorNode/CreatorNode.ts b/libs/src/services/creatorNode/CreatorNode.ts index cdf25383c14..776eed901c6 100644 --- a/libs/src/services/creatorNode/CreatorNode.ts +++ b/libs/src/services/creatorNode/CreatorNode.ts @@ -8,7 +8,6 @@ import { playlistSchemaType, Schemas } from '../schemaValidator/SchemaValidator' -import type { Nullable } from '../../utils' import type { Web3Manager } from '../web3Manager' import type { CurrentUser, UserStateManager } from '../../userStateManager' import type { MonitoringCallbacks } from '../types' @@ -19,19 +18,10 @@ const MAX_TRACK_TRANSCODE_TIMEOUT = 3600000 // 1 hour const POLL_STATUS_INTERVAL = 3000 // 3s const BROWSER_SESSION_REFRESH_TIMEOUT = 604800000 // 1 week -type Metadata = { - track_segments: unknown - download?: { - is_downloadable: boolean - cid: string - } - cover_art_sizes: string -} - -type PlaylistTrackId = { time: number; track: number; } +type PlaylistTrackId = { time: number; track: number } type PlaylistContents = { - track_ids: Array + track_ids: PlaylistTrackId[] } export type PlaylistMetadata = { @@ -345,7 +335,7 @@ export class CreatorNode { trackFile: File, coverArtFile: File, metadata: TrackMetadata, - onProgress: ProgressCB = () => { } + onProgress: ProgressCB = () => {} ) { let loadedImageBytes = 0 let loadedTrackBytes = 0 @@ -990,7 +980,7 @@ export class CreatorNode { async _uploadFile( file: File, route: string, - onProgress: ProgressCB = () => { }, + onProgress: ProgressCB = () => {}, extraFormDataOptions: Record = {}, retries = 2, timeoutMs: number | null = null @@ -1100,8 +1090,9 @@ export class CreatorNode { if ('response' in e && e.response?.data?.error) { const cnRequestID = e.response.headers['cn-request-id'] // cnRequestID will be the same as requestId if it receives the X-Request-ID header - const errMessage = `Server returned error: [${e.response.status.toString()}] [${e.response.data.error - }] for request: [${cnRequestID}, ${requestId}]` + const errMessage = `Server returned error: [${e.response.status.toString()}] [${ + e.response.data.error + }] for request: [${cnRequestID}, ${requestId}]` console.error(errMessage) throw new Error(errMessage) From a28d6b9137ae01ca82ff150757c9345dac8f1aa5 Mon Sep 17 00:00:00 2001 From: isaacsolo Date: Mon, 15 Aug 2022 21:22:02 +0000 Subject: [PATCH 101/101] default to 0 track length --- .../src/solana/audius_data_transaction_handlers.py | 2 +- discovery-provider/src/tasks/tracks.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/discovery-provider/src/solana/audius_data_transaction_handlers.py b/discovery-provider/src/solana/audius_data_transaction_handlers.py index 4a74ff05d11..ccbcba083d1 100644 --- a/discovery-provider/src/solana/audius_data_transaction_handlers.py +++ b/discovery-provider/src/solana/audius_data_transaction_handlers.py @@ -511,7 +511,7 @@ def update_track_model_metadata( session: Session, track_record: Track, track_metadata: Dict ): track_record.title = track_metadata["title"] - track_record.length = track_metadata["length"] or 0 + track_record.length = track_metadata.get("length", 0) or 0 track_record.cover_art_sizes = track_metadata["cover_art_sizes"] if track_metadata["cover_art"]: track_record.cover_art_sizes = track_record.cover_art diff --git a/discovery-provider/src/tasks/tracks.py b/discovery-provider/src/tasks/tracks.py index fdbd93f01a0..a6a93b70201 100644 --- a/discovery-provider/src/tasks/tracks.py +++ b/discovery-provider/src/tasks/tracks.py @@ -558,7 +558,7 @@ def is_valid_json_field(metadata, field): def populate_track_record_metadata(track_record, track_metadata, handle): track_record.title = track_metadata["title"] - track_record.length = track_metadata.get("length", 0) + track_record.length = track_metadata.get("length", 0) or 0 track_record.cover_art = track_metadata["cover_art"] if track_metadata["cover_art_sizes"]: track_record.cover_art = track_metadata["cover_art_sizes"]