From 5d7982b3d3fb46a698c0a0d2e6ff0afccedb9119 Mon Sep 17 00:00:00 2001 From: Michael Piazza Date: Wed, 11 May 2022 19:05:31 +0000 Subject: [PATCH 01/13] Identity: change column names --- .../20220509223231-change_reaction_columns.js | 16 ++++++++++++++ identity-service/src/models/reaction.js | 6 ++--- identity-service/src/routes/reactions.js | 22 +++++++++---------- 3 files changed, 30 insertions(+), 14 deletions(-) create mode 100644 identity-service/sequelize/migrations/20220509223231-change_reaction_columns.js diff --git a/identity-service/sequelize/migrations/20220509223231-change_reaction_columns.js b/identity-service/sequelize/migrations/20220509223231-change_reaction_columns.js new file mode 100644 index 00000000000..a49b650b713 --- /dev/null +++ b/identity-service/sequelize/migrations/20220509223231-change_reaction_columns.js @@ -0,0 +1,16 @@ +'use strict'; + +module.exports = { + up: (queryInterface, Sequelize) => { + return queryInterface.renameColumn('Reactions', 'entityId', 'reactedTo') + .then(() => queryInterface.renameColumn('Reactions', 'entityType', 'reactionType')) + .then(() => queryInterface.renameColumn('Reactions', 'reaction', 'reactionValue')) + }, + + down: (queryInterface, Sequelize) => { + return queryInterface.renameColumn('Reactions', 'reactedTo', 'entityId') + .then(() => queryInterface.renameColumn('Reactions', 'reactionType', 'entityType')) + .then(() => queryInterface.renameColumn('Reactions', 'reactionValue', 'reaction')) + } +}; + diff --git a/identity-service/src/models/reaction.js b/identity-service/src/models/reaction.js index c4193690736..d6d0822a618 100644 --- a/identity-service/src/models/reaction.js +++ b/identity-service/src/models/reaction.js @@ -11,7 +11,7 @@ module.exports = (sequelize, DataTypes) => { type: DataTypes.INTEGER, allowNull: false }, - reaction: { + reactionValue: { type: DataTypes.INTEGER, allowNull: false }, @@ -19,11 +19,11 @@ module.exports = (sequelize, DataTypes) => { type: DataTypes.STRING, allowNull: false }, - entityId: { + reactedTo: { type: DataTypes.STRING, allowNull: false }, - entityType: { + reactionType: { type: DataTypes.STRING, allowNull: false }, diff --git a/identity-service/src/routes/reactions.js b/identity-service/src/routes/reactions.js index 162de742647..1db8b213204 100644 --- a/identity-service/src/routes/reactions.js +++ b/identity-service/src/routes/reactions.js @@ -5,17 +5,17 @@ const { logger } = require('../logging') const MAX_REACTIONS_PER_FETCH = 100 -const handleReaction = async ({ senderWallet, entityType, entityId, libs, reaction }) => { +const handleReaction = async ({ senderWallet, reactionType, reactedTo, libs, reactionValue }) => { const { solanaWeb3Manager } = libs const currentSlot = await solanaWeb3Manager.getSlot() const now = Date.now() await models.Reactions.create({ slot: currentSlot, - reaction, + reactionValue, senderWallet, - entityId, - entityType, + reactionType, + reactedTo, createdAt: now, updatedAt: now }) @@ -34,26 +34,26 @@ const getReactions = async ({ startIndex, limit }) => { module.exports = function (app) { /** - * POST a new reaction, represented by a numeric ID (reaction) and entityId (entity being reacted upon) + * POST a new reaction, represented by a numeric ID (reaction) and reactedTo (entity being reacted upon) */ app.post('/reactions', authMiddleware, handleResponse(async (req, res, next) => { // Validation const senderWallet = req.user.walletAddress - const { entityId, reaction } = req.body + const { reactedTo,reactionValue } = req.body - if (!(senderWallet && entityId && reaction)) return errorResponseBadRequest(`Missing argument: ${JSON.stringify({ senderWallet, entityId, reaction })}`) + if (!(senderWallet && reactedTo && reactionValue)) return errorResponseBadRequest(`Missing argument: ${JSON.stringify({ senderWallet, reactedTo, reactionValue })}`) - const parsedReaction = parseInt(reaction) + const parsedReaction = parseInt(reactionValue) if (!parsedReaction) return errorResponseBadRequest('Invalid reaction type') const libs = req.app.get('audiusLibs') try { - logger.info(`Creating reaction ${reaction} for entityId: ${entityId}`) - await handleReaction({ senderWallet, entityId, reaction: parsedReaction, entityType: 'tip', libs }) + logger.info(`Creating reaction ${reactionValue} for reactedTo: ${reactedTo}`) + await handleReaction({ senderWallet, reactedTo, reactionValue: parsedReaction, reactionType: 'tip', libs }) return successResponse() } catch (e) { - logger.error(`Caught error trying to create reaction ${reaction} for id: ${entityId}: ${e}`) + logger.error(`Caught error trying to create reaction ${reactionValue} for id: ${reactedTo}: ${e}`) return errorResponseServerError('Something went wrong') } })) From 264b8eab631b219b445945f688a216c3df12f655 Mon Sep 17 00:00:00 2001 From: Michael Piazza Date: Wed, 11 May 2022 19:06:20 +0000 Subject: [PATCH 02/13] Discovery: change column names --- ...2067242dd5_change_reaction_column_names.py | 39 +++++++++++++++++++ discovery-provider/src/models/reaction.py | 6 +-- .../src/tasks/index_reactions.py | 12 +++--- 3 files changed, 48 insertions(+), 9 deletions(-) create mode 100644 discovery-provider/alembic/versions/0d2067242dd5_change_reaction_column_names.py diff --git a/discovery-provider/alembic/versions/0d2067242dd5_change_reaction_column_names.py b/discovery-provider/alembic/versions/0d2067242dd5_change_reaction_column_names.py new file mode 100644 index 00000000000..8dd432bf814 --- /dev/null +++ b/discovery-provider/alembic/versions/0d2067242dd5_change_reaction_column_names.py @@ -0,0 +1,39 @@ +"""Change reaction column names + +Revision ID: 0d2067242dd5 +Revises: 35198266d709 +Create Date: 2022-05-09 22:03:16.838837 + +""" +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "0d2067242dd5" +down_revision = "35198266d709" +branch_labels = None +depends_on = None + + +def upgrade(): + op.alter_column("reactions", "entity_id", new_column_name="reacted_to") + op.alter_column("reactions", "entity_type", new_column_name="reaction_type") + op.alter_column("reactions", "reaction", new_column_name="reaction_value") + op.create_index( + op.f("ix_reactions_reacted_to_reaction_type"), + "reactions", + ["reacted_to", "reaction_type"], + unique=False, + info={"if_not_exists": True}, + ) + + +def downgrade(): + op.alter_column("reactions", "reacted_to", new_column_name="entity_id") + op.alter_column("reactions", "reaction_type", new_column_name="entity_type") + op.alter_column("reactions", "reaction_value", new_column_name="reaction") + op.drop_index( + op.f("ix_reactions_reacted_to_reaction_type"), + table_name="reaction", + info={"if_not_exists": True}, + ) diff --git a/discovery-provider/src/models/reaction.py b/discovery-provider/src/models/reaction.py index f515d20273e..2c405606c07 100644 --- a/discovery-provider/src/models/reaction.py +++ b/discovery-provider/src/models/reaction.py @@ -8,9 +8,9 @@ class Reaction(Base, RepresentableMixin): id = Column(Integer, nullable=False, primary_key=True) slot = Column(Integer, nullable=False) - reaction = Column(Integer, nullable=False) + reaction_value = Column(Integer, nullable=False) sender_wallet = Column(String, nullable=False) - entity_type = Column(String, nullable=False) - entity_id = Column(String, nullable=False) + reaction_type = Column(String, nullable=False) + reacted_to = Column(String, nullable=False) timestamp = Column(DateTime, nullable=False) tx_signature = Column(String, nullable=True) diff --git a/discovery-provider/src/tasks/index_reactions.py b/discovery-provider/src/tasks/index_reactions.py index 989d25714d2..3c4f4157fed 100644 --- a/discovery-provider/src/tasks/index_reactions.py +++ b/discovery-provider/src/tasks/index_reactions.py @@ -32,10 +32,10 @@ class ReactionResponse(TypedDict): id: int slot: int - reaction: int + reactionValue: int senderWallet: str - entityId: str - entityType: str + reactedTo: str + reactionType: str createdAt: str updatedAt: str @@ -46,10 +46,10 @@ def reaction_dict_to_model(reaction: ReactionResponse) -> Union[Reaction, None]: try: reaction_model = Reaction( slot=reaction["slot"], - reaction=reaction["reaction"], + reaction_value=reaction["reactionValue"], sender_wallet=reaction["senderWallet"], - entity_type=reaction["entityType"], - entity_id=reaction["entityId"], + reaction_type=reaction["reactionType"], + reacted_to=reaction["reactedTo"], timestamp=cast(datetime, reaction["createdAt"]), tx_signature=None, # no tx_signature for now ) From afa8e6a2d8372ae7c6c2b66401dec1825180b11f Mon Sep 17 00:00:00 2001 From: Michael Piazza Date: Wed, 11 May 2022 19:07:34 +0000 Subject: [PATCH 03/13] Discovery: reaction notifications --- .../src/queries/notifications.py | 23 ++++++++++++++++++- .../src/queries/response_name_constants.py | 7 ++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/discovery-provider/src/queries/notifications.py b/discovery-provider/src/queries/notifications.py index cda2f80b6aa..8d93d6147dd 100644 --- a/discovery-provider/src/queries/notifications.py +++ b/discovery-provider/src/queries/notifications.py @@ -23,6 +23,7 @@ UserBalanceChange, UserTip, ) +from src.models.reaction import Reaction from src.queries import response_name_constants as const from src.queries.get_prev_track_entries import get_prev_track_entries from src.queries.query_helpers import ( @@ -1030,7 +1031,7 @@ def solana_notifications(): Response - Json object w/ the following fields notifications: Array of notifications of shape: - type: 'ChallengeReward' | 'MilestoneListen' | 'SupporterRankUp' + type: 'ChallengeReward' | 'MilestoneListen' | 'SupporterRankUp' | 'Reaction' slot: (int) slot number of notification initiator: (int) the user id that caused this notification metadata?: (any) additional information about the notification @@ -1158,10 +1159,30 @@ def solana_notifications(): } ) + reaction_results: List[Reaction] = ( + session.query(Reaction) + .filter(Reaction.slot >= min_slot_number, Reaction.slot <= max_slot_number) + .all() + ) + reactions = [] + for reaction in reaction_results: + reactions.append( + { + const.solana_notification_type: const.solana_notification_type_reaction, + const.solana_notification_slot: reaction.slot, + const.notification_initiator: reaction.sender_wallet, + const.solana_notification_metadata: { + const.solana_notification_reaction_type: reaction.reaction_type, + const.solana_notification_reaction_reaction_value: reaction.reaction_value, + const.solana_notification_reaction_reacted_to: reaction.reacted_to, + }, + } + ) notifications_unsorted.extend(challenge_reward_notifications) notifications_unsorted.extend(track_listen_milestones) notifications_unsorted.extend(supporter_rank_ups) notifications_unsorted.extend(tips) + notifications_unsorted.extend(reactions) # Final sort sorted_notifications = sorted( diff --git a/discovery-provider/src/queries/response_name_constants.py b/discovery-provider/src/queries/response_name_constants.py index 7129f1a3767..e6615a7208b 100644 --- a/discovery-provider/src/queries/response_name_constants.py +++ b/discovery-provider/src/queries/response_name_constants.py @@ -125,5 +125,12 @@ solana_notification_tip_rank = "rank" solana_notification_tip_amount = "amount" +solana_notification_type_reaction = "reaction" +solana_notification_reaction_type = "reaction_type" +solana_notification_reaction_type_tip = "tip" +solana_notification_reaction_reacted_to = "reacted_to" +solana_notification_reaction_reaction_value = "reaction_value" + + # Trending owner_follower_count = "owner_follower_count" From ab0226be7e501ef72dd68a7a0c53d9dd58a3854c Mon Sep 17 00:00:00 2001 From: Michael Piazza Date: Wed, 11 May 2022 19:12:14 +0000 Subject: [PATCH 04/13] Discovery: bulk get reactions endpoint --- discovery-provider/src/api/v1/api.py | 2 + discovery-provider/src/api/v1/helpers.py | 5 ++ .../src/api/v1/models/reactions.py | 13 +++++ discovery-provider/src/api/v1/reactions.py | 47 +++++++++++++++++++ discovery-provider/src/queries/reactions.py | 32 +++++++++++++ 5 files changed, 99 insertions(+) create mode 100644 discovery-provider/src/api/v1/models/reactions.py create mode 100644 discovery-provider/src/api/v1/reactions.py create mode 100644 discovery-provider/src/queries/reactions.py diff --git a/discovery-provider/src/api/v1/api.py b/discovery-provider/src/api/v1/api.py index 4189b8701f4..dcca6574325 100644 --- a/discovery-provider/src/api/v1/api.py +++ b/discovery-provider/src/api/v1/api.py @@ -6,6 +6,7 @@ from src.api.v1.models.users import ns as models_ns from src.api.v1.playlists import full_ns as full_playlists_ns from src.api.v1.playlists import ns as playlists_ns +from src.api.v1.reactions import ns as reactions_ns from src.api.v1.resolve import ns as resolve_ns from src.api.v1.search import full_ns as full_search_ns from src.api.v1.tips import full_ns as full_tips_ns @@ -37,6 +38,7 @@ def specs_url(self): api_v1.add_namespace(tips_ns) api_v1.add_namespace(metrics_ns) api_v1.add_namespace(resolve_ns) +api_v1.add_namespace(reactions_ns) bp_full = Blueprint("api_v1_full", __name__, url_prefix="/v1/full") api_v1_full = ApiWithHTTPS(bp_full, version="1.0") diff --git a/discovery-provider/src/api/v1/helpers.py b/discovery-provider/src/api/v1/helpers.py index 3a13e09e643..fb764e1f32c 100644 --- a/discovery-provider/src/api/v1/helpers.py +++ b/discovery-provider/src/api/v1/helpers.py @@ -8,6 +8,7 @@ from src.queries.get_challenges import ChallengeResponse from src.queries.get_support_for_user import SupportResponse from src.queries.get_undisbursed_challenges import UndisbursedChallengeResponse +from src.queries.reactions import ReactionResponse from src.utils.config import shared_config from src.utils.helpers import decode_string_id, encode_int_id @@ -304,6 +305,10 @@ def extend_supporting(support: SupportResponse): "receiver": extend_user(support["user"]), } +def extend_reaction(reaction: ReactionResponse): + new_reaction = reaction.copy() + new_reaction["sender_user_id"] = encode_int_id(reaction["sender_user_id"]) + return new_reaction def extend_tip(tip): new_tip = tip.copy() diff --git a/discovery-provider/src/api/v1/models/reactions.py b/discovery-provider/src/api/v1/models/reactions.py new file mode 100644 index 00000000000..2113b822947 --- /dev/null +++ b/discovery-provider/src/api/v1/models/reactions.py @@ -0,0 +1,13 @@ +from flask_restx import fields + +from .common import ns + +reaction = ns.model( + "reaction", + { + "reaction_value": fields.String(required=True), + "reaction_type": fields.String(required=True), + "sender_user_id": fields.String(required=True), + "reacted_to": fields.String(required=True), + }, +) diff --git a/discovery-provider/src/api/v1/reactions.py b/discovery-provider/src/api/v1/reactions.py new file mode 100644 index 00000000000..39213f80bb0 --- /dev/null +++ b/discovery-provider/src/api/v1/reactions.py @@ -0,0 +1,47 @@ + +from flask_restx import Namespace, Resource, fields, marshal_with, reqparse +from src.api.v1.helpers import DescriptiveArgument, make_response, success_response +from src.api.v1.models.reactions import reaction +from src.utils.db_session import get_db_read_replica +from src.utils.helpers import extend_reaction +from src.utils.redis_cache import cache +from src.utils.redis_metrics import record_metrics +from src.queries.reactions import get_reactions + +ns = Namespace("reactions", description="Reaction related operations") + +get_reactions_parser = reqparse.RequestParser(argument_class=DescriptiveArgument) +get_reactions_parser.add_argument( + "type", required=True, description="The type of reactions for which to query." +) +get_reactions_parser.add_argument( + "tx_id", + required=True, + action="append", + description="The `reacted_to` transaction id(s) of the reactions in question.", +) + +get_reactions_response = make_response( + "reactions", ns, fields.List(fields.Nested(reaction)) +) + + +@ns.route("", doc=False) +class BulkReactions(Resource): + @record_metrics + @ns.doc( + id="Bulk get Reactions", + description="Gets reactions by transaction_id and type", + responses={200: "Success", 400: "Bad request", 500: "Server error"}, + ) + @ns.expect(get_reactions_parser) + @marshal_with(get_reactions_response) + @cache(ttl_sec=5) + def get(self): + args = get_reactions_parser.parse_args() + tx_ids, type = args.get("tx_id"), args.get("type") + db = get_db_read_replica() + with db.scoped_session() as session: + reactions = get_reactions(session, tx_ids, type) + reactions = list(map(extend_reaction, reactions)) + return success_response(reactions) diff --git a/discovery-provider/src/queries/reactions.py b/discovery-provider/src/queries/reactions.py new file mode 100644 index 00000000000..335563a4b16 --- /dev/null +++ b/discovery-provider/src/queries/reactions.py @@ -0,0 +1,32 @@ +from typing import List, Tuple, TypedDict +from sqlalchemy.orm.session import Session +from src.models.models import User +from src.models.reaction import Reaction + +class ReactionResponse(TypedDict): + reaction_value: str + reaction_type: str + reacted_to: str + sender_user_id: int + +def get_reactions(session: Session, transaction_ids: List[str], type: str) -> List[ReactionResponse]: + results: List[Tuple[Reaction, int]] = ( + session.query(Reaction, User.user_id) + .join(User, User.wallet == Reaction.sender_wallet) + .filter( + Reaction.reacted_to.in_(transaction_ids), + Reaction.reaction_type == type, + User.is_current == True, + ) + .all() + ) + + return [ + { + "reaction_value": r.reaction_value, + "reaction_type": r.reaction_type, + "reacted_to": r.reacted_to, + "sender_user_id": user_id, + } + for (r, user_id) in results + ] \ No newline at end of file From d66a89d1960b551f3852446f0b52925c1c5f9071 Mon Sep 17 00:00:00 2001 From: Michael Piazza Date: Wed, 11 May 2022 19:12:30 +0000 Subject: [PATCH 05/13] Service commands: fix seed audio script --- service-commands/scripts/seed-audio.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/service-commands/scripts/seed-audio.sh b/service-commands/scripts/seed-audio.sh index aeb4a3b6c79..f1d09da7b5c 100644 --- a/service-commands/scripts/seed-audio.sh +++ b/service-commands/scripts/seed-audio.sh @@ -16,6 +16,7 @@ seed-audio () { userBank=$(get-user-bank $user_id) echo "Configuring ownerWallet..." jq .ownerWallet < ~/.audius/solana-program-config.json > ~/.audius/owner-wallet.json + solana config set -k ~/.audius/owner-wallet.json echo "Minting Audio..." mint=$(jq -rc .splToken < ~/.audius/solana-program-config.json) echo "spl-token mint $mint $amount" From 958b3d7a6fad6bcf5b238f6a4e7f9676eafab865 Mon Sep 17 00:00:00 2001 From: Michael Piazza Date: Wed, 11 May 2022 19:14:21 +0000 Subject: [PATCH 06/13] Identity lint fix --- .../migrations/20220509223231-change_reaction_columns.js | 5 ++--- identity-service/src/routes/reactions.js | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/identity-service/sequelize/migrations/20220509223231-change_reaction_columns.js b/identity-service/sequelize/migrations/20220509223231-change_reaction_columns.js index a49b650b713..40f243cc27f 100644 --- a/identity-service/sequelize/migrations/20220509223231-change_reaction_columns.js +++ b/identity-service/sequelize/migrations/20220509223231-change_reaction_columns.js @@ -1,4 +1,4 @@ -'use strict'; +'use strict' module.exports = { up: (queryInterface, Sequelize) => { @@ -12,5 +12,4 @@ module.exports = { .then(() => queryInterface.renameColumn('Reactions', 'reactionType', 'entityType')) .then(() => queryInterface.renameColumn('Reactions', 'reactionValue', 'reaction')) } -}; - +} diff --git a/identity-service/src/routes/reactions.js b/identity-service/src/routes/reactions.js index 1db8b213204..166e41e6630 100644 --- a/identity-service/src/routes/reactions.js +++ b/identity-service/src/routes/reactions.js @@ -39,7 +39,7 @@ module.exports = function (app) { app.post('/reactions', authMiddleware, handleResponse(async (req, res, next) => { // Validation const senderWallet = req.user.walletAddress - const { reactedTo,reactionValue } = req.body + const { reactedTo, reactionValue } = req.body if (!(senderWallet && reactedTo && reactionValue)) return errorResponseBadRequest(`Missing argument: ${JSON.stringify({ senderWallet, reactedTo, reactionValue })}`) From 1f42df905ca1e9aadca6e9860b4c00fa617bffa6 Mon Sep 17 00:00:00 2001 From: Michael Piazza Date: Wed, 11 May 2022 19:18:21 +0000 Subject: [PATCH 07/13] Discovery: fix lint --- discovery-provider/src/api/v1/helpers.py | 2 ++ discovery-provider/src/api/v1/reactions.py | 11 +++++++---- discovery-provider/src/queries/reactions.py | 17 +++++++++++------ 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/discovery-provider/src/api/v1/helpers.py b/discovery-provider/src/api/v1/helpers.py index fb764e1f32c..1a9af31316b 100644 --- a/discovery-provider/src/api/v1/helpers.py +++ b/discovery-provider/src/api/v1/helpers.py @@ -305,11 +305,13 @@ def extend_supporting(support: SupportResponse): "receiver": extend_user(support["user"]), } + def extend_reaction(reaction: ReactionResponse): new_reaction = reaction.copy() new_reaction["sender_user_id"] = encode_int_id(reaction["sender_user_id"]) return new_reaction + def extend_tip(tip): new_tip = tip.copy() new_tip["sender"] = extend_user(tip["sender"]) diff --git a/discovery-provider/src/api/v1/reactions.py b/discovery-provider/src/api/v1/reactions.py index 39213f80bb0..356cc269534 100644 --- a/discovery-provider/src/api/v1/reactions.py +++ b/discovery-provider/src/api/v1/reactions.py @@ -1,12 +1,15 @@ - from flask_restx import Namespace, Resource, fields, marshal_with, reqparse -from src.api.v1.helpers import DescriptiveArgument, make_response, success_response +from src.api.v1.helpers import ( + DescriptiveArgument, + extend_reaction, + make_response, + success_response, +) from src.api.v1.models.reactions import reaction +from src.queries.reactions import get_reactions from src.utils.db_session import get_db_read_replica -from src.utils.helpers import extend_reaction from src.utils.redis_cache import cache from src.utils.redis_metrics import record_metrics -from src.queries.reactions import get_reactions ns = Namespace("reactions", description="Reaction related operations") diff --git a/discovery-provider/src/queries/reactions.py b/discovery-provider/src/queries/reactions.py index 335563a4b16..2fed3f99f21 100644 --- a/discovery-provider/src/queries/reactions.py +++ b/discovery-provider/src/queries/reactions.py @@ -1,15 +1,20 @@ from typing import List, Tuple, TypedDict + from sqlalchemy.orm.session import Session from src.models.models import User from src.models.reaction import Reaction + class ReactionResponse(TypedDict): - reaction_value: str - reaction_type: str - reacted_to: str - sender_user_id: int + reaction_value: str + reaction_type: str + reacted_to: str + sender_user_id: int + -def get_reactions(session: Session, transaction_ids: List[str], type: str) -> List[ReactionResponse]: +def get_reactions( + session: Session, transaction_ids: List[str], type: str +) -> List[ReactionResponse]: results: List[Tuple[Reaction, int]] = ( session.query(Reaction, User.user_id) .join(User, User.wallet == Reaction.sender_wallet) @@ -29,4 +34,4 @@ def get_reactions(session: Session, transaction_ids: List[str], type: str) -> Li "sender_user_id": user_id, } for (r, user_id) in results - ] \ No newline at end of file + ] From b8f31acbb58e665cb4d0fca9b219318fb6abdf52 Mon Sep 17 00:00:00 2001 From: Michael Piazza Date: Thu, 12 May 2022 01:17:33 +0000 Subject: [PATCH 08/13] PR revisions --- discovery-provider/src/api/v1/api.py | 2 +- discovery-provider/src/api/v1/reactions.py | 10 +++++----- discovery-provider/src/queries/notifications.py | 15 +++++++++++---- discovery-provider/src/queries/reactions.py | 14 ++++++++------ 4 files changed, 25 insertions(+), 16 deletions(-) diff --git a/discovery-provider/src/api/v1/api.py b/discovery-provider/src/api/v1/api.py index dcca6574325..3784bb0fd2b 100644 --- a/discovery-provider/src/api/v1/api.py +++ b/discovery-provider/src/api/v1/api.py @@ -38,7 +38,6 @@ def specs_url(self): api_v1.add_namespace(tips_ns) api_v1.add_namespace(metrics_ns) api_v1.add_namespace(resolve_ns) -api_v1.add_namespace(reactions_ns) bp_full = Blueprint("api_v1_full", __name__, url_prefix="/v1/full") api_v1_full = ApiWithHTTPS(bp_full, version="1.0") @@ -48,3 +47,4 @@ def specs_url(self): api_v1_full.add_namespace(full_users_ns) api_v1_full.add_namespace(full_search_ns) api_v1_full.add_namespace(full_tips_ns) +api_v1_full.add_namespace(reactions_ns) diff --git a/discovery-provider/src/api/v1/reactions.py b/discovery-provider/src/api/v1/reactions.py index 356cc269534..f90e410cdf5 100644 --- a/discovery-provider/src/api/v1/reactions.py +++ b/discovery-provider/src/api/v1/reactions.py @@ -15,12 +15,12 @@ get_reactions_parser = reqparse.RequestParser(argument_class=DescriptiveArgument) get_reactions_parser.add_argument( - "type", required=True, description="The type of reactions for which to query." + "type", required=False, description="The type of reactions for which to query." ) get_reactions_parser.add_argument( - "tx_id", + "tx_signatures", required=True, - action="append", + action="split", description="The `reacted_to` transaction id(s) of the reactions in question.", ) @@ -29,7 +29,7 @@ ) -@ns.route("", doc=False) +@ns.route("") class BulkReactions(Resource): @record_metrics @ns.doc( @@ -42,7 +42,7 @@ class BulkReactions(Resource): @cache(ttl_sec=5) def get(self): args = get_reactions_parser.parse_args() - tx_ids, type = args.get("tx_id"), args.get("type") + tx_ids, type = args.get("tx_signatures"), args.get("type") db = get_db_read_replica() with db.scoped_session() as session: reactions = get_reactions(session, tx_ids, type) diff --git a/discovery-provider/src/queries/notifications.py b/discovery-provider/src/queries/notifications.py index 8d93d6147dd..8fefb3b1a96 100644 --- a/discovery-provider/src/queries/notifications.py +++ b/discovery-provider/src/queries/notifications.py @@ -6,6 +6,7 @@ from redis import Redis from sqlalchemy import desc from src import api_helpers +from src.api.v1.users import User from src.models import ( AggregateUser, Block, @@ -1160,17 +1161,23 @@ def solana_notifications(): ) reaction_results: List[Reaction] = ( - session.query(Reaction) - .filter(Reaction.slot >= min_slot_number, Reaction.slot <= max_slot_number) + session.query(Reaction, User.user_id) + .join(User, User.wallet == Reaction.sender_wallet) + .filter( + Reaction.slot >= min_slot_number, + Reaction.slot <= max_slot_number, + User.is_current == True, + ) .all() ) + reactions = [] - for reaction in reaction_results: + for (reaction, user_id) in reaction_results: reactions.append( { const.solana_notification_type: const.solana_notification_type_reaction, const.solana_notification_slot: reaction.slot, - const.notification_initiator: reaction.sender_wallet, + const.notification_initiator: user_id, const.solana_notification_metadata: { const.solana_notification_reaction_type: reaction.reaction_type, const.solana_notification_reaction_reaction_value: reaction.reaction_value, diff --git a/discovery-provider/src/queries/reactions.py b/discovery-provider/src/queries/reactions.py index 2fed3f99f21..901176480be 100644 --- a/discovery-provider/src/queries/reactions.py +++ b/discovery-provider/src/queries/reactions.py @@ -1,4 +1,4 @@ -from typing import List, Tuple, TypedDict +from typing import List, Optional, Tuple, TypedDict from sqlalchemy.orm.session import Session from src.models.models import User @@ -6,22 +6,24 @@ class ReactionResponse(TypedDict): - reaction_value: str + reaction_value: int reaction_type: str reacted_to: str sender_user_id: int def get_reactions( - session: Session, transaction_ids: List[str], type: str + session: Session, transaction_ids: List[str], type: Optional[str] ) -> List[ReactionResponse]: + filters = [Reaction.reacted_to.in_(transaction_ids), User.is_current == True] + if type: + filters.append(Reaction.reaction_type == type) + results: List[Tuple[Reaction, int]] = ( session.query(Reaction, User.user_id) .join(User, User.wallet == Reaction.sender_wallet) .filter( - Reaction.reacted_to.in_(transaction_ids), - Reaction.reaction_type == type, - User.is_current == True, + *filters, ) .all() ) From 50289217bfe2d4fea203bec115b6f304f195cb10 Mon Sep 17 00:00:00 2001 From: Michael Piazza Date: Thu, 12 May 2022 21:38:43 +0000 Subject: [PATCH 09/13] Fix migration idempotency --- ...2067242dd5_change_reaction_column_names.py | 43 +++++++++++-------- 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/discovery-provider/alembic/versions/0d2067242dd5_change_reaction_column_names.py b/discovery-provider/alembic/versions/0d2067242dd5_change_reaction_column_names.py index 8dd432bf814..57760fea5a6 100644 --- a/discovery-provider/alembic/versions/0d2067242dd5_change_reaction_column_names.py +++ b/discovery-provider/alembic/versions/0d2067242dd5_change_reaction_column_names.py @@ -16,24 +16,31 @@ def upgrade(): - op.alter_column("reactions", "entity_id", new_column_name="reacted_to") - op.alter_column("reactions", "entity_type", new_column_name="reaction_type") - op.alter_column("reactions", "reaction", new_column_name="reaction_value") - op.create_index( - op.f("ix_reactions_reacted_to_reaction_type"), - "reactions", - ["reacted_to", "reaction_type"], - unique=False, - info={"if_not_exists": True}, - ) + # Handle lack of idempotency for this migration + try: + op.alter_column("reactions", "entity_id", new_column_name="reacted_to") + op.alter_column("reactions", "entity_type", new_column_name="reaction_type") + op.alter_column("reactions", "reaction", new_column_name="reaction_value") + op.create_index( + op.f("ix_reactions_reacted_to_reaction_type"), + "reactions", + ["reacted_to", "reaction_type"], + unique=False, + info={"if_not_exists": True}, + ) + except BaseException as e: + pass def downgrade(): - op.alter_column("reactions", "reacted_to", new_column_name="entity_id") - op.alter_column("reactions", "reaction_type", new_column_name="entity_type") - op.alter_column("reactions", "reaction_value", new_column_name="reaction") - op.drop_index( - op.f("ix_reactions_reacted_to_reaction_type"), - table_name="reaction", - info={"if_not_exists": True}, - ) + try: + op.alter_column("reactions", "reacted_to", new_column_name="entity_id") + op.alter_column("reactions", "reaction_type", new_column_name="entity_type") + op.alter_column("reactions", "reaction_value", new_column_name="reaction") + op.drop_index( + op.f("ix_reactions_reacted_to_reaction_type"), + table_name="reaction", + info={"if_exists": True}, + ) + except BaseException as e: + pass From b81e2aaaa32cbc48bffc9f79a37a2ba0332b97d3 Mon Sep 17 00:00:00 2001 From: Michael Piazza Date: Fri, 13 May 2022 00:23:05 +0000 Subject: [PATCH 10/13] Fix migrations --- discovery-provider/alembic/env.py | 10 ++++- ...2067242dd5_change_reaction_column_names.py | 37 ++++++++++--------- 2 files changed, 27 insertions(+), 20 deletions(-) diff --git a/discovery-provider/alembic/env.py b/discovery-provider/alembic/env.py index 3e2d1f1ad9d..fe873c630b9 100644 --- a/discovery-provider/alembic/env.py +++ b/discovery-provider/alembic/env.py @@ -3,9 +3,9 @@ import os import re -from alembic import context +from alembic import context, op from alembic.ddl.base import AddColumn, DropColumn, visit_add_column, visit_drop_column -from sqlalchemy import engine_from_config, pool +from sqlalchemy import engine_from_config, pool, inspect from sqlalchemy.ext.compiler import compiles from sqlalchemy.schema import CreateIndex, CreateTable, DropIndex, DropTable @@ -114,6 +114,12 @@ def _add_if_exists(element, compiler, **kw): ) +def column_exists(table_name, column_name): + bind = op.get_context().bind + insp = inspect(bind) + columns = insp.get_columns(table_name) + return any(c["name"] == column_name for c in columns) + if context.is_offline_mode(): run_migrations_offline() else: diff --git a/discovery-provider/alembic/versions/0d2067242dd5_change_reaction_column_names.py b/discovery-provider/alembic/versions/0d2067242dd5_change_reaction_column_names.py index 57760fea5a6..af9a8ec3dc2 100644 --- a/discovery-provider/alembic/versions/0d2067242dd5_change_reaction_column_names.py +++ b/discovery-provider/alembic/versions/0d2067242dd5_change_reaction_column_names.py @@ -7,6 +7,7 @@ """ import sqlalchemy as sa from alembic import op +from alembic.env import column_exists # revision identifiers, used by Alembic. revision = "0d2067242dd5" @@ -17,30 +18,30 @@ def upgrade(): # Handle lack of idempotency for this migration - try: + if column_exists("reactions", "entity_id"): op.alter_column("reactions", "entity_id", new_column_name="reacted_to") + if column_exists("reactions", "entity_type"): op.alter_column("reactions", "entity_type", new_column_name="reaction_type") + if column_exists("reactions", "reaction"): op.alter_column("reactions", "reaction", new_column_name="reaction_value") - op.create_index( - op.f("ix_reactions_reacted_to_reaction_type"), - "reactions", - ["reacted_to", "reaction_type"], - unique=False, - info={"if_not_exists": True}, - ) - except BaseException as e: - pass + op.create_index( + op.f("ix_reactions_reacted_to_reaction_type"), + "reactions", + ["reacted_to", "reaction_type"], + unique=False, + info={"if_not_exists": True}, + ) def downgrade(): - try: + if column_exists("reactions", "reacted_to"): op.alter_column("reactions", "reacted_to", new_column_name="entity_id") + if column_exists("reactions", "reaction_type"): op.alter_column("reactions", "reaction_type", new_column_name="entity_type") + if column_exists("reactions", "reaction_value"): op.alter_column("reactions", "reaction_value", new_column_name="reaction") - op.drop_index( - op.f("ix_reactions_reacted_to_reaction_type"), - table_name="reaction", - info={"if_exists": True}, - ) - except BaseException as e: - pass + op.drop_index( + op.f("ix_reactions_reacted_to_reaction_type"), + table_name="reaction", + info={"if_exists": True}, + ) From 4c36525aa0e866995b47d3444059d2faa350d240 Mon Sep 17 00:00:00 2001 From: Michael Piazza Date: Fri, 13 May 2022 00:37:36 +0000 Subject: [PATCH 11/13] Use a local fn --- discovery-provider/alembic/env.py | 10 ++-------- .../0d2067242dd5_change_reaction_column_names.py | 8 ++++++-- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/discovery-provider/alembic/env.py b/discovery-provider/alembic/env.py index fe873c630b9..3e2d1f1ad9d 100644 --- a/discovery-provider/alembic/env.py +++ b/discovery-provider/alembic/env.py @@ -3,9 +3,9 @@ import os import re -from alembic import context, op +from alembic import context from alembic.ddl.base import AddColumn, DropColumn, visit_add_column, visit_drop_column -from sqlalchemy import engine_from_config, pool, inspect +from sqlalchemy import engine_from_config, pool from sqlalchemy.ext.compiler import compiles from sqlalchemy.schema import CreateIndex, CreateTable, DropIndex, DropTable @@ -114,12 +114,6 @@ def _add_if_exists(element, compiler, **kw): ) -def column_exists(table_name, column_name): - bind = op.get_context().bind - insp = inspect(bind) - columns = insp.get_columns(table_name) - return any(c["name"] == column_name for c in columns) - if context.is_offline_mode(): run_migrations_offline() else: diff --git a/discovery-provider/alembic/versions/0d2067242dd5_change_reaction_column_names.py b/discovery-provider/alembic/versions/0d2067242dd5_change_reaction_column_names.py index af9a8ec3dc2..e09ebe36f61 100644 --- a/discovery-provider/alembic/versions/0d2067242dd5_change_reaction_column_names.py +++ b/discovery-provider/alembic/versions/0d2067242dd5_change_reaction_column_names.py @@ -5,9 +5,8 @@ Create Date: 2022-05-09 22:03:16.838837 """ -import sqlalchemy as sa +import sqlalchemy as sa, inspect from alembic import op -from alembic.env import column_exists # revision identifiers, used by Alembic. revision = "0d2067242dd5" @@ -15,6 +14,11 @@ branch_labels = None depends_on = None +def column_exists(table_name, column_name): + bind = op.get_context().bind + insp = inspect(bind) + columns = insp.get_columns(table_name) + return any(c["name"] == column_name for c in columns) def upgrade(): # Handle lack of idempotency for this migration From 80b9d3b5380b98da0f71f3ad094752d2ec941cd5 Mon Sep 17 00:00:00 2001 From: Michael Piazza Date: Fri, 13 May 2022 16:40:26 +0000 Subject: [PATCH 12/13] Fix migration again --- .../0d2067242dd5_change_reaction_column_names.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/discovery-provider/alembic/versions/0d2067242dd5_change_reaction_column_names.py b/discovery-provider/alembic/versions/0d2067242dd5_change_reaction_column_names.py index e09ebe36f61..c67bdcaf16a 100644 --- a/discovery-provider/alembic/versions/0d2067242dd5_change_reaction_column_names.py +++ b/discovery-provider/alembic/versions/0d2067242dd5_change_reaction_column_names.py @@ -1,25 +1,29 @@ """Change reaction column names Revision ID: 0d2067242dd5 -Revises: 35198266d709 +Revises: f11f9e83b28b Create Date: 2022-05-09 22:03:16.838837 """ -import sqlalchemy as sa, inspect +import inspect + +import sqlalchemy as sa from alembic import op # revision identifiers, used by Alembic. revision = "0d2067242dd5" -down_revision = "35198266d709" +down_revision = "f11f9e83b28b" branch_labels = None depends_on = None + def column_exists(table_name, column_name): bind = op.get_context().bind insp = inspect(bind) columns = insp.get_columns(table_name) return any(c["name"] == column_name for c in columns) + def upgrade(): # Handle lack of idempotency for this migration if column_exists("reactions", "entity_id"): From dc412f757b7770dd855e211348f56dba6e82d1af Mon Sep 17 00:00:00 2001 From: Michael Piazza Date: Fri, 13 May 2022 21:44:33 +0000 Subject: [PATCH 13/13] Last migration fix --- .../versions/0d2067242dd5_change_reaction_column_names.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/discovery-provider/alembic/versions/0d2067242dd5_change_reaction_column_names.py b/discovery-provider/alembic/versions/0d2067242dd5_change_reaction_column_names.py index c67bdcaf16a..7c0e2974d2c 100644 --- a/discovery-provider/alembic/versions/0d2067242dd5_change_reaction_column_names.py +++ b/discovery-provider/alembic/versions/0d2067242dd5_change_reaction_column_names.py @@ -19,7 +19,7 @@ def column_exists(table_name, column_name): bind = op.get_context().bind - insp = inspect(bind) + insp = sa.inspect(bind) columns = insp.get_columns(table_name) return any(c["name"] == column_name for c in columns)