From d5bb20fb200c40fc851754be5331bb4946a4f2a1 Mon Sep 17 00:00:00 2001 From: Randy Schott <1815175+schottra@users.noreply.github.com> Date: Wed, 17 Jan 2024 11:25:58 -0500 Subject: [PATCH 01/13] helper commands for setting up withdrawals and recoveries --- .../compose/docker-compose.dev-tools.prod.yml | 1 + package-lock.json | 1 + packages/commands/package.json | 1 + packages/commands/src/index.mjs | 52 ++-- .../src/route-tokens-to-user-bank.mjs | 249 ++++++++++++++++++ packages/commands/src/withdraw-tokens.mjs | 133 ++++++++++ 6 files changed, 412 insertions(+), 25 deletions(-) create mode 100644 packages/commands/src/route-tokens-to-user-bank.mjs create mode 100644 packages/commands/src/withdraw-tokens.mjs diff --git a/dev-tools/compose/docker-compose.dev-tools.prod.yml b/dev-tools/compose/docker-compose.dev-tools.prod.yml index 21d0d71535e..9d5d43ca35c 100644 --- a/dev-tools/compose/docker-compose.dev-tools.prod.yml +++ b/dev-tools/compose/docker-compose.dev-tools.prod.yml @@ -42,6 +42,7 @@ services: SOLANA_REWARD_MANAGER_PUBLIC_KEY: '${SOLANA_REWARD_MANAGER_PUBLIC_KEY}' SOLANA_REWARD_MANAGER_PDA_PUBLIC_KEY: '${SOLANA_REWARD_MANAGER_PDA_PUBLIC_KEY}' SOLANA_REWARD_MANAGER_TOKEN_PDA_PUBLIC_KEY: '${SOLANA_REWARD_MANAGER_TOKEN_PDA_PUBLIC_KEY}' + SOLANA_PAYMENT_ROUTER_PUBLIC_KEY: '${SOLANA_PAYMENT_ROUTER_PUBLIC_KEY}' SOLANA_FEEPAYER_SECRET_KEY: '${SOLANA_FEEPAYER_SECRET_KEY}' IDENTITY_SERVICE_URL: 'http://identity-service:7000' diff --git a/package-lock.json b/package-lock.json index 01078565b75..b75292ef72a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -107152,6 +107152,7 @@ "license": "Apache-2.0", "dependencies": { "@audius/sdk": "*", + "@audius/spl": "*", "@solana/spl-token": "0.3.8", "@solana/web3.js": "1.78.4", "bn.js": "^5.2.1", diff --git a/packages/commands/package.json b/packages/commands/package.json index 85d7d39edab..bf1ec682531 100644 --- a/packages/commands/package.json +++ b/packages/commands/package.json @@ -15,6 +15,7 @@ "license": "Apache-2.0", "dependencies": { "@audius/sdk": "*", + "@audius/spl": "*", "@solana/spl-token": "0.3.8", "@solana/web3.js": "1.78.4", "bn.js": "^5.2.1", diff --git a/packages/commands/src/index.mjs b/packages/commands/src/index.mjs index 49c6f3ad16d..12d8d95263e 100755 --- a/packages/commands/src/index.mjs +++ b/packages/commands/src/index.mjs @@ -1,32 +1,34 @@ #!/usr/bin/env node -import { program } from "commander"; +import { program } from 'commander' -import "./create-playlist.mjs"; -import "./edit-playlist.mjs"; -import "./create-user.mjs"; -import "./edit-user.mjs"; -import "./favorite-track.mjs"; -import "./unfavorite-track.mjs"; -import "./favorite-playlist.mjs"; -import "./unfavorite-playlist.mjs"; -import "./follow.mjs"; -import "./unfollow.mjs"; -import "./repost-track.mjs"; -import "./unrepost-track.mjs"; -import "./repost-playlist.mjs"; -import "./unrepost-playlist.mjs"; -import "./upload-track.mjs"; -import "./edit-track.mjs"; -import "./mint-tokens.mjs"; -import "./tip-audio.mjs"; -import "./auth-headers.mjs"; -import "./get-audio-balance.mjs"; -import "./create-user-bank.mjs"; -import "./purchase-track.mjs"; +import './create-playlist.mjs' +import './edit-playlist.mjs' +import './create-user.mjs' +import './edit-user.mjs' +import './favorite-track.mjs' +import './unfavorite-track.mjs' +import './favorite-playlist.mjs' +import './unfavorite-playlist.mjs' +import './follow.mjs' +import './unfollow.mjs' +import './repost-track.mjs' +import './unrepost-track.mjs' +import './repost-playlist.mjs' +import './unrepost-playlist.mjs' +import './upload-track.mjs' +import './edit-track.mjs' +import './mint-tokens.mjs' +import './tip-audio.mjs' +import './auth-headers.mjs' +import './get-audio-balance.mjs' +import './create-user-bank.mjs' +import './purchase-track.mjs' +import './route-tokens-to-user-bank.mjs' +import './withdraw-tokens.mjs' async function main() { - program.parseAsync(process.argv); + program.parseAsync(process.argv) } -main(); +main() diff --git a/packages/commands/src/route-tokens-to-user-bank.mjs b/packages/commands/src/route-tokens-to-user-bank.mjs new file mode 100644 index 00000000000..a0b2dbc43fe --- /dev/null +++ b/packages/commands/src/route-tokens-to-user-bank.mjs @@ -0,0 +1,249 @@ +import { + TOKEN_PROGRAM_ID, + createAssociatedTokenAccountInstruction, + createTransferCheckedInstruction, + getAccount, + mintTo +} from '@solana/spl-token' +import { + Connection, + Keypair, + PublicKey, + Transaction, + TransactionInstruction, + sendAndConfirmTransaction +} from '@solana/web3.js' +import chalk from 'chalk' +import { Option, program } from 'commander' + +import { route } from '@audius/spl' + +import { initializeAudiusLibs } from './utils.mjs' + +const TOKEN_DECIMALS = { + audio: 8, + usdc: 6 +} + +const MEMO_PROGRAM_ID = new PublicKey( + 'Memo1UhkJRfHyvLMcVucJwxXeuD728EqVDDwQDxFMNo' +) + +program + .command('route-tokens-to-user-bank') + .argument('', 'The amount of tokens to send') + .description( + 'Transfer $USDC or $AUDIO tokens using payment router into a user bank from the owner wallet' + ) + .addOption( + new Option('-m, --mint [mint]', 'The currency to use') + .choices(['audio', 'usdc']) + .default('usdc') + ) + .option('-f, --from ', 'The account to send tokens to (handle)') + .option('--memo ', 'A data string to attach as a memo') + .action(async (amountInput, { from, mint, memo }) => { + const amount = BigInt(amountInput) + const audiusLibs = await initializeAudiusLibs(from) + const { solanaWeb3Manager } = audiusLibs + + const { userbank: userbankPublicKey } = + await solanaWeb3Manager.createUserBankIfNeeded({ + mint + }) + + if (!process.env.SOLANA_ENDPOINT) { + program.error('SOLANA_ENDPOINT environment variable not set') + } + + const connection = new Connection(process.env.SOLANA_ENDPOINT) + const feePayer = Keypair.fromSecretKey( + Uint8Array.from(JSON.parse(process.env.SOLANA_FEEPAYER_SECRET_KEY)) + ) + + const paymentRouterPublicKey = new PublicKey( + process.env.SOLANA_PAYMENT_ROUTER_PUBLIC_KEY + ) + + const tokenMint = + mint === 'audio' + ? new PublicKey(process.env.SOLANA_TOKEN_MINT_PUBLIC_KEY) + : new PublicKey(process.env.SOLANA_USDC_TOKEN_MINT_PUBLIC_KEY) + + const senderAccountKeypair = Keypair.fromSecretKey( + Uint8Array.from(JSON.parse(process.env.SOLANA_OWNER_SECRET_KEY)) + ) + const senderAccountPublicKey = senderAccountKeypair.publicKey + + try { + console.log('checking for source usdc account') + const senderTokenAccount = + await solanaWeb3Manager.findAssociatedTokenAddress( + senderAccountPublicKey.toString(), + mint + ) + let senderTokenAccountInfo = await solanaWeb3Manager.getTokenAccountInfo( + senderTokenAccount.toString() + ) + + // If it's not a valid token account, we need to make one first + if (!senderTokenAccountInfo) { + console.log( + 'Provided recipient solana address has no associated token account, creating' + ) + const { blockhash, lastValidBlockHeight } = + await connection.getLatestBlockhash() + const accountCreationTx = new Transaction({ + blockhash, + lastValidBlockHeight + }) + const createTokenAccountInstruction = + createAssociatedTokenAccountInstruction( + senderAccountPublicKey, // fee payer + senderTokenAccount, // account to create + senderAccountPublicKey, // owner + tokenMint // mint + ) + accountCreationTx.add(createTokenAccountInstruction) + + const accountCreationTxSignature = await sendAndConfirmTransaction( + connection, + accountCreationTx, + [senderAccountKeypair], + { + skipPreflight: true + } + ) + console.log(chalk.green(`Successfully created new ${mint} account`)) + console.log( + chalk.yellow('Transaction Signature:'), + accountCreationTxSignature + ) + + senderTokenAccountInfo = await solanaWeb3Manager.getTokenAccountInfo( + senderTokenAccount.toString() + ) + } + + if (senderTokenAccountInfo.amount < amount) { + console.log('Source account has insufficient funds, minting...') + // Fund source wallet first + const fundingTxSignature = await mintTo( + connection, + feePayer, + tokenMint, + senderTokenAccount, + senderAccountKeypair, + amount + ) + console.log( + chalk.green(`Successfully minted ${mint} to source account`) + ) + console.log(chalk.yellow('Transaction Signature:'), fundingTxSignature) + } else { + console.log( + `Souce account has sufficient funds (${senderTokenAccountInfo.amount})` + ) + } + + // Transfer into user bank + console.info('Transferring from source account to user bank...') + + const { blockhash, lastValidBlockHeight } = + await connection.getLatestBlockhash() + const transferTx = new Transaction({ + blockhash, + lastValidBlockHeight + }) + + const [paymentRouterPda, paymentRouterPdaBump] = + PublicKey.findProgramAddressSync( + [Buffer.from('payment_router')], + paymentRouterPublicKey + ) + + // Associated token account owned by the PDA + const paymentRouterTokenAccount = + await solanaWeb3Manager.findAssociatedTokenAddress( + paymentRouterPda.toString(), + mint + ) + const paymentRouterTokenAccountInfo = + solanaWeb3Manager.getTokenAccountInfo( + paymentRouterTokenAccount.toString() + ) + if (paymentRouterTokenAccountInfo === null) { + throw new Error( + 'Payment Router balance PDA token account does not exist' + ) + } + + const transferInstruction = createTransferCheckedInstruction( + senderTokenAccount, + tokenMint, + paymentRouterTokenAccount, + senderAccountPublicKey, + amount, + TOKEN_DECIMALS[mint] + ) + + const paymentRouterInstruction = await route( + paymentRouterTokenAccount, + paymentRouterPda, + paymentRouterPdaBump, + [userbankPublicKey], // recipients + [amount], + amount, + TOKEN_PROGRAM_ID, + paymentRouterPublicKey + ) + + transferTx.add(transferInstruction, paymentRouterInstruction) + + if (memo) { + transferTx.add( + new TransactionInstruction({ + keys: [ + { + pubkey: senderAccountPublicKey, + isSigner: true, + isWritable: true + } + ], + programId: MEMO_PROGRAM_ID, + data: Buffer.from(memo) + }) + ) + } + + const transferTxSignature = await sendAndConfirmTransaction( + connection, + transferTx, + [senderAccountKeypair], + { + skipPreflight: true + } + ) + + console.log( + chalk.green(`Successfully transferred ${mint} to dest account`) + ) + console.log(chalk.yellow('Transaction Signature:'), transferTxSignature) + + const accountInfo = await getAccount(connection, userbankPublicKey) + + console.log( + chalk.yellow('User bank address: '), + userbankPublicKey.toBase58() + ) + console.log( + chalk.yellow('Balance: '), + accountInfo.amount.toString() + ) + } catch (err) { + console.error(err) + program.error(`Failed to transfer tokens ${err.message}`) + } + + process.exit(0) + }) diff --git a/packages/commands/src/withdraw-tokens.mjs b/packages/commands/src/withdraw-tokens.mjs new file mode 100644 index 00000000000..ea7f467dc05 --- /dev/null +++ b/packages/commands/src/withdraw-tokens.mjs @@ -0,0 +1,133 @@ +import { Connection, Keypair } from '@solana/web3.js' +import BN from 'bn.js' +import chalk from 'chalk' +import { program, Option } from 'commander' + +import { initializeAudiusLibs } from './utils.mjs' + +import { createAssociatedTokenAccountInstruction } from '@solana/spl-token' +import { + Transaction, + sendAndConfirmTransaction, + PublicKey +} from '@solana/web3.js' + +program + .command('withdraw-tokens') + .description('Send USDC from a user bank to an external address') + .argument('', 'The solana address of the recipient') + .argument('', 'The amount of tokens to tip (in wei)') + .addOption( + new Option('-m, --mint [mint]', 'The currency to use') + .choices(['audio', 'usdc']) + .default('usdc') + ) + .option('-f, --from [from]', 'The account to tip from') + .action(async (recipientAccount, amount, { from, mint }) => { + const audiusLibs = await initializeAudiusLibs(from) + const { solanaWeb3Manager } = audiusLibs + + if (!process.env.SOLANA_ENDPOINT) { + program.error('SOLANA_ENDPOINT environment variable not set') + } + + const connection = new Connection(process.env.SOLANA_ENDPOINT) + const feePayer = Keypair.fromSecretKey( + Uint8Array.from(JSON.parse(process.env.SOLANA_FEEPAYER_SECRET_KEY)) + ) + + const tokenMint = + mint === 'audio' + ? new PublicKey(process.env.SOLANA_TOKEN_MINT_PUBLIC_KEY) + : new PublicKey(process.env.SOLANA_USDC_TOKEN_MINT_PUBLIC_KEY) + + const recipientAccountPublicKey = new PublicKey(recipientAccount) + + const { userbank: userbankPublicKey } = + await solanaWeb3Manager.createUserBankIfNeeded({ + mint + }) + + try { + console.log('checking for recipient usdc account...') + const recipientTokenAccount = + await solanaWeb3Manager.findAssociatedTokenAddress( + recipientAccountPublicKey.toString(), + mint + ) + let recipientTokenAccountInfo = + await solanaWeb3Manager.getTokenAccountInfo( + recipientTokenAccount.toString() + ) + + // If it's not a valid token account, we need to make one first + if (!recipientTokenAccountInfo) { + console.log( + 'Provided recipient solana address has no associated token account, creating' + ) + const { blockhash, lastValidBlockHeight } = + await connection.getLatestBlockhash() + const accountCreationTx = new Transaction({ + blockhash, + lastValidBlockHeight + }) + const createTokenAccountInstruction = + createAssociatedTokenAccountInstruction( + feePayer.publicKey, // fee payer + recipientTokenAccount, // account to create + recipientAccountPublicKey, // owner + tokenMint // mint + ) + accountCreationTx.add(createTokenAccountInstruction) + + const accountCreationTxSignature = await sendAndConfirmTransaction( + connection, + accountCreationTx, + [feePayer], + { + skipPreflight: true + } + ) + console.log(chalk.green(`Successfully created new ${mint} account`)) + console.log( + chalk.yellow('Transaction Signature:'), + accountCreationTxSignature + ) + + recipientTokenAccountInfo = await solanaWeb3Manager.getTokenAccountInfo( + recipientTokenAccount.toString() + ) + } + + console.log('transferring USDC to recipient...') + const instructions = + await audiusLibs.solanaWeb3Manager.createTransferInstructionsFromCurrentUser( + { + amount: new BN(amount), + feePayerKey: feePayer.publicKey, + senderSolanaAddress: userbankPublicKey, + recipientSolanaAddress: recipientTokenAccount, + mint + } + ) + + const { blockhash, lastValidBlockHeight } = + await connection.getLatestBlockhash() + const transferTx = new Transaction({ + blockhash, + feePayer: feePayer.publicKey, + lastValidBlockHeight + }) + transferTx.add(...instructions) + const tx = await sendAndConfirmTransaction(connection, transferTx, [ + feePayer + ]) + + console.log(chalk.green('Successfully withdrew USDC')) + console.log(chalk.yellow('Transaction Signature:'), tx) + } catch (err) { + program.error(err) + } + + process.exit(0) + }) From abe96722528de5dbef683696ffed6caddb20e0da Mon Sep 17 00:00:00 2001 From: Randy Schott <1815175+schottra@users.noreply.github.com> Date: Thu, 18 Jan 2024 18:33:41 -0500 Subject: [PATCH 02/13] checkpoint for indexing recovery transactions --- .vscode/settings.json | 4 +- .../discovery-provider/.vscode/settings.json | 2 +- .../tasks/payment_router_mock_transactions.py | 201 +++++++++ .../tasks/test_index_payment_router.py | 154 +++++++ .../src/tasks/index_payment_router.py | 401 +++++++++++++----- 5 files changed, 654 insertions(+), 108 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index f4af686f1b9..ceefbc3fd9c 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -21,14 +21,14 @@ "python.testing.unittestEnabled": false, "[python]": { "editor.codeActionsOnSave": { - "source.organizeImports": true + "source.organizeImports": "explicit" }, "editor.defaultFormatter": "ms-python.black-formatter" }, "editor.find.addExtraSpaceOnTop": false, "eslint.workingDirectories": [{ "pattern": "./packages/*/" }], "editor.codeActionsOnSave": { - "source.fixAll": true + "source.fixAll": "explicit" }, "gitlens.advanced.fileHistoryFollowsRenames": true } diff --git a/packages/discovery-provider/.vscode/settings.json b/packages/discovery-provider/.vscode/settings.json index fc51bc3e1c6..6a52dfd6fad 100644 --- a/packages/discovery-provider/.vscode/settings.json +++ b/packages/discovery-provider/.vscode/settings.json @@ -15,7 +15,7 @@ "python.testing.unittestEnabled": false, "[python]": { "editor.codeActionsOnSave": { - "source.organizeImports": true + "source.organizeImports": "explicit" }, "editor.defaultFormatter": "ms-python.black-formatter" }, diff --git a/packages/discovery-provider/integration_tests/tasks/payment_router_mock_transactions.py b/packages/discovery-provider/integration_tests/tasks/payment_router_mock_transactions.py index 7b1e0abb452..e3d817a1a48 100644 --- a/packages/discovery-provider/integration_tests/tasks/payment_router_mock_transactions.py +++ b/packages/discovery-provider/integration_tests/tasks/payment_router_mock_transactions.py @@ -2311,3 +2311,204 @@ } ) ) + + +# Routes $1.00 to a single recipient w/ a recovery memo that does not specify +# a source transaction. The sending address should match the recipient of a previous +# transaction in order to trigger recovery +mock_valid_transfer_single_recipient_recovery_tx = GetTransactionResp.from_json( + json.dumps( + { + "jsonrpc": "2.0", + "result": { + "slot": 190957, + "transaction": { + "signatures": [ + "5wPxiuLSF3MzXZt9XG99UEPNdxs8DtE2vWKezrB6zuMCrkMBJx6iU7xw5icaowpfgj96iLGnAgEAaBNSbneWdbZw" + ], + "message": { + "header": { + "numRequiredSignatures": 1, + "numReadonlySignedAccounts": 0, + "numReadonlyUnsignedAccounts": 5, + }, + "accountKeys": [ + "HXLN9UWwAjMPgHaFZDfgabT79SmLSdTeu2fUha2xHz9W", + "3XmVeZ6M1FYDdUQaNeQZf8dipvtzNP6NVb5xjDkdeiNb", + "A76eNhRrfdy6WfMoQf4ALasMxzRWHajH4TrVuX2NUjZT", + "7gfRGGdp89N9g3mCsZjaGmDDRdcTnZh9u3vYyBab2tRy", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "26Q7gP8UfkDzi7GMFEQxTJaNJ8D2ybCUjex58M5MLu8y", + "Memo1UhkJRfHyvLMcVucJwxXeuD728EqVDDwQDxFMNo", + "apaySbqV1XAmuiGszeN4NyWrXkkMrnuJVoNhzmS1AMa", + "G231EZsMoCNBiQKP5quEeAM3oG516Zspirjnh7ywP71i", + ], + "recentBlockhash": "6D65tSU7pjSmFvSj9qK2W2bjkESw4XZebeNmgA1rCqnF", + "instructions": [ + { + "programIdIndex": 4, + "accounts": [1, 5, 2, 0], + "data": "hYECWfYe8vYqs", + "stackHeight": None, + }, + { + "programIdIndex": 6, + "accounts": [0], + # "recovery" + "data": "L8nXdrNsKha", + "stackHeight": None, + }, + { + "programIdIndex": 7, + "accounts": [2, 8, 4, 3], + "data": "BQD4GnQPrhbq6Y9NJLnwDUziXhfF6BjkLYFbnKZH", + "stackHeight": None, + }, + ], + "addressTableLookups": [], + }, + }, + "meta": { + "err": None, + "status": {"Ok": None}, + "fee": 5000, + "preBalances": [ + 8420804160, + 2039280, + 2039280, + 2039280, + 929020800, + 1461600, + 119712000, + 1141440, + 946560, + ], + "postBalances": [ + 8420799160, + 2039280, + 2039280, + 2039280, + 929020800, + 1461600, + 119712000, + 1141440, + 946560, + ], + "innerInstructions": [ + { + "index": 2, + "instructions": [ + { + "programIdIndex": 4, + "accounts": [2, 3, 8, 8], + "data": "3YKuzAsyicvj", + "stackHeight": 2, + } + ], + } + ], + "logMessages": [ + "Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA invoke [1]", + "Program log: Instruction: TransferChecked", + "Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA consumed 6173 of 600000 compute units", + "Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA success", + "Program Memo1UhkJRfHyvLMcVucJwxXeuD728EqVDDwQDxFMNo invoke [1]", + "Program Memo1UhkJRfHyvLMcVucJwxXeuD728EqVDDwQDxFMNo consumed 480 of 593827 compute units", + "Program Memo1UhkJRfHyvLMcVucJwxXeuD728EqVDDwQDxFMNo success", + "Program apaySbqV1XAmuiGszeN4NyWrXkkMrnuJVoNhzmS1AMa invoke [1]", + "Program log: Instruction: Route", + "Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA invoke [2]", + "Program log: Instruction: Transfer", + "Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA consumed 4728 of 576902 compute units", + "Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA success", + "Program log: All transfers complete!", + "Program apaySbqV1XAmuiGszeN4NyWrXkkMrnuJVoNhzmS1AMa consumed 21782 of 593347 compute units", + "Program apaySbqV1XAmuiGszeN4NyWrXkkMrnuJVoNhzmS1AMa success", + ], + "preTokenBalances": [ + { + "accountIndex": 1, + "mint": "26Q7gP8UfkDzi7GMFEQxTJaNJ8D2ybCUjex58M5MLu8y", + "uiTokenAmount": { + "uiAmount": 10.0, + "decimals": 6, + "amount": "10000000", + "uiAmountString": "10.0", + }, + "owner": "HXLN9UWwAjMPgHaFZDfgabT79SmLSdTeu2fUha2xHz9W", + "programId": "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + }, + { + "accountIndex": 2, + "mint": "26Q7gP8UfkDzi7GMFEQxTJaNJ8D2ybCUjex58M5MLu8y", + "uiTokenAmount": { + "uiAmount": None, + "decimals": 6, + "amount": "0", + "uiAmountString": "0", + }, + "owner": "G231EZsMoCNBiQKP5quEeAM3oG516Zspirjnh7ywP71i", + "programId": "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + }, + { + "accountIndex": 3, + "mint": "26Q7gP8UfkDzi7GMFEQxTJaNJ8D2ybCUjex58M5MLu8y", + "uiTokenAmount": { + "uiAmount": 0, + "decimals": 6, + "amount": "0", + "uiAmountString": "0", + }, + "owner": "7vKR1WSmyHvBmCvKPZBiN66PHZqYQbXw51SZdwtVd9Dt", + "programId": "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + }, + ], + "postTokenBalances": [ + { + "accountIndex": 1, + "mint": "26Q7gP8UfkDzi7GMFEQxTJaNJ8D2ybCUjex58M5MLu8y", + "uiTokenAmount": { + "uiAmount": 9.0, + "decimals": 6, + "amount": "9000000", + "uiAmountString": "9.0", + }, + "owner": "HXLN9UWwAjMPgHaFZDfgabT79SmLSdTeu2fUha2xHz9W", + "programId": "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + }, + { + "accountIndex": 2, + "mint": "26Q7gP8UfkDzi7GMFEQxTJaNJ8D2ybCUjex58M5MLu8y", + "uiTokenAmount": { + "uiAmount": None, + "decimals": 6, + "amount": "0", + "uiAmountString": "0", + }, + "owner": "G231EZsMoCNBiQKP5quEeAM3oG516Zspirjnh7ywP71i", + "programId": "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + }, + { + "accountIndex": 3, + "mint": "26Q7gP8UfkDzi7GMFEQxTJaNJ8D2ybCUjex58M5MLu8y", + "uiTokenAmount": { + "uiAmount": 1.0, + "decimals": 6, + "amount": "1000000", + "uiAmountString": "1.0", + }, + "owner": "7vKR1WSmyHvBmCvKPZBiN66PHZqYQbXw51SZdwtVd9Dt", + "programId": "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + }, + ], + "rewards": [], + "loadedAddresses": {"writable": [], "readonly": []}, + "computeUnitsConsumed": 28435, + }, + "version": 0, + "blockTime": 1701922096, + }, + "id": 0, + } + ) +) diff --git a/packages/discovery-provider/integration_tests/tasks/test_index_payment_router.py b/packages/discovery-provider/integration_tests/tasks/test_index_payment_router.py index df26eee0342..93aebbe0122 100644 --- a/packages/discovery-provider/integration_tests/tasks/test_index_payment_router.py +++ b/packages/discovery-provider/integration_tests/tasks/test_index_payment_router.py @@ -13,6 +13,7 @@ mock_valid_track_purchase_single_recipient_tx, mock_valid_transfer_without_purchase_multi_recipient_tx, mock_valid_transfer_without_purchase_single_recipient_tx, + mock_valid_transfer_single_recipient_recovery_tx, ) from integration_tests.utils import populate_mock_db @@ -33,8 +34,10 @@ trackBuyerUserBank = "38YSndmPWVF3UdzczbB3UMYUgPQtZrgvvPVHa3M4yQVX" thirdPartyId = 3 thirdPartyUserBank = "7dw7W4Yv7F1uWb9dVH1CFPm39mePyypuCji2zxcFA556" + # Used as the source wallet for all the mock transactions transactionSenderAddress = "G231EZsMoCNBiQKP5quEeAM3oG516Zspirjnh7ywP71i" +transactionSenderUsdcAccount = "3XmVeZ6M1FYDdUQaNeQZf8dipvtzNP6NVb5xjDkdeiNb" test_entries = { "users": [ @@ -192,6 +195,157 @@ def test_process_payment_router_tx_details_transfer_transfer_without_purchase( assert transaction_record.tx_metadata == transactionSenderAddress +# Should revert the most recent outbound transaction to the sending address +# of the recovery transaction +def test_process_payment_router_tx_details_transfer_recovery( + app, +): + tx_response = mock_valid_transfer_single_recipient_recovery_tx + with app.app_context(): + db = get_db() + + transaction = tx_response.value.transaction.transaction + + tx_sig_str = str(transaction.signatures[0]) + + challenge_event_bus = create_autospec(ChallengeEventBus) + + test_entries_with_transaction = test_entries.copy() + test_entries_with_transaction["usdc_transactions_history"] = [ + { + "user_bank": trackOwnerUserBank, + "signature": "existingWithdrawal", + "transaction_type": USDCTransactionType.transfer, + "method": USDCTransactionMethod.send, + "change": -1000000, + "balance": 0, + "tx_metadata": transactionSenderUsdcAccount, + } + ] + + populate_mock_db(db, test_entries_with_transaction) + + with db.scoped_session() as session: + process_payment_router_tx_details( + session=session, + tx_info=tx_response, + tx_sig=tx_sig_str, + timestamp=datetime.now(), + challenge_event_bus=challenge_event_bus, + ) + + # Expect original transaction to have been removed + transaction_record = ( + session.query(USDCTransactionsHistory) + .filter(USDCTransactionsHistory.signature == "existingWithdrawal") + .filter(USDCTransactionsHistory.user_bank == trackOwnerUserBank) + .first() + ) + assert transaction_record is None + + +# Recovery transaction is for less than the original, should update original +# to be the difference +# def test_process_payment_router_tx_details_transfer_partial_recovery( +# app, +# ): +# tx_response = mock_valid_transfer_without_purchase_single_recipient_tx +# with app.app_context(): +# db = get_db() + +# transaction = tx_response.value.transaction.transaction + +# tx_sig_str = str(transaction.signatures[0]) + +# challenge_event_bus = create_autospec(ChallengeEventBus) + +# populate_mock_db(db, test_entries) + +# with db.scoped_session() as session: +# process_payment_router_tx_details( +# session=session, +# tx_info=tx_response, +# tx_sig=tx_sig_str, +# timestamp=datetime.now(), +# challenge_event_bus=challenge_event_bus, +# ) + +# # Expect no purchase record +# purchase = ( +# session.query(USDCPurchase) +# .filter(USDCPurchase.signature == tx_sig_str) +# .first() +# ) +# assert purchase is None + +# # We do still expect the transfers to get indexed, but as regular transfers +# transaction_record = ( +# session.query(USDCTransactionsHistory) +# .filter(USDCTransactionsHistory.signature == tx_sig_str) +# .filter(USDCTransactionsHistory.user_bank == trackOwnerUserBank) +# .first() +# ) +# assert transaction_record is not None +# assert transaction_record.user_bank == trackOwnerUserBank +# # Regular transfer, not a purchase +# assert transaction_record.transaction_type == USDCTransactionType.transfer +# assert transaction_record.method == USDCTransactionMethod.receive +# assert transaction_record.change == 1000000 +# # For transfers, the metadata is the source address +# assert transaction_record.tx_metadata == transactionSenderAddress + + +# Recovery transaction doesn't match the most recent outbound transfer. Should index +# as a regular inbound transfer +# def test_process_payment_router_tx_details_transfer_recovery_no_match( +# app, +# ): +# tx_response = mock_valid_transfer_without_purchase_single_recipient_tx +# with app.app_context(): +# db = get_db() + +# transaction = tx_response.value.transaction.transaction + +# tx_sig_str = str(transaction.signatures[0]) + +# challenge_event_bus = create_autospec(ChallengeEventBus) + +# populate_mock_db(db, test_entries) + +# with db.scoped_session() as session: +# process_payment_router_tx_details( +# session=session, +# tx_info=tx_response, +# tx_sig=tx_sig_str, +# timestamp=datetime.now(), +# challenge_event_bus=challenge_event_bus, +# ) + +# # Expect no purchase record +# purchase = ( +# session.query(USDCPurchase) +# .filter(USDCPurchase.signature == tx_sig_str) +# .first() +# ) +# assert purchase is None + +# # We do still expect the transfers to get indexed, but as regular transfers +# transaction_record = ( +# session.query(USDCTransactionsHistory) +# .filter(USDCTransactionsHistory.signature == tx_sig_str) +# .filter(USDCTransactionsHistory.user_bank == trackOwnerUserBank) +# .first() +# ) +# assert transaction_record is not None +# assert transaction_record.user_bank == trackOwnerUserBank +# # Regular transfer, not a purchase +# assert transaction_record.transaction_type == USDCTransactionType.transfer +# assert transaction_record.method == USDCTransactionMethod.receive +# assert transaction_record.change == 1000000 +# # For transfers, the metadata is the source address +# assert transaction_record.tx_metadata == transactionSenderAddress + + def test_process_payment_router_tx_details_valid_purchase_with_pay_extra(app): tx_response = mock_valid_track_purchase_single_recipient_pay_extra_tx with app.app_context(): diff --git a/packages/discovery-provider/src/tasks/index_payment_router.py b/packages/discovery-provider/src/tasks/index_payment_router.py index ae94730152c..c24670cced6 100644 --- a/packages/discovery-provider/src/tasks/index_payment_router.py +++ b/packages/discovery-provider/src/tasks/index_payment_router.py @@ -1,8 +1,9 @@ import concurrent.futures +import enum import time from datetime import datetime from decimal import Decimal -from typing import List, Optional, Tuple, TypedDict, Union, cast +from typing import List, Optional, Tuple, TypedDict, cast from redis import Redis from solders.instruction import CompiledInstruction @@ -92,6 +93,8 @@ # The amount of USDC that represents one USD cent USDC_PER_USD_CENT = 10000 +RECOVERY_MEMO_STRING = "recovery" + # Used to limit tx history if needed MIN_SLOT = int(shared_config["solana"]["payment_router_min_slot"]) INITIAL_FETCH_SIZE = 10 @@ -109,7 +112,7 @@ ROUTE_INSTRUCTION_RECEIVER_ACCOUNTS_START_INDEX = 3 -class ReceiverUserAccount(TypedDict): +class UserIdBankAccount(TypedDict): user_id: int user_bank_account: str @@ -123,6 +126,17 @@ class PurchaseMetadataDict(TypedDict): content_owner_id: int +class RouteTransactionMemoType(str, enum.Enum): + purchase = "purchase" + recovery = "recovery" + unknown = "unknown" + + +class RouteTransactionMemo(TypedDict): + type: RouteTransactionMemoType + metadata: PurchaseMetadataDict | None + + def check_config(): if not all([WAUDIO_MINT_PUBKEY, USDC_MINT_PUBKEY, PAYMENT_ROUTER_PUBKEY]): logger.error( @@ -182,11 +196,15 @@ def get_tx_in_db(session: Session, tx_sig: str) -> bool: return exists -def get_purchase_metadata_from_memo( +def parse_route_transaction_memo( session: Session, memos: List[str], timestamp: datetime -) -> Union[PurchaseMetadataDict, None]: - """Checks the list of memos for one matching the format of a purchase's content_metadata, and then uses that content_metadata to find the stream_conditions associated with that content to get the price""" +) -> RouteTransactionMemo: + """Checks the list of memos for one matching a format of a purchase's content_metadata, and then uses that content_metadata to find the stream_conditions associated with that content to get the price""" for memo in memos: + if memo == RECOVERY_MEMO_STRING: + return RouteTransactionMemo( + type=RouteTransactionMemoType.recovery, metadata=None + ) try: content_metadata = memo.split(":") if len(content_metadata) == 4: @@ -244,14 +262,17 @@ def get_purchase_metadata_from_memo( and isinstance(splits, dict) and content_owner_id is not None ): - return { - "type": type, - "id": id, - "price": price * USDC_PER_USD_CENT, - "splits": splits, - "purchaser_user_id": purchaser_user_id, - "content_owner_id": content_owner_id, - } + return RouteTransactionMemo( + type=RouteTransactionMemoType.purchase, + metadata={ + "type": type, + "id": id, + "price": price * USDC_PER_USD_CENT, + "splits": splits, + "purchaser_user_id": purchaser_user_id, + "content_owner_id": content_owner_id, + }, + ) else: logger.error( f"index_payment_router.py | Couldn't find relevant price for {content_metadata}" @@ -264,8 +285,8 @@ def get_purchase_metadata_from_memo( logger.debug( f"index_payment_router.py | Ignoring memo, failed to parse content metadata: {memo}, Error: {e}" ) - logger.error("index_payment_router.py | Failed to find any content metadata") - return None + logger.error("index_payment_router.py | Failed to find known memo format") + return RouteTransactionMemo(type=RouteTransactionMemoType.unknown, metadata=None) def validate_purchase( @@ -289,7 +310,8 @@ def validate_purchase( def index_purchase( session: Session, - receiver_user_accounts: List[ReceiverUserAccount], + sender_user_account: UserIdBankAccount | None, + receiver_user_accounts: List[UserIdBankAccount], receiver_accounts: List[str], balance_changes: dict[str, BalanceChange], purchase_metadata: PurchaseMetadataDict, @@ -314,7 +336,7 @@ def index_purchase( content_id=purchase_metadata["id"], ) logger.debug( - f"index_payment_router.py | Creating usdc_purchase for purchase {usdc_purchase}" + f"index_payment_router.py | tx: {tx_sig} | Creating usdc_purchase for purchase {usdc_purchase}" ) session.add(usdc_purchase) @@ -333,60 +355,217 @@ def index_purchase( ) session.add(usdc_tx_received) logger.debug( - f"index_payment_router.py | Creating usdc_tx_history received tx for purchase {usdc_tx_received}" + f"index_payment_router.py | tx: {tx_sig} | Created usdc_tx_history received tx for purchase {usdc_tx_received}" + ) + # If the sender for this transaction was a user bank, index the transaction for that + # user bank as well + if sender_user_account is not None: + balance_change = balance_changes[sender_user_account["user_bank_account"]] + usdc_tx_sent = USDCTransactionsHistory( + user_bank=sender_user_account["user_bank_account"], + slot=slot, + signature=tx_sig, + transaction_type=USDCTransactionType.purchase_content, + method=USDCTransactionMethod.send, + transaction_created_at=timestamp, + change=Decimal(balance_change["change"]), + balance=Decimal(balance_change["post_balance"]), + tx_metadata=str(purchase_metadata["content_owner_id"]), + ) + session.add(usdc_tx_sent) + logger.debug( + f"index_payment_router.py | tx: {tx_sig} | Created usdc_tx_history sent tx for purchase {usdc_tx_sent}" + ) + + +def attempt_index_recovery_transfer( + session: Session, + sender_account: str, + sender_user_account: UserIdBankAccount | None, + receiver_user_accounts: List[UserIdBankAccount], + balance_changes: dict[str, BalanceChange], + tx_sig: str, +): + if sender_user_account is None: + raise Exception( + f"Sender account for recovery cannot be a user bank: {sender_user_account}" + ) + if len(receiver_user_accounts) != 1: + raise Exception( + f"Recovery transfer must have exactly one receiver account. Received: {','.join([a.user_bank_account for a in receiver_user_accounts])}" + ) + + receiver_user_account = receiver_user_accounts[0] + receiver_bank_account = receiver_user_account["user_bank_account"] + receiver_balance_change = balance_changes[receiver_bank_account]["change"] + + # Find the previous transaction in the other direction (receiver -> sender) + # So that we can modify it + previous_transaction = ( + session.query(USDCTransactionsHistory) + .filter( + and_( + USDCTransactionsHistory.user_bank == receiver_bank_account, + USDCTransactionsHistory.method == USDCTransactionMethod.send, + USDCTransactionsHistory.tx_metadata == sender_account, + ) + ) + .order_by(desc(USDCTransactionsHistory.transaction_created_at)) + .first() + ) + if previous_transaction is None: + raise Exception( + f"Failed to find matching previous transcation from {receiver_bank_account} to {sender_account}" + ) + + # Previous transaction is completely undone by this recovery + if -previous_transaction.change <= receiver_balance_change: + logger.info( + f"index_payment_router.py | tx: {tx_sig} | Removing previous transaction {previous_transaction.signature}." + ) + session.delete(previous_transaction) + else: + # Previous transaction is partially undone by this recovery + # Modifying the object returned by the query will update the DB + # when the session is flushed + previous_transaction.change = ( + previous_transaction.change + receiver_balance_change + ) + logger.info( + f"index_payment_router.py | tx: {tx_sig} | Found partial recovery for {previous_transaction.signature}. New change value is {previous_transaction.change}." + ) + + +def index_transfer( + session: Session, + sender_account: str, + sender_user_account: UserIdBankAccount | None, + receiver_user_accounts: List[UserIdBankAccount], + receiver_accounts: List[str], + balance_changes: dict[str, BalanceChange], + slot: int, + timestamp: datetime, + tx_sig: str, +): + for user_account in receiver_user_accounts: + balance_change = balance_changes[user_account["user_bank_account"]] + usdc_tx_received = USDCTransactionsHistory( + user_bank=user_account["user_bank_account"], + slot=slot, + signature=tx_sig, + transaction_type=USDCTransactionType.transfer, + method=USDCTransactionMethod.receive, + transaction_created_at=timestamp, + change=Decimal(balance_change["change"]), + balance=Decimal(balance_change["post_balance"]), + tx_metadata=sender_account, + ) + session.add(usdc_tx_received) + logger.debug( + f"index_payment_router.py | tx: {tx_sig} | Creating transfer received tx {usdc_tx_received}" + ) + if sender_user_account is not None: + balance_change = balance_changes[sender_user_account["user_bank_account"]] + usdc_tx_received = USDCTransactionsHistory( + user_bank=sender_user_account["user_bank_account"], + slot=slot, + signature=tx_sig, + transaction_type=USDCTransactionType.transfer, + method=USDCTransactionMethod.send, + transaction_created_at=timestamp, + change=Decimal(balance_change["change"]), + balance=Decimal(balance_change["post_balance"]), + # TODO: Payment router account? comma separated list of receiver accounts? + tx_metadata=",".join(receiver_accounts), + ) + session.add(usdc_tx_received) + logger.debug( + f"index_payment_router.py | tx: {tx_sig} | Creating transfer received tx {usdc_tx_received}" ) def validate_and_index_usdc_transfers( session: Session, sender_account: str, - receiver_user_accounts: List[ReceiverUserAccount], + sender_user_account: UserIdBankAccount | None, + receiver_user_accounts: List[UserIdBankAccount], receiver_accounts: List[str], balance_changes: dict[str, BalanceChange], - purchase_metadata: Union[PurchaseMetadataDict, None], + memo: RouteTransactionMemo, slot: int, timestamp: datetime, tx_sig: str, ): """Checks if the transaction is a valid purchase and if so creates the purchase record. Otherwise, indexes a transfer.""" - if purchase_metadata is not None and validate_purchase( - purchase_metadata=purchase_metadata, balance_changes=balance_changes + if memo["type"] is RouteTransactionMemoType.purchase and validate_purchase( + purchase_metadata=memo["metadata"], balance_changes=balance_changes ): index_purchase( session=session, + sender_user_account=sender_user_account, receiver_user_accounts=receiver_user_accounts, receiver_accounts=receiver_accounts, balance_changes=balance_changes, - purchase_metadata=purchase_metadata, + purchase_metadata=memo["metadata"], slot=slot, timestamp=timestamp, tx_sig=tx_sig, ) # For invalid purchases or transfers not related to a purchase, we'll index - # it as a regular transfer, though it will always show as being sent from - # the payment router PDA - # TODO: We _could_ receive the actual sender from the first tranfer instruction here - # and use that instead. - else: - # TODO: Detect transfers _out_ of user banks to payment router and index them - # here as "sent" transactions - for user_account in receiver_user_accounts: - balance_change = balance_changes[user_account["user_bank_account"]] - usdc_tx_received = USDCTransactionsHistory( - user_bank=user_account["user_bank_account"], - slot=slot, - signature=tx_sig, - transaction_type=USDCTransactionType.transfer, - method=USDCTransactionMethod.receive, - transaction_created_at=timestamp, - change=Decimal(balance_change["change"]), - balance=Decimal(balance_change["post_balance"]), - tx_metadata=sender_account, + # it as a regular transfer from the sender_account. + elif memo["type"] is RouteTransactionMemoType.recovery: + try: + attempt_index_recovery_transfer( + session=session, + sender_account=sender_account, + sender_user_account=sender_user_account, + receiver_user_accounts=receiver_user_accounts, + balance_changes=balance_changes, + tx_sig=tx_sig, ) - session.add(usdc_tx_received) - logger.debug( - f"index_payment_router.py | Creating transfer received tx {usdc_tx_received}" + except: + index_transfer( + session=session, + sender_account=sender_account, + sender_user_account=sender_user_account, + receiver_user_accounts=receiver_user_accounts, + receiver_accounts=receiver_accounts, + balance_changes=balance_changes, + slot=slot, + timestamp=timestamp, + tx_sig=tx_sig, ) + else: + index_transfer( + session=session, + sender_account=sender_account, + sender_user_account=sender_user_account, + receiver_user_accounts=receiver_user_accounts, + receiver_accounts=receiver_accounts, + balance_changes=balance_changes, + slot=slot, + timestamp=timestamp, + tx_sig=tx_sig, + ) + + +def find_sender_account_from_balance_changes( + balance_changes: dict[str, BalanceChange], + receiver_accounts: List[str], +): + """Finds the sender account from the balance changes and receiver accounts""" + total_sent = 0 + for account in receiver_accounts: + if account in balance_changes: + total_sent += balance_changes[account]["change"] + return next( + ( + account + for account, change in balance_changes.items() + if change.amount == -total_sent + ), + None, + ) def process_route_instruction( @@ -411,26 +590,44 @@ def process_route_instruction( get_account_index(instruction, ROUTE_INSTRUCTION_PAYMENT_ROUTER_PDA_INDEX) ] - sender_address = account_keys[ + route_source_address = account_keys[ get_account_index(instruction, ROUTE_INSTRUCTION_SENDER_INDEX) ] - is_audio = sender_address == PAYMENT_ROUTER_WAUDIO_ATA_ADDRESS - is_usdc = sender_address == PAYMENT_ROUTER_USDC_ATA_ADDRESS + is_audio = route_source_address == PAYMENT_ROUTER_WAUDIO_ATA_ADDRESS + is_usdc = route_source_address == PAYMENT_ROUTER_USDC_ATA_ADDRESS + + balance_changes = get_solana_tx_token_balance_changes( + account_keys=account_keys, meta=meta + ) + + # Detect the account which sent tokens _into_ payment router, as that's + # our real source account + sender_account = ( + find_sender_account_from_balance_changes( + balance_changes=balance_changes, receiver_accounts=receiver_accounts + ) + or sender_pda_account + ) user_id_accounts = [] if is_audio: - logger.info( - "index_payment_router.py | $AUDIO payment router transactions are not yet indexed. Skipping balance refresh" + logger.warn( + f"index_payment_router.py | tx: {tx_sig} | $AUDIO payment router transactions are not yet indexed. Skipping balance refresh" ) elif is_usdc: + search_accounts = ( + receiver_accounts + [sender_account] + if sender_account is not None + else receiver_accounts + ) user_id_accounts = ( session.query(User.user_id, USDCUserBankAccount.bank_account) .join( USDCUserBankAccount, and_( - USDCUserBankAccount.bank_account.in_(receiver_accounts), + USDCUserBankAccount.bank_account.in_(search_accounts), USDCUserBankAccount.ethereum_address == User.wallet, ), ) @@ -438,80 +635,76 @@ def process_route_instruction( ) # Payment Router recipients may not be Audius user banks if not user_id_accounts: - logger.info( - f"index_payment_router.py | No receiver accounts are user banks | {str(receiver_accounts)}" + logger.warn( + f"index_payment_router.py | tx: {tx_sig} | No sender or receiver accounts are user banks | {str(search_accounts)}" ) else: logger.error( - f"index_payment_router.py | Unrecognized source ATA {sender_address}. Expected AUDIO={PAYMENT_ROUTER_WAUDIO_ATA_ADDRESS} or USDC={PAYMENT_ROUTER_USDC_ATA_ADDRESS}" + f"index_payment_router.py | tx: {tx_sig} | Unrecognized source ATA {route_source_address}. Expected AUDIO={PAYMENT_ROUTER_WAUDIO_ATA_ADDRESS} or USDC={PAYMENT_ROUTER_USDC_ATA_ADDRESS}" ) return - receiver_user_accounts: List[ReceiverUserAccount] = [] - for user_id_account in user_id_accounts: - if user_id_account[1] in receiver_accounts: + receiver_user_accounts: List[UserIdBankAccount] = [] + sender_user_account: UserIdBankAccount | None = None + for user_id, bank_account in user_id_accounts: + if bank_account in receiver_accounts: receiver_user_accounts.append( { - "user_id": user_id_account[0], - "user_bank_account": user_id_account[1], + "user_id": user_id, + "user_bank_account": bank_account, } ) - - balance_changes = get_solana_tx_token_balance_changes( - account_keys=account_keys, meta=meta - ) - - # TODO: Adapt this to detecting external transfers via payment router. It would require that - # we have already parsed the sender of the TransferChecked instruction _before_ the Route - # instruction and have passed that into this function. Then we could create a - # TranscationType.Transfer w/ the external addresess listed. + elif bank_account == sender_account: + sender_user_account = { + "user_id": user_id, + "user_bank_account": bank_account, + } if is_audio: logger.warning( - "index_payment_router.py | $AUDIO payment router transactions are not yet indexed. Skipping instruction indexing." + f"index_payment_router.py | tx: {tx_sig} | $AUDIO payment router transactions are not yet indexed. Skipping instruction indexing." ) elif is_usdc: - # Index as a purchase of some content - purchase_metadata = get_purchase_metadata_from_memo( + memo = parse_route_transaction_memo( session=session, memos=memos, timestamp=timestamp ) validate_and_index_usdc_transfers( session=session, - sender_account=sender_pda_account, + sender_account=sender_account, + sender_user_account=sender_user_account, receiver_user_accounts=receiver_user_accounts, receiver_accounts=receiver_accounts, balance_changes=balance_changes, - purchase_metadata=purchase_metadata, + memo=memo, slot=slot, timestamp=timestamp, tx_sig=tx_sig, ) - # We can have a USDC payment router transfer with no purchase attached - if purchase_metadata is None: + # If the memo had purchase information, dispatch challenge events + if memo["type"] is RouteTransactionMemoType.purchase: logger.info( - f"index_payment_router.py | No purchase metadata found on {tx_sig}" + f"index_payment_router.py | tx: {tx_sig} | Purchase memo found. Dispatching challenge events" + ) + purchase_metadata = memo["metadata"] + sender_user_id = purchase_metadata["purchaser_user_id"] + amount = int(round(purchase_metadata["price"]) / 10**USDC_DECIMALS) + challenge_event_bus.dispatch( + ChallengeEvent.audio_matching_buyer, + slot, + sender_user_id, + {"track_id": purchase_metadata["id"], "amount": amount}, + ) + challenge_event_bus.dispatch( + ChallengeEvent.audio_matching_seller, + slot, + purchase_metadata["content_owner_id"], + { + "track_id": purchase_metadata["id"], + "sender_user_id": sender_user_id, + "amount": amount, + }, ) - return - - sender_user_id = purchase_metadata["purchaser_user_id"] - amount = int(round(purchase_metadata["price"]) / 10**USDC_DECIMALS) - challenge_event_bus.dispatch( - ChallengeEvent.audio_matching_buyer, - slot, - sender_user_id, - {"track_id": purchase_metadata["id"], "amount": amount}, - ) - challenge_event_bus.dispatch( - ChallengeEvent.audio_matching_seller, - slot, - purchase_metadata["content_owner_id"], - { - "track_id": purchase_metadata["id"], - "sender_user_id": sender_user_id, - "amount": amount, - }, - ) def process_payment_router_tx_details( @@ -524,22 +717,22 @@ def process_payment_router_tx_details( logger.debug(f"index_payment_router.py | Processing tx={tx_info.to_json()}") result = tx_info.value if not result: - logger.error("index_payment_router.py | No result") + logger.error("index_payment_router.py | tx: {tx_sig} | No result") return meta = result.transaction.meta if not meta: - logger.error("index_payment_router.py | No result meta") + logger.error("index_payment_router.py | tx: {tx_sig} | No result meta") return error = meta.err if error: logger.error( - f"index_payment_router.py | Skipping error transaction from chain {tx_info}" + f"index_payment_router.py | tx: {tx_sig} | Skipping error transaction from chain {tx_info}" ) return transaction = result.transaction.transaction if not hasattr(transaction, "message"): logger.error( - f"index_payment_router.py | No transaction message found {transaction}" + f"index_payment_router.py | tx: {tx_sig} | No transaction message found {transaction}" ) return @@ -551,13 +744,11 @@ def process_payment_router_tx_details( instruction = get_valid_instruction(tx_message, meta, PAYMENT_ROUTER_ADDRESS) if instruction is None: - logger.error(f"index_payment_router.py | {tx_sig} No Valid instruction found") + logger.error( + f"index_payment_router.py | tx: {tx_sig} | No Valid instruction found" + ) return - # TODO: Parse existing TransferChecked instruction first to get the address which sent - # money _into_ the payment router. This will be necessary to correctly index - # external transfers via Payment Router from a userbank. - if has_route_instruction: process_route_instruction( session=session, From 3be8c17e524d2b7f0a744825bbd5fd2df4726cc8 Mon Sep 17 00:00:00 2001 From: Randy Schott <1815175+schottra@users.noreply.github.com> Date: Fri, 19 Jan 2024 11:24:48 -0500 Subject: [PATCH 03/13] finish implementation and tests for recovery --- .../tasks/test_index_payment_router.py | 236 ++++++++++-------- .../src/tasks/index_payment_router.py | 22 +- 2 files changed, 144 insertions(+), 114 deletions(-) diff --git a/packages/discovery-provider/integration_tests/tasks/test_index_payment_router.py b/packages/discovery-provider/integration_tests/tasks/test_index_payment_router.py index 93aebbe0122..13db7051a56 100644 --- a/packages/discovery-provider/integration_tests/tasks/test_index_payment_router.py +++ b/packages/discovery-provider/integration_tests/tasks/test_index_payment_router.py @@ -1,4 +1,5 @@ from datetime import datetime +import logging from unittest.mock import call, create_autospec from payment_router_mock_transactions import ( @@ -192,14 +193,12 @@ def test_process_payment_router_tx_details_transfer_transfer_without_purchase( assert transaction_record.method == USDCTransactionMethod.receive assert transaction_record.change == 1000000 # For transfers, the metadata is the source address - assert transaction_record.tx_metadata == transactionSenderAddress + assert transaction_record.tx_metadata == transactionSenderUsdcAccount # Should revert the most recent outbound transaction to the sending address # of the recovery transaction -def test_process_payment_router_tx_details_transfer_recovery( - app, -): +def test_process_payment_router_tx_details_transfer_recovery(app, caplog): tx_response = mock_valid_transfer_single_recipient_recovery_tx with app.app_context(): db = get_db() @@ -246,104 +245,125 @@ def test_process_payment_router_tx_details_transfer_recovery( # Recovery transaction is for less than the original, should update original # to be the difference -# def test_process_payment_router_tx_details_transfer_partial_recovery( -# app, -# ): -# tx_response = mock_valid_transfer_without_purchase_single_recipient_tx -# with app.app_context(): -# db = get_db() - -# transaction = tx_response.value.transaction.transaction - -# tx_sig_str = str(transaction.signatures[0]) - -# challenge_event_bus = create_autospec(ChallengeEventBus) - -# populate_mock_db(db, test_entries) - -# with db.scoped_session() as session: -# process_payment_router_tx_details( -# session=session, -# tx_info=tx_response, -# tx_sig=tx_sig_str, -# timestamp=datetime.now(), -# challenge_event_bus=challenge_event_bus, -# ) - -# # Expect no purchase record -# purchase = ( -# session.query(USDCPurchase) -# .filter(USDCPurchase.signature == tx_sig_str) -# .first() -# ) -# assert purchase is None - -# # We do still expect the transfers to get indexed, but as regular transfers -# transaction_record = ( -# session.query(USDCTransactionsHistory) -# .filter(USDCTransactionsHistory.signature == tx_sig_str) -# .filter(USDCTransactionsHistory.user_bank == trackOwnerUserBank) -# .first() -# ) -# assert transaction_record is not None -# assert transaction_record.user_bank == trackOwnerUserBank -# # Regular transfer, not a purchase -# assert transaction_record.transaction_type == USDCTransactionType.transfer -# assert transaction_record.method == USDCTransactionMethod.receive -# assert transaction_record.change == 1000000 -# # For transfers, the metadata is the source address -# assert transaction_record.tx_metadata == transactionSenderAddress - - -# Recovery transaction doesn't match the most recent outbound transfer. Should index +def test_process_payment_router_tx_details_transfer_partial_recovery( + app, +): + tx_response = mock_valid_transfer_single_recipient_recovery_tx + with app.app_context(): + db = get_db() + + transaction = tx_response.value.transaction.transaction + + tx_sig_str = str(transaction.signatures[0]) + + challenge_event_bus = create_autospec(ChallengeEventBus) + + test_entries_with_transaction = test_entries.copy() + test_entries_with_transaction["usdc_transactions_history"] = [ + { + "user_bank": trackOwnerUserBank, + "signature": "existingWithdrawal", + "transaction_type": USDCTransactionType.transfer, + "method": USDCTransactionMethod.send, + "change": -2000000, + "balance": 0, + "tx_metadata": transactionSenderUsdcAccount, + } + ] + + populate_mock_db(db, test_entries_with_transaction) + + with db.scoped_session() as session: + process_payment_router_tx_details( + session=session, + tx_info=tx_response, + tx_sig=tx_sig_str, + timestamp=datetime.now(), + challenge_event_bus=challenge_event_bus, + ) + + # Expect original transaction to be modified + transaction_record = ( + session.query(USDCTransactionsHistory) + .filter(USDCTransactionsHistory.signature == "existingWithdrawal") + .filter(USDCTransactionsHistory.user_bank == trackOwnerUserBank) + .filter(USDCTransactionsHistory.tx_metadata == transactionSenderUsdcAccount) + .first() + ) + # Recovery transaction was for half of the original amount, expect the difference + assert transaction_record.change == -1000000 + + +# Recovery transaction doesn't match the most recent outbound transfer (different addresses). Should index # as a regular inbound transfer -# def test_process_payment_router_tx_details_transfer_recovery_no_match( -# app, -# ): -# tx_response = mock_valid_transfer_without_purchase_single_recipient_tx -# with app.app_context(): -# db = get_db() - -# transaction = tx_response.value.transaction.transaction - -# tx_sig_str = str(transaction.signatures[0]) - -# challenge_event_bus = create_autospec(ChallengeEventBus) - -# populate_mock_db(db, test_entries) - -# with db.scoped_session() as session: -# process_payment_router_tx_details( -# session=session, -# tx_info=tx_response, -# tx_sig=tx_sig_str, -# timestamp=datetime.now(), -# challenge_event_bus=challenge_event_bus, -# ) - -# # Expect no purchase record -# purchase = ( -# session.query(USDCPurchase) -# .filter(USDCPurchase.signature == tx_sig_str) -# .first() -# ) -# assert purchase is None - -# # We do still expect the transfers to get indexed, but as regular transfers -# transaction_record = ( -# session.query(USDCTransactionsHistory) -# .filter(USDCTransactionsHistory.signature == tx_sig_str) -# .filter(USDCTransactionsHistory.user_bank == trackOwnerUserBank) -# .first() -# ) -# assert transaction_record is not None -# assert transaction_record.user_bank == trackOwnerUserBank -# # Regular transfer, not a purchase -# assert transaction_record.transaction_type == USDCTransactionType.transfer -# assert transaction_record.method == USDCTransactionMethod.receive -# assert transaction_record.change == 1000000 -# # For transfers, the metadata is the source address -# assert transaction_record.tx_metadata == transactionSenderAddress +def test_process_payment_router_tx_details_transfer_recovery_address_mismatch( + app, +): + tx_response = mock_valid_transfer_single_recipient_recovery_tx + with app.app_context(): + db = get_db() + + transaction = tx_response.value.transaction.transaction + + tx_sig_str = str(transaction.signatures[0]) + + challenge_event_bus = create_autospec(ChallengeEventBus) + + test_entries_with_transaction = test_entries.copy() + test_entries_with_transaction["usdc_transactions_history"] = [ + { + "user_bank": trackOwnerUserBank, + "signature": "existingWithdrawal", + "transaction_type": USDCTransactionType.transfer, + "method": USDCTransactionMethod.send, + "change": -1000000, + "balance": 0, + "tx_metadata": "randomOtherAccount", + } + ] + + populate_mock_db(db, test_entries_with_transaction) + + with db.scoped_session() as session: + process_payment_router_tx_details( + session=session, + tx_info=tx_response, + tx_sig=tx_sig_str, + timestamp=datetime.now(), + challenge_event_bus=challenge_event_bus, + ) + + # Original transaction should remain unchanged + existing_transaction_record = ( + session.query(USDCTransactionsHistory) + .filter(USDCTransactionsHistory.signature == "existingWithdrawal") + .filter(USDCTransactionsHistory.user_bank == trackOwnerUserBank) + .filter(USDCTransactionsHistory.tx_metadata == "randomOtherAccount") + .first() + ) + assert existing_transaction_record is not None + assert existing_transaction_record.change == -1000000 + assert existing_transaction_record.balance == 0 + assert existing_transaction_record.method == USDCTransactionMethod.send + assert ( + existing_transaction_record.transaction_type == USDCTransactionType.transfer + ) + + # Expect new transaction to have been added + transaction_record = ( + session.query(USDCTransactionsHistory) + .filter(USDCTransactionsHistory.signature == tx_sig_str) + .filter(USDCTransactionsHistory.user_bank == trackOwnerUserBank) + .first() + ) + assert transaction_record is not None + assert transaction_record.user_bank == trackOwnerUserBank + # Regular transfer, not a purchase + assert transaction_record.transaction_type == USDCTransactionType.transfer + assert transaction_record.method == USDCTransactionMethod.receive + assert transaction_record.change == 1000000 + # For transfers, the metadata is the source address + assert transaction_record.tx_metadata == transactionSenderUsdcAccount def test_process_payment_router_tx_details_valid_purchase_with_pay_extra(app): @@ -576,7 +596,7 @@ def test_process_payment_router_tx_details_invalid_purchase_bad_splits(app): assert owner_transaction_record.method == USDCTransactionMethod.receive assert owner_transaction_record.change == 1000000 # For transfers, the metadata is the source address - assert owner_transaction_record.tx_metadata == transactionSenderAddress + assert owner_transaction_record.tx_metadata == transactionSenderUsdcAccount third_party_transaction_record = ( session.query(USDCTransactionsHistory) @@ -593,7 +613,9 @@ def test_process_payment_router_tx_details_invalid_purchase_bad_splits(app): assert third_party_transaction_record.method == USDCTransactionMethod.receive assert third_party_transaction_record.change == 500000 # For transfers, the metadata is the source address - assert third_party_transaction_record.tx_metadata == transactionSenderAddress + assert ( + third_party_transaction_record.tx_metadata == transactionSenderUsdcAccount + ) # Transaction is for the correct amount, but one of the splits is missing @@ -642,7 +664,7 @@ def test_process_payment_router_tx_details_invalid_purchase_missing_splits(app): assert owner_transaction_record.method == USDCTransactionMethod.receive assert owner_transaction_record.change == 2000000 # For transfers, the metadata is the source address - assert owner_transaction_record.tx_metadata == transactionSenderAddress + assert owner_transaction_record.tx_metadata == transactionSenderUsdcAccount def test_process_payment_router_tx_details_transfer_multiple_users_without_purchase( @@ -691,7 +713,7 @@ def test_process_payment_router_tx_details_transfer_multiple_users_without_purch assert owner_transaction_record.method == USDCTransactionMethod.receive assert owner_transaction_record.change == 1000000 # For transfers, the metadata is the source address - assert owner_transaction_record.tx_metadata == transactionSenderAddress + assert owner_transaction_record.tx_metadata == transactionSenderUsdcAccount third_party_transaction_record = ( session.query(USDCTransactionsHistory) @@ -708,7 +730,9 @@ def test_process_payment_router_tx_details_transfer_multiple_users_without_purch assert third_party_transaction_record.method == USDCTransactionMethod.receive assert third_party_transaction_record.change == 1000000 # For transfers, the metadata is the source address - assert third_party_transaction_record.tx_metadata == transactionSenderAddress + assert ( + third_party_transaction_record.tx_metadata == transactionSenderUsdcAccount + ) def test_process_payment_router_txs_details_create_challenge_events_for_purchase(app): diff --git a/packages/discovery-provider/src/tasks/index_payment_router.py b/packages/discovery-provider/src/tasks/index_payment_router.py index c24670cced6..0e8beecd2ee 100644 --- a/packages/discovery-provider/src/tasks/index_payment_router.py +++ b/packages/discovery-provider/src/tasks/index_payment_router.py @@ -130,6 +130,7 @@ class RouteTransactionMemoType(str, enum.Enum): purchase = "purchase" recovery = "recovery" unknown = "unknown" + none = "none" class RouteTransactionMemo(TypedDict): @@ -200,6 +201,8 @@ def parse_route_transaction_memo( session: Session, memos: List[str], timestamp: datetime ) -> RouteTransactionMemo: """Checks the list of memos for one matching a format of a purchase's content_metadata, and then uses that content_metadata to find the stream_conditions associated with that content to get the price""" + if len(memos) == 0: + return RouteTransactionMemo(type=RouteTransactionMemoType.none, metadata=None) for memo in memos: if memo == RECOVERY_MEMO_STRING: return RouteTransactionMemo( @@ -278,14 +281,14 @@ def parse_route_transaction_memo( f"index_payment_router.py | Couldn't find relevant price for {content_metadata}" ) else: - logger.debug( + logger.info( f"index_payment_router.py | Ignoring memo, no content metadata found: {memo}" ) except (ValueError, KeyError) as e: - logger.debug( + logger.info( f"index_payment_router.py | Ignoring memo, failed to parse content metadata: {memo}, Error: {e}" ) - logger.error("index_payment_router.py | Failed to find known memo format") + logger.info("index_payment_router.py | Failed to find known memo format") return RouteTransactionMemo(type=RouteTransactionMemoType.unknown, metadata=None) @@ -386,7 +389,7 @@ def attempt_index_recovery_transfer( balance_changes: dict[str, BalanceChange], tx_sig: str, ): - if sender_user_account is None: + if sender_user_account is not None: raise Exception( f"Sender account for recovery cannot be a user bank: {sender_user_account}" ) @@ -464,6 +467,7 @@ def index_transfer( logger.debug( f"index_payment_router.py | tx: {tx_sig} | Creating transfer received tx {usdc_tx_received}" ) + # If sender was a user bank, index a single transfer with a list of recipients if sender_user_account is not None: balance_change = balance_changes[sender_user_account["user_bank_account"]] usdc_tx_received = USDCTransactionsHistory( @@ -475,7 +479,6 @@ def index_transfer( transaction_created_at=timestamp, change=Decimal(balance_change["change"]), balance=Decimal(balance_change["post_balance"]), - # TODO: Payment router account? comma separated list of receiver accounts? tx_metadata=",".join(receiver_accounts), ) session.add(usdc_tx_received) @@ -523,7 +526,10 @@ def validate_and_index_usdc_transfers( balance_changes=balance_changes, tx_sig=tx_sig, ) - except: + except Exception as e: + logger.warn( + f"index_payment_router.py | tx: {tx_sig} | Failed to index recovery transfer. Will index as plain transfer. Exception received was: {e}." + ) index_transfer( session=session, sender_account=sender_account, @@ -561,8 +567,8 @@ def find_sender_account_from_balance_changes( return next( ( account - for account, change in balance_changes.items() - if change.amount == -total_sent + for account, balance_change in balance_changes.items() + if balance_change["change"] == -total_sent ), None, ) From c2a04d13d45140e1e956db6e39f6001a7dd16594 Mon Sep 17 00:00:00 2001 From: Randy Schott <1815175+schottra@users.noreply.github.com> Date: Fri, 19 Jan 2024 11:38:15 -0500 Subject: [PATCH 04/13] add tests for user bank as routing source --- .../tasks/payment_router_mock_transactions.py | 389 ++++++++++++++++++ .../tasks/test_index_payment_router.py | 137 +++++- 2 files changed, 525 insertions(+), 1 deletion(-) diff --git a/packages/discovery-provider/integration_tests/tasks/payment_router_mock_transactions.py b/packages/discovery-provider/integration_tests/tasks/payment_router_mock_transactions.py index e3d817a1a48..07f980264a3 100644 --- a/packages/discovery-provider/integration_tests/tasks/payment_router_mock_transactions.py +++ b/packages/discovery-provider/integration_tests/tasks/payment_router_mock_transactions.py @@ -214,6 +214,206 @@ ) ) + +# Routes $1.00 from a user bank to a single recipient w/ memo for track purchase +mock_valid_track_purchase_from_user_bank_single_recipient_tx = GetTransactionResp.from_json( + json.dumps( + { + "jsonrpc": "2.0", + "result": { + "slot": 190957, + "transaction": { + "signatures": [ + "5wPxiuLSF3MzXZt9XG99UEPNdxs8DtE2vWKezrB6zuMCrkMBJx6iU7xw5icaowpfgj96iLGnAgEAaBNSbneWdbZw" + ], + "message": { + "header": { + "numRequiredSignatures": 1, + "numReadonlySignedAccounts": 0, + "numReadonlyUnsignedAccounts": 5, + }, + "accountKeys": [ + "HXLN9UWwAjMPgHaFZDfgabT79SmLSdTeu2fUha2xHz9W", + # User bank as source address + "38YSndmPWVF3UdzczbB3UMYUgPQtZrgvvPVHa3M4yQVX", + "A76eNhRrfdy6WfMoQf4ALasMxzRWHajH4TrVuX2NUjZT", + "7gfRGGdp89N9g3mCsZjaGmDDRdcTnZh9u3vYyBab2tRy", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "26Q7gP8UfkDzi7GMFEQxTJaNJ8D2ybCUjex58M5MLu8y", + "Memo1UhkJRfHyvLMcVucJwxXeuD728EqVDDwQDxFMNo", + "apaySbqV1XAmuiGszeN4NyWrXkkMrnuJVoNhzmS1AMa", + "G231EZsMoCNBiQKP5quEeAM3oG516Zspirjnh7ywP71i", + ], + "recentBlockhash": "6D65tSU7pjSmFvSj9qK2W2bjkESw4XZebeNmgA1rCqnF", + "instructions": [ + { + "programIdIndex": 4, + "accounts": [1, 5, 2, 0], + "data": "hYECWfYe8vYqs", + "stackHeight": None, + }, + { + "programIdIndex": 6, + "accounts": [0], + # "track:1:1:2" + "data": "VsoUab4LQ4yax8R", + "stackHeight": None, + }, + { + "programIdIndex": 7, + "accounts": [2, 8, 4, 3], + "data": "BQD4GnQPrhbq6Y9NJLnwDUziXhfF6BjkLYFbnKZH", + "stackHeight": None, + }, + ], + "addressTableLookups": [], + }, + }, + "meta": { + "err": None, + "status": {"Ok": None}, + "fee": 5000, + "preBalances": [ + 8420804160, + 2039280, + 2039280, + 2039280, + 929020800, + 1461600, + 119712000, + 1141440, + 946560, + ], + "postBalances": [ + 8420799160, + 2039280, + 2039280, + 2039280, + 929020800, + 1461600, + 119712000, + 1141440, + 946560, + ], + "innerInstructions": [ + { + "index": 2, + "instructions": [ + { + "programIdIndex": 4, + "accounts": [2, 3, 8, 8], + "data": "3YKuzAsyicvj", + "stackHeight": 2, + } + ], + } + ], + "logMessages": [ + "Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA invoke [1]", + "Program log: Instruction: TransferChecked", + "Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA consumed 6173 of 600000 compute units", + "Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA success", + "Program Memo1UhkJRfHyvLMcVucJwxXeuD728EqVDDwQDxFMNo invoke [1]", + "Program Memo1UhkJRfHyvLMcVucJwxXeuD728EqVDDwQDxFMNo consumed 480 of 593827 compute units", + "Program Memo1UhkJRfHyvLMcVucJwxXeuD728EqVDDwQDxFMNo success", + "Program apaySbqV1XAmuiGszeN4NyWrXkkMrnuJVoNhzmS1AMa invoke [1]", + "Program log: Instruction: Route", + "Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA invoke [2]", + "Program log: Instruction: Transfer", + "Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA consumed 4728 of 576902 compute units", + "Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA success", + "Program log: All transfers complete!", + "Program apaySbqV1XAmuiGszeN4NyWrXkkMrnuJVoNhzmS1AMa consumed 21782 of 593347 compute units", + "Program apaySbqV1XAmuiGszeN4NyWrXkkMrnuJVoNhzmS1AMa success", + ], + "preTokenBalances": [ + { + "accountIndex": 1, + "mint": "26Q7gP8UfkDzi7GMFEQxTJaNJ8D2ybCUjex58M5MLu8y", + "uiTokenAmount": { + "uiAmount": 10.0, + "decimals": 6, + "amount": "10000000", + "uiAmountString": "10.0", + }, + "owner": "HXLN9UWwAjMPgHaFZDfgabT79SmLSdTeu2fUha2xHz9W", + "programId": "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + }, + { + "accountIndex": 2, + "mint": "26Q7gP8UfkDzi7GMFEQxTJaNJ8D2ybCUjex58M5MLu8y", + "uiTokenAmount": { + "uiAmount": None, + "decimals": 6, + "amount": "0", + "uiAmountString": "0", + }, + "owner": "G231EZsMoCNBiQKP5quEeAM3oG516Zspirjnh7ywP71i", + "programId": "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + }, + { + "accountIndex": 3, + "mint": "26Q7gP8UfkDzi7GMFEQxTJaNJ8D2ybCUjex58M5MLu8y", + "uiTokenAmount": { + "uiAmount": 0, + "decimals": 6, + "amount": "0", + "uiAmountString": "0", + }, + "owner": "7vKR1WSmyHvBmCvKPZBiN66PHZqYQbXw51SZdwtVd9Dt", + "programId": "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + }, + ], + "postTokenBalances": [ + { + "accountIndex": 1, + "mint": "26Q7gP8UfkDzi7GMFEQxTJaNJ8D2ybCUjex58M5MLu8y", + "uiTokenAmount": { + "uiAmount": 9.0, + "decimals": 6, + "amount": "9000000", + "uiAmountString": "9.0", + }, + "owner": "HXLN9UWwAjMPgHaFZDfgabT79SmLSdTeu2fUha2xHz9W", + "programId": "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + }, + { + "accountIndex": 2, + "mint": "26Q7gP8UfkDzi7GMFEQxTJaNJ8D2ybCUjex58M5MLu8y", + "uiTokenAmount": { + "uiAmount": None, + "decimals": 6, + "amount": "0", + "uiAmountString": "0", + }, + "owner": "G231EZsMoCNBiQKP5quEeAM3oG516Zspirjnh7ywP71i", + "programId": "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + }, + { + "accountIndex": 3, + "mint": "26Q7gP8UfkDzi7GMFEQxTJaNJ8D2ybCUjex58M5MLu8y", + "uiTokenAmount": { + "uiAmount": 1.0, + "decimals": 6, + "amount": "1000000", + "uiAmountString": "1.0", + }, + "owner": "7vKR1WSmyHvBmCvKPZBiN66PHZqYQbXw51SZdwtVd9Dt", + "programId": "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + }, + ], + "rewards": [], + "loadedAddresses": {"writable": [], "readonly": []}, + "computeUnitsConsumed": 28435, + }, + "version": 0, + "blockTime": 1701922096, + }, + "id": 0, + } + ) +) + # Routes $1.00 to a single recipient w/ NO memo for track purchase mock_valid_transfer_without_purchase_single_recipient_tx = GetTransactionResp.from_json( json.dumps( @@ -402,6 +602,195 @@ ) ) +# Routes $1.00 from a user bank to a single recipient w/ NO memo for track purchase +mock_valid_transfer_from_user_bank_without_purchase_single_recipient_tx = GetTransactionResp.from_json( + json.dumps( + { + "jsonrpc": "2.0", + "result": { + "slot": 190957, + "transaction": { + "signatures": [ + "5wPxiuLSF3MzXZt9XG99UEPNdxs8DtE2vWKezrB6zuMCrkMBJx6iU7xw5icaowpfgj96iLGnAgEAaBNSbneWdbZw" + ], + "message": { + "header": { + "numRequiredSignatures": 1, + "numReadonlySignedAccounts": 0, + "numReadonlyUnsignedAccounts": 5, + }, + "accountKeys": [ + "HXLN9UWwAjMPgHaFZDfgabT79SmLSdTeu2fUha2xHz9W", + # User bank as source address + "38YSndmPWVF3UdzczbB3UMYUgPQtZrgvvPVHa3M4yQVX", + "A76eNhRrfdy6WfMoQf4ALasMxzRWHajH4TrVuX2NUjZT", + "7gfRGGdp89N9g3mCsZjaGmDDRdcTnZh9u3vYyBab2tRy", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + "26Q7gP8UfkDzi7GMFEQxTJaNJ8D2ybCUjex58M5MLu8y", + "Memo1UhkJRfHyvLMcVucJwxXeuD728EqVDDwQDxFMNo", + "apaySbqV1XAmuiGszeN4NyWrXkkMrnuJVoNhzmS1AMa", + "G231EZsMoCNBiQKP5quEeAM3oG516Zspirjnh7ywP71i", + ], + "recentBlockhash": "6D65tSU7pjSmFvSj9qK2W2bjkESw4XZebeNmgA1rCqnF", + "instructions": [ + { + "programIdIndex": 4, + "accounts": [1, 5, 2, 0], + "data": "hYECWfYe8vYqs", + "stackHeight": None, + }, + { + "programIdIndex": 7, + "accounts": [2, 8, 4, 3], + "data": "BQD4GnQPrhbq6Y9NJLnwDUziXhfF6BjkLYFbnKZH", + "stackHeight": None, + }, + ], + "addressTableLookups": [], + }, + }, + "meta": { + "err": None, + "status": {"Ok": None}, + "fee": 5000, + "preBalances": [ + 8420804160, + 2039280, + 2039280, + 2039280, + 929020800, + 1461600, + 119712000, + 1141440, + 946560, + ], + "postBalances": [ + 8420799160, + 2039280, + 2039280, + 2039280, + 929020800, + 1461600, + 119712000, + 1141440, + 946560, + ], + "innerInstructions": [ + { + "index": 2, + "instructions": [ + { + "programIdIndex": 4, + "accounts": [2, 3, 8, 8], + "data": "3YKuzAsyicvj", + "stackHeight": 2, + } + ], + } + ], + "logMessages": [ + "Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA invoke [1]", + "Program log: Instruction: TransferChecked", + "Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA consumed 6173 of 600000 compute units", + "Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA success", + "Program apaySbqV1XAmuiGszeN4NyWrXkkMrnuJVoNhzmS1AMa invoke [1]", + "Program log: Instruction: Route", + "Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA invoke [2]", + "Program log: Instruction: Transfer", + "Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA consumed 4728 of 576902 compute units", + "Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA success", + "Program log: All transfers complete!", + "Program apaySbqV1XAmuiGszeN4NyWrXkkMrnuJVoNhzmS1AMa consumed 21782 of 593347 compute units", + "Program apaySbqV1XAmuiGszeN4NyWrXkkMrnuJVoNhzmS1AMa success", + ], + "preTokenBalances": [ + { + "accountIndex": 1, + "mint": "26Q7gP8UfkDzi7GMFEQxTJaNJ8D2ybCUjex58M5MLu8y", + "uiTokenAmount": { + "uiAmount": 10.0, + "decimals": 6, + "amount": "10000000", + "uiAmountString": "10.0", + }, + "owner": "HXLN9UWwAjMPgHaFZDfgabT79SmLSdTeu2fUha2xHz9W", + "programId": "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + }, + { + "accountIndex": 2, + "mint": "26Q7gP8UfkDzi7GMFEQxTJaNJ8D2ybCUjex58M5MLu8y", + "uiTokenAmount": { + "uiAmount": None, + "decimals": 6, + "amount": "0", + "uiAmountString": "0", + }, + "owner": "G231EZsMoCNBiQKP5quEeAM3oG516Zspirjnh7ywP71i", + "programId": "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + }, + { + "accountIndex": 3, + "mint": "26Q7gP8UfkDzi7GMFEQxTJaNJ8D2ybCUjex58M5MLu8y", + "uiTokenAmount": { + "uiAmount": 0, + "decimals": 6, + "amount": "0", + "uiAmountString": "0", + }, + "owner": "7vKR1WSmyHvBmCvKPZBiN66PHZqYQbXw51SZdwtVd9Dt", + "programId": "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + }, + ], + "postTokenBalances": [ + { + "accountIndex": 1, + "mint": "26Q7gP8UfkDzi7GMFEQxTJaNJ8D2ybCUjex58M5MLu8y", + "uiTokenAmount": { + "uiAmount": 9.0, + "decimals": 6, + "amount": "9000000", + "uiAmountString": "9.0", + }, + "owner": "HXLN9UWwAjMPgHaFZDfgabT79SmLSdTeu2fUha2xHz9W", + "programId": "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + }, + { + "accountIndex": 2, + "mint": "26Q7gP8UfkDzi7GMFEQxTJaNJ8D2ybCUjex58M5MLu8y", + "uiTokenAmount": { + "uiAmount": None, + "decimals": 6, + "amount": "0", + "uiAmountString": "0", + }, + "owner": "G231EZsMoCNBiQKP5quEeAM3oG516Zspirjnh7ywP71i", + "programId": "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + }, + { + "accountIndex": 3, + "mint": "26Q7gP8UfkDzi7GMFEQxTJaNJ8D2ybCUjex58M5MLu8y", + "uiTokenAmount": { + "uiAmount": 1.0, + "decimals": 6, + "amount": "1000000", + "uiAmountString": "1.0", + }, + "owner": "7vKR1WSmyHvBmCvKPZBiN66PHZqYQbXw51SZdwtVd9Dt", + "programId": "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", + }, + ], + "rewards": [], + "loadedAddresses": {"writable": [], "readonly": []}, + "computeUnitsConsumed": 28435, + }, + "version": 0, + "blockTime": 1701922096, + }, + "id": 0, + } + ) +) + # Routes $2.50 to a single recipient w/ memo for track purchase mock_valid_track_purchase_single_recipient_pay_extra_tx = GetTransactionResp.from_json( json.dumps( diff --git a/packages/discovery-provider/integration_tests/tasks/test_index_payment_router.py b/packages/discovery-provider/integration_tests/tasks/test_index_payment_router.py index 13db7051a56..7a11ee75314 100644 --- a/packages/discovery-provider/integration_tests/tasks/test_index_payment_router.py +++ b/packages/discovery-provider/integration_tests/tasks/test_index_payment_router.py @@ -4,6 +4,7 @@ from payment_router_mock_transactions import ( mock_failed_track_purchase_single_recipient_tx, + mock_valid_track_purchase_from_user_bank_single_recipient_tx, mock_invalid_track_purchase_bad_PDA_account_single_recipient_tx, mock_invalid_track_purchase_insufficient_split_tx, mock_invalid_track_purchase_missing_split_tx, @@ -14,6 +15,7 @@ mock_valid_track_purchase_single_recipient_tx, mock_valid_transfer_without_purchase_multi_recipient_tx, mock_valid_transfer_without_purchase_single_recipient_tx, + mock_valid_transfer_from_user_bank_without_purchase_single_recipient_tx, mock_valid_transfer_single_recipient_recovery_tx, ) @@ -147,7 +149,73 @@ def test_process_payment_router_tx_details_valid_purchase(app): assert transaction_record.tx_metadata == str(trackBuyerId) -def test_process_payment_router_tx_details_transfer_transfer_without_purchase( +def test_process_payment_router_tx_details_valid_purchase_from_user_bank(app): + tx_response = mock_valid_track_purchase_from_user_bank_single_recipient_tx + with app.app_context(): + db = get_db() + + transaction = tx_response.value.transaction.transaction + + tx_sig_str = str(transaction.signatures[0]) + + challenge_event_bus = create_autospec(ChallengeEventBus) + + populate_mock_db(db, test_entries) + + with db.scoped_session() as session: + process_payment_router_tx_details( + session=session, + tx_info=tx_response, + tx_sig=tx_sig_str, + timestamp=datetime.now(), + challenge_event_bus=challenge_event_bus, + ) + + purchase = ( + session.query(USDCPurchase) + .filter(USDCPurchase.signature == tx_sig_str) + .first() + ) + assert purchase is not None + assert purchase.seller_user_id == 1 + assert purchase.buyer_user_id == 2 + assert purchase.amount == 1000000 + assert purchase.extra_amount == 0 + assert purchase.content_type == PurchaseType.track + assert purchase.content_id == 1 + + seller_transaction_record = ( + session.query(USDCTransactionsHistory) + .filter(USDCTransactionsHistory.signature == tx_sig_str) + .filter(USDCTransactionsHistory.user_bank == trackOwnerUserBank) + .first() + ) + assert seller_transaction_record is not None + assert ( + seller_transaction_record.transaction_type + == USDCTransactionType.purchase_content + ) + assert seller_transaction_record.method == USDCTransactionMethod.receive + assert seller_transaction_record.change == 1000000 + assert seller_transaction_record.tx_metadata == str(trackBuyerId) + + buyer_transaction_record = ( + session.query(USDCTransactionsHistory) + .filter(USDCTransactionsHistory.signature == tx_sig_str) + .filter(USDCTransactionsHistory.user_bank == trackBuyerUserBank) + .first() + ) + assert buyer_transaction_record is not None + assert ( + buyer_transaction_record.transaction_type + == USDCTransactionType.purchase_content + ) + assert buyer_transaction_record.method == USDCTransactionMethod.send + assert buyer_transaction_record.change == -1000000 + assert buyer_transaction_record.tx_metadata == str(trackOwnerId) + + +def test_process_payment_router_tx_details_transfer_without_purchase( app, ): tx_response = mock_valid_transfer_without_purchase_single_recipient_tx @@ -196,6 +264,73 @@ def test_process_payment_router_tx_details_transfer_transfer_without_purchase( assert transaction_record.tx_metadata == transactionSenderUsdcAccount +def test_process_payment_router_tx_details_transfer_from_user_bank_without_purchase( + app, +): + tx_response = ( + mock_valid_transfer_from_user_bank_without_purchase_single_recipient_tx + ) + with app.app_context(): + db = get_db() + + transaction = tx_response.value.transaction.transaction + + tx_sig_str = str(transaction.signatures[0]) + + challenge_event_bus = create_autospec(ChallengeEventBus) + + populate_mock_db(db, test_entries) + + with db.scoped_session() as session: + process_payment_router_tx_details( + session=session, + tx_info=tx_response, + tx_sig=tx_sig_str, + timestamp=datetime.now(), + challenge_event_bus=challenge_event_bus, + ) + + # Expect no purchase record + purchase = ( + session.query(USDCPurchase) + .filter(USDCPurchase.signature == tx_sig_str) + .first() + ) + assert purchase is None + + # We do still expect the transfers to get indexed, but as regular transfers + receiver_transaction_record = ( + session.query(USDCTransactionsHistory) + .filter(USDCTransactionsHistory.signature == tx_sig_str) + .filter(USDCTransactionsHistory.user_bank == trackOwnerUserBank) + .first() + ) + assert receiver_transaction_record is not None + assert receiver_transaction_record.user_bank == trackOwnerUserBank + # Regular transfer, not a purchase + assert ( + receiver_transaction_record.transaction_type == USDCTransactionType.transfer + ) + assert receiver_transaction_record.method == USDCTransactionMethod.receive + assert receiver_transaction_record.change == 1000000 + assert receiver_transaction_record.tx_metadata == trackBuyerUserBank + + sender_transaction_record = ( + session.query(USDCTransactionsHistory) + .filter(USDCTransactionsHistory.signature == tx_sig_str) + .filter(USDCTransactionsHistory.user_bank == trackBuyerUserBank) + .first() + ) + assert sender_transaction_record is not None + # Regular transfer, not a purchase + assert ( + sender_transaction_record.transaction_type == USDCTransactionType.transfer + ) + assert sender_transaction_record.method == USDCTransactionMethod.send + assert sender_transaction_record.change == -1000000 + assert sender_transaction_record.tx_metadata == trackOwnerUserBank + + # Should revert the most recent outbound transaction to the sending address # of the recovery transaction def test_process_payment_router_tx_details_transfer_recovery(app, caplog): From 2b36899e66054565cd44053e5131e67532ff1a7b Mon Sep 17 00:00:00 2001 From: Randy Schott <1815175+schottra@users.noreply.github.com> Date: Fri, 19 Jan 2024 11:53:05 -0500 Subject: [PATCH 05/13] add test for user bank ignoring payment router --- .../tasks/test_index_payment_router.py | 3 +- .../tasks/test_index_user_bank.py | 51 +++++++++++++++++++ 2 files changed, 52 insertions(+), 2 deletions(-) diff --git a/packages/discovery-provider/integration_tests/tasks/test_index_payment_router.py b/packages/discovery-provider/integration_tests/tasks/test_index_payment_router.py index 7a11ee75314..12db3f470e0 100644 --- a/packages/discovery-provider/integration_tests/tasks/test_index_payment_router.py +++ b/packages/discovery-provider/integration_tests/tasks/test_index_payment_router.py @@ -1,5 +1,4 @@ from datetime import datetime -import logging from unittest.mock import call, create_autospec from payment_router_mock_transactions import ( @@ -333,7 +332,7 @@ def test_process_payment_router_tx_details_transfer_from_user_bank_without_purch # Should revert the most recent outbound transaction to the sending address # of the recovery transaction -def test_process_payment_router_tx_details_transfer_recovery(app, caplog): +def test_process_payment_router_tx_details_transfer_recovery(app): tx_response = mock_valid_transfer_single_recipient_recovery_tx with app.app_context(): db = get_db() diff --git a/packages/discovery-provider/integration_tests/tasks/test_index_user_bank.py b/packages/discovery-provider/integration_tests/tasks/test_index_user_bank.py index 3172ec0b78b..106da76a811 100644 --- a/packages/discovery-provider/integration_tests/tasks/test_index_user_bank.py +++ b/packages/discovery-provider/integration_tests/tasks/test_index_user_bank.py @@ -16,6 +16,10 @@ mock_valid_transfer_without_purchase_tx, ) +from payment_router_mock_transactions import ( + mock_valid_transfer_from_user_bank_without_purchase_single_recipient_tx, +) + from integration_tests.utils import populate_mock_db from src.challenges.challenge_event import ChallengeEvent from src.challenges.challenge_event_bus import ChallengeEventBus @@ -632,6 +636,53 @@ def test_process_user_bank_txs_details_skip_unknown_instructions(app): assert transaction_record is None +def test_process_user_bank_txs_details_ignore_payment_router_transfers(app): + # Payment router transaction, should be ignored by user bank indexer + tx_response = ( + mock_valid_transfer_from_user_bank_without_purchase_single_recipient_tx + ) + with app.app_context(): + db = get_db() + redis = get_redis() + solana_client_manager_mock = create_autospec(SolanaClientManager) + + transaction = tx_response.value.transaction.transaction + + tx_sig_str = str(transaction.signatures[0]) + + challenge_event_bus = create_autospec(ChallengeEventBus) + + populate_mock_db(db, test_entries) + + with db.scoped_session() as session: + process_user_bank_tx_details( + solana_client_manager=solana_client_manager_mock, + redis=redis, + session=session, + tx_info=tx_response, + tx_sig=tx_sig_str, + timestamp=datetime.now(), + challenge_event_bus=challenge_event_bus, + ) + + # We do still expect the transfers to get indexed, but as regular transfers + receiver_transaction_record = ( + session.query(USDCTransactionsHistory) + .filter(USDCTransactionsHistory.signature == tx_sig_str) + .filter(USDCTransactionsHistory.user_bank == trackOwnerUserBank) + .first() + ) + assert receiver_transaction_record is None + + sender_transaction_record = ( + session.query(USDCTransactionsHistory) + .filter(USDCTransactionsHistory.signature == tx_sig_str) + .filter(USDCTransactionsHistory.user_bank == trackBuyerUserBank) + .first() + ) + assert sender_transaction_record is None + + # TODO: https://linear.app/audius/issue/PAY-2314/add-user-bank-indexer-tests-for-audio-operations From 20302b7423be24b1adb735c94817513b238aed84 Mon Sep 17 00:00:00 2001 From: Randy Schott <1815175+schottra@users.noreply.github.com> Date: Fri, 19 Jan 2024 16:49:00 -0500 Subject: [PATCH 06/13] Fix test cases and handling for user bank -> payment router --- .../tasks/payment_router_mock_transactions.py | 308 +++++++++++------- .../tasks/test_index_payment_router.py | 2 + .../tasks/test_index_user_bank.py | 2 +- .../src/tasks/index_user_bank.py | 25 ++ 4 files changed, 219 insertions(+), 118 deletions(-) diff --git a/packages/discovery-provider/integration_tests/tasks/payment_router_mock_transactions.py b/packages/discovery-provider/integration_tests/tasks/payment_router_mock_transactions.py index 07f980264a3..9acc83536ea 100644 --- a/packages/discovery-provider/integration_tests/tasks/payment_router_mock_transactions.py +++ b/packages/discovery-provider/integration_tests/tasks/payment_router_mock_transactions.py @@ -4,8 +4,8 @@ # Notes about the transactions below to make things easier: # - accountKeys[1] is the account which sent the money into payment router -# - accountKeys[3] is the first recipient account. Values up to the "Tokenkeg..." account are all recipients -# - The "data" field in the memo transaction is a base58 encoded string consisting of +# - For transactions where the sender is not a user bank, accountKeys[3] is the first recipient account. Values up to the "Tokenkeg..." account are all recipients +# - The "data" field in the memo transaction is a base58 encoded string consisting of either "recovery" or # :::. # - `meta.preTokenBalances` and `meta.postTokenBalances` determine the amount paid # for the content. The negative balance change in the "sending" account should match the sum of @@ -221,126 +221,165 @@ { "jsonrpc": "2.0", "result": { - "slot": 190957, + "slot": 26227, "transaction": { "signatures": [ - "5wPxiuLSF3MzXZt9XG99UEPNdxs8DtE2vWKezrB6zuMCrkMBJx6iU7xw5icaowpfgj96iLGnAgEAaBNSbneWdbZw" + "4NgZHuZ1LNLRoJJi43B1iHbDKWwRhRnVKJJHYc2MPV5ArTkpc5sfcyd7Rgv5JZnXqEdSZdT9bpJ76myqHgCG2pfU" ], "message": { "header": { "numRequiredSignatures": 1, "numReadonlySignedAccounts": 0, - "numReadonlyUnsignedAccounts": 5, + "numReadonlyUnsignedAccounts": 10, }, "accountKeys": [ - "HXLN9UWwAjMPgHaFZDfgabT79SmLSdTeu2fUha2xHz9W", - # User bank as source address + "HunCgdP91aVeoh8J7cbKTcFRoUwwhHwqYqVVLVkkqQjg", + # Sender user bank "38YSndmPWVF3UdzczbB3UMYUgPQtZrgvvPVHa3M4yQVX", + "5B6jwaPf4mMdwyRjD9x7H9y8fFR5iwvZK64Ri3xkSXGh", "A76eNhRrfdy6WfMoQf4ALasMxzRWHajH4TrVuX2NUjZT", + # Receiver user bank "7gfRGGdp89N9g3mCsZjaGmDDRdcTnZh9u3vYyBab2tRy", - "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", - "26Q7gP8UfkDzi7GMFEQxTJaNJ8D2ybCUjex58M5MLu8y", - "Memo1UhkJRfHyvLMcVucJwxXeuD728EqVDDwQDxFMNo", + "11111111111111111111111111111111", + "7vKR1WSmyHvBmCvKPZBiN66PHZqYQbXw51SZdwtVd9Dt", "apaySbqV1XAmuiGszeN4NyWrXkkMrnuJVoNhzmS1AMa", "G231EZsMoCNBiQKP5quEeAM3oG516Zspirjnh7ywP71i", + "KeccakSecp256k11111111111111111111111111111", + "Memo1UhkJRfHyvLMcVucJwxXeuD728EqVDDwQDxFMNo", + "Sysvar1nstructions1111111111111111111111111", + "SysvarRent111111111111111111111111111111111", + "testHKV1B56fbvop4w6f2cTGEub9dRQ2Euta5VmqdX9", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", ], - "recentBlockhash": "6D65tSU7pjSmFvSj9qK2W2bjkESw4XZebeNmgA1rCqnF", + "recentBlockhash": "8CTRtyjoo275ZwHuq9nBEdYtTwG6jP7VxMwxv2yeiS76", "instructions": [ { - "programIdIndex": 4, - "accounts": [1, 5, 2, 0], - "data": "hYECWfYe8vYqs", + "programIdIndex": 9, + "accounts": [], + "data": "H4eCheRWTZDTCFYXS5QxY3AH8Yb7NdpeFkwCufrAPtfgLzs1F3cLufX635zDuSDUdZkpWMrub9ppgfEWsDPiSBsxnP8oEboGFbQn8h7VeMoJqAtPhCHcBfPtcrKFzgrW1YY2HQ18wZCXK4e71NH5dWaqkKvUYCCq6jADEQP87zRpNzprDiTQUTWwPi7yed3MyuRzP", "stackHeight": None, }, { - "programIdIndex": 6, - "accounts": [0], - # "track:1:1:2" - "data": "VsoUab4LQ4yax8R", + "programIdIndex": 13, + "accounts": [0, 1, 3, 2, 6, 12, 11, 5, 14], + "data": "7PxDYbdhSHWR3DpxB6uwMyCTgjXC", "stackHeight": None, }, { "programIdIndex": 7, - "accounts": [2, 8, 4, 3], - "data": "BQD4GnQPrhbq6Y9NJLnwDUziXhfF6BjkLYFbnKZH", + "accounts": [3, 8, 14, 4], + "data": "BQD4GnQPrhbq6Y9NJLgwWxBtNf2BL8pPGkJm9rAs", + "stackHeight": None, + }, + { + "programIdIndex": 10, + "accounts": [0], + # "track:1:1:2" + "data": "VsoUab4LQ4yax8R", "stackHeight": None, }, ], - "addressTableLookups": [], }, }, "meta": { "err": None, "status": {"Ok": None}, - "fee": 5000, + "fee": 10000, "preBalances": [ - 8420804160, + 19991459680, 2039280, + 953520, 2039280, 2039280, - 929020800, - 1461600, - 119712000, + 1, + 0, 1141440, 946560, + 1, + 119712000, + 0, + 1009200, + 1141440, + 929020800, ], "postBalances": [ - 8420799160, + 19991449680, 2039280, + 953520, 2039280, 2039280, - 929020800, - 1461600, - 119712000, + 1, + 0, 1141440, 946560, + 1, + 119712000, + 0, + 1009200, + 1141440, + 929020800, ], "innerInstructions": [ + { + "index": 1, + "instructions": [ + { + "programIdIndex": 14, + "accounts": [1, 3, 6, 6], + "data": "3QCwqmHZ4mdq", + "stackHeight": 2, + } + ], + }, { "index": 2, "instructions": [ { - "programIdIndex": 4, - "accounts": [2, 3, 8, 8], - "data": "3YKuzAsyicvj", + "programIdIndex": 14, + "accounts": [3, 4, 8, 8], + "data": "3QCwqmHZ4mdq", "stackHeight": 2, } ], - } + }, ], "logMessages": [ - "Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA invoke [1]", - "Program log: Instruction: TransferChecked", - "Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA consumed 6173 of 600000 compute units", + "Program testHKV1B56fbvop4w6f2cTGEub9dRQ2Euta5VmqdX9 invoke [1]", + "Program log: Instruction: Transfer", + "Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA invoke [2]", + "Program log: Instruction: Transfer", + "Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA consumed 4728 of 777300 compute units", "Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA success", - "Program Memo1UhkJRfHyvLMcVucJwxXeuD728EqVDDwQDxFMNo invoke [1]", - "Program Memo1UhkJRfHyvLMcVucJwxXeuD728EqVDDwQDxFMNo consumed 480 of 593827 compute units", - "Program Memo1UhkJRfHyvLMcVucJwxXeuD728EqVDDwQDxFMNo success", + "Program testHKV1B56fbvop4w6f2cTGEub9dRQ2Euta5VmqdX9 consumed 27936 of 800000 compute units", + "Program testHKV1B56fbvop4w6f2cTGEub9dRQ2Euta5VmqdX9 success", "Program apaySbqV1XAmuiGszeN4NyWrXkkMrnuJVoNhzmS1AMa invoke [1]", "Program log: Instruction: Route", "Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA invoke [2]", "Program log: Instruction: Transfer", - "Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA consumed 4728 of 576902 compute units", + "Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA consumed 4728 of 755619 compute units", "Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA success", "Program log: All transfers complete!", - "Program apaySbqV1XAmuiGszeN4NyWrXkkMrnuJVoNhzmS1AMa consumed 21782 of 593347 compute units", + "Program apaySbqV1XAmuiGszeN4NyWrXkkMrnuJVoNhzmS1AMa consumed 21782 of 772064 compute units", "Program apaySbqV1XAmuiGszeN4NyWrXkkMrnuJVoNhzmS1AMa success", + "Program Memo1UhkJRfHyvLMcVucJwxXeuD728EqVDDwQDxFMNo invoke [1]", + "Program Memo1UhkJRfHyvLMcVucJwxXeuD728EqVDDwQDxFMNo consumed 480 of 750282 compute units", + "Program Memo1UhkJRfHyvLMcVucJwxXeuD728EqVDDwQDxFMNo success", ], "preTokenBalances": [ { "accountIndex": 1, "mint": "26Q7gP8UfkDzi7GMFEQxTJaNJ8D2ybCUjex58M5MLu8y", "uiTokenAmount": { - "uiAmount": 10.0, + "uiAmount": 7.0, "decimals": 6, - "amount": "10000000", - "uiAmountString": "10.0", + "amount": "7000000", + "uiAmountString": "7", }, - "owner": "HXLN9UWwAjMPgHaFZDfgabT79SmLSdTeu2fUha2xHz9W", + "owner": "7vKR1WSmyHvBmCvKPZBiN66PHZqYQbXw51SZdwtVd9Dt", "programId": "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", }, { - "accountIndex": 2, + "accountIndex": 3, "mint": "26Q7gP8UfkDzi7GMFEQxTJaNJ8D2ybCUjex58M5MLu8y", "uiTokenAmount": { "uiAmount": None, @@ -352,13 +391,13 @@ "programId": "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", }, { - "accountIndex": 3, + "accountIndex": 4, "mint": "26Q7gP8UfkDzi7GMFEQxTJaNJ8D2ybCUjex58M5MLu8y", "uiTokenAmount": { - "uiAmount": 0, + "uiAmount": 2.001, "decimals": 6, - "amount": "0", - "uiAmountString": "0", + "amount": "2001000", + "uiAmountString": "2.001", }, "owner": "7vKR1WSmyHvBmCvKPZBiN66PHZqYQbXw51SZdwtVd9Dt", "programId": "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", @@ -369,16 +408,16 @@ "accountIndex": 1, "mint": "26Q7gP8UfkDzi7GMFEQxTJaNJ8D2ybCUjex58M5MLu8y", "uiTokenAmount": { - "uiAmount": 9.0, + "uiAmount": 6.0, "decimals": 6, - "amount": "9000000", - "uiAmountString": "9.0", + "amount": "6000000", + "uiAmountString": "6", }, - "owner": "HXLN9UWwAjMPgHaFZDfgabT79SmLSdTeu2fUha2xHz9W", + "owner": "7vKR1WSmyHvBmCvKPZBiN66PHZqYQbXw51SZdwtVd9Dt", "programId": "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", }, { - "accountIndex": 2, + "accountIndex": 3, "mint": "26Q7gP8UfkDzi7GMFEQxTJaNJ8D2ybCUjex58M5MLu8y", "uiTokenAmount": { "uiAmount": None, @@ -390,13 +429,13 @@ "programId": "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", }, { - "accountIndex": 3, + "accountIndex": 4, "mint": "26Q7gP8UfkDzi7GMFEQxTJaNJ8D2ybCUjex58M5MLu8y", "uiTokenAmount": { - "uiAmount": 1.0, + "uiAmount": 3.001, "decimals": 6, - "amount": "1000000", - "uiAmountString": "1.0", + "amount": "3001000", + "uiAmountString": "3.001", }, "owner": "7vKR1WSmyHvBmCvKPZBiN66PHZqYQbXw51SZdwtVd9Dt", "programId": "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", @@ -404,10 +443,9 @@ ], "rewards": [], "loadedAddresses": {"writable": [], "readonly": []}, - "computeUnitsConsumed": 28435, + "computeUnitsConsumed": 50198, }, - "version": 0, - "blockTime": 1701922096, + "blockTime": 1701757366, }, "id": 0, } @@ -608,99 +646,135 @@ { "jsonrpc": "2.0", "result": { - "slot": 190957, + "slot": 23952, "transaction": { "signatures": [ - "5wPxiuLSF3MzXZt9XG99UEPNdxs8DtE2vWKezrB6zuMCrkMBJx6iU7xw5icaowpfgj96iLGnAgEAaBNSbneWdbZw" + "5w63hkGRYB95H7yuDTFxAw9json2EXVSNatQQ3444y3NHZPTQBT91G3iUeFXpPYyRouPdfDAeGTKa2oB4iDw8wHV" ], "message": { "header": { "numRequiredSignatures": 1, "numReadonlySignedAccounts": 0, - "numReadonlyUnsignedAccounts": 5, + "numReadonlyUnsignedAccounts": 9, }, "accountKeys": [ - "HXLN9UWwAjMPgHaFZDfgabT79SmLSdTeu2fUha2xHz9W", - # User bank as source address + "HunCgdP91aVeoh8J7cbKTcFRoUwwhHwqYqVVLVkkqQjg", + # source user bank "38YSndmPWVF3UdzczbB3UMYUgPQtZrgvvPVHa3M4yQVX", + "5B6jwaPf4mMdwyRjD9x7H9y8fFR5iwvZK64Ri3xkSXGh", "A76eNhRrfdy6WfMoQf4ALasMxzRWHajH4TrVuX2NUjZT", + # recipient account "7gfRGGdp89N9g3mCsZjaGmDDRdcTnZh9u3vYyBab2tRy", - "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", - "26Q7gP8UfkDzi7GMFEQxTJaNJ8D2ybCUjex58M5MLu8y", - "Memo1UhkJRfHyvLMcVucJwxXeuD728EqVDDwQDxFMNo", + "11111111111111111111111111111111", + "7vKR1WSmyHvBmCvKPZBiN66PHZqYQbXw51SZdwtVd9Dt", "apaySbqV1XAmuiGszeN4NyWrXkkMrnuJVoNhzmS1AMa", "G231EZsMoCNBiQKP5quEeAM3oG516Zspirjnh7ywP71i", + "KeccakSecp256k11111111111111111111111111111", + "Sysvar1nstructions1111111111111111111111111", + "SysvarRent111111111111111111111111111111111", + "testHKV1B56fbvop4w6f2cTGEub9dRQ2Euta5VmqdX9", + "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", ], - "recentBlockhash": "6D65tSU7pjSmFvSj9qK2W2bjkESw4XZebeNmgA1rCqnF", + "recentBlockhash": "8dq8k9h2yZk7nZomUzYYvAjVtFcux9sZEHa7te5mfFcv", "instructions": [ { - "programIdIndex": 4, - "accounts": [1, 5, 2, 0], - "data": "hYECWfYe8vYqs", + "programIdIndex": 9, + "accounts": [], + "data": "H4eCheRWTZDTCFYXS5QxY3AH8Yb7NdpeFkwCufrAPtiDBtroMe2WhDEphzWPJ6ThwujM8Cmg8zQ8sKxubdGatUu43ej2kNSNdUm46dvwGUcVxAtJRbiYh96WDSFiiydXDABtLJYPZLkTaiJ72cUaNcBep18Cy6T9NTe3crrUGgEPwZ4wXCujYCakP7MLUJ2C22bno", + "stackHeight": None, + }, + { + "programIdIndex": 12, + "accounts": [0, 1, 3, 2, 6, 11, 10, 5, 13], + "data": "7PxDYbdhSHWR3DpxB6uwMyCTgjXC", "stackHeight": None, }, { "programIdIndex": 7, - "accounts": [2, 8, 4, 3], - "data": "BQD4GnQPrhbq6Y9NJLnwDUziXhfF6BjkLYFbnKZH", + "accounts": [3, 8, 13, 4], + "data": "BQD4GnQPrhbq6Y9NJLgwWxBtNf2BL8pPGkJm9rAs", "stackHeight": None, }, ], - "addressTableLookups": [], }, }, "meta": { "err": None, "status": {"Ok": None}, - "fee": 5000, + "fee": 10000, "preBalances": [ - 8420804160, + 19991469680, 2039280, + 953520, 2039280, 2039280, - 929020800, - 1461600, - 119712000, + 1, + 0, 1141440, 946560, + 1, + 0, + 1009200, + 1141440, + 929020800, ], "postBalances": [ - 8420799160, + 19991459680, 2039280, + 953520, 2039280, 2039280, - 929020800, - 1461600, - 119712000, + 1, + 0, 1141440, 946560, + 1, + 0, + 1009200, + 1141440, + 929020800, ], "innerInstructions": [ + { + "index": 1, + "instructions": [ + { + "programIdIndex": 13, + "accounts": [1, 3, 6, 6], + "data": "3QCwqmHZ4mdq", + "stackHeight": 2, + } + ], + }, { "index": 2, "instructions": [ { - "programIdIndex": 4, - "accounts": [2, 3, 8, 8], - "data": "3YKuzAsyicvj", + "programIdIndex": 13, + "accounts": [3, 4, 8, 8], + "data": "3QCwqmHZ4mdq", "stackHeight": 2, } ], - } + }, ], "logMessages": [ - "Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA invoke [1]", - "Program log: Instruction: TransferChecked", - "Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA consumed 6173 of 600000 compute units", + "Program testHKV1B56fbvop4w6f2cTGEub9dRQ2Euta5VmqdX9 invoke [1]", + "Program log: Instruction: Transfer", + "Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA invoke [2]", + "Program log: Instruction: Transfer", + "Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA consumed 4728 of 577300 compute units", "Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA success", + "Program testHKV1B56fbvop4w6f2cTGEub9dRQ2Euta5VmqdX9 consumed 27936 of 600000 compute units", + "Program testHKV1B56fbvop4w6f2cTGEub9dRQ2Euta5VmqdX9 success", "Program apaySbqV1XAmuiGszeN4NyWrXkkMrnuJVoNhzmS1AMa invoke [1]", "Program log: Instruction: Route", "Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA invoke [2]", "Program log: Instruction: Transfer", - "Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA consumed 4728 of 576902 compute units", + "Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA consumed 4728 of 555619 compute units", "Program TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA success", "Program log: All transfers complete!", - "Program apaySbqV1XAmuiGszeN4NyWrXkkMrnuJVoNhzmS1AMa consumed 21782 of 593347 compute units", + "Program apaySbqV1XAmuiGszeN4NyWrXkkMrnuJVoNhzmS1AMa consumed 21782 of 572064 compute units", "Program apaySbqV1XAmuiGszeN4NyWrXkkMrnuJVoNhzmS1AMa success", ], "preTokenBalances": [ @@ -708,16 +782,16 @@ "accountIndex": 1, "mint": "26Q7gP8UfkDzi7GMFEQxTJaNJ8D2ybCUjex58M5MLu8y", "uiTokenAmount": { - "uiAmount": 10.0, + "uiAmount": 8.0, "decimals": 6, - "amount": "10000000", - "uiAmountString": "10.0", + "amount": "8000000", + "uiAmountString": "8", }, - "owner": "HXLN9UWwAjMPgHaFZDfgabT79SmLSdTeu2fUha2xHz9W", + "owner": "7vKR1WSmyHvBmCvKPZBiN66PHZqYQbXw51SZdwtVd9Dt", "programId": "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", }, { - "accountIndex": 2, + "accountIndex": 3, "mint": "26Q7gP8UfkDzi7GMFEQxTJaNJ8D2ybCUjex58M5MLu8y", "uiTokenAmount": { "uiAmount": None, @@ -729,13 +803,13 @@ "programId": "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", }, { - "accountIndex": 3, + "accountIndex": 4, "mint": "26Q7gP8UfkDzi7GMFEQxTJaNJ8D2ybCUjex58M5MLu8y", "uiTokenAmount": { - "uiAmount": 0, + "uiAmount": 1.001, "decimals": 6, - "amount": "0", - "uiAmountString": "0", + "amount": "1001000", + "uiAmountString": "1.001", }, "owner": "7vKR1WSmyHvBmCvKPZBiN66PHZqYQbXw51SZdwtVd9Dt", "programId": "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", @@ -746,16 +820,16 @@ "accountIndex": 1, "mint": "26Q7gP8UfkDzi7GMFEQxTJaNJ8D2ybCUjex58M5MLu8y", "uiTokenAmount": { - "uiAmount": 9.0, + "uiAmount": 7.0, "decimals": 6, - "amount": "9000000", - "uiAmountString": "9.0", + "amount": "7000000", + "uiAmountString": "7", }, - "owner": "HXLN9UWwAjMPgHaFZDfgabT79SmLSdTeu2fUha2xHz9W", + "owner": "7vKR1WSmyHvBmCvKPZBiN66PHZqYQbXw51SZdwtVd9Dt", "programId": "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", }, { - "accountIndex": 2, + "accountIndex": 3, "mint": "26Q7gP8UfkDzi7GMFEQxTJaNJ8D2ybCUjex58M5MLu8y", "uiTokenAmount": { "uiAmount": None, @@ -767,13 +841,13 @@ "programId": "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", }, { - "accountIndex": 3, + "accountIndex": 4, "mint": "26Q7gP8UfkDzi7GMFEQxTJaNJ8D2ybCUjex58M5MLu8y", "uiTokenAmount": { - "uiAmount": 1.0, + "uiAmount": 2.001, "decimals": 6, - "amount": "1000000", - "uiAmountString": "1.0", + "amount": "2001000", + "uiAmountString": "2.001", }, "owner": "7vKR1WSmyHvBmCvKPZBiN66PHZqYQbXw51SZdwtVd9Dt", "programId": "TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA", @@ -781,16 +855,16 @@ ], "rewards": [], "loadedAddresses": {"writable": [], "readonly": []}, - "computeUnitsConsumed": 28435, + "computeUnitsConsumed": 49718, }, - "version": 0, - "blockTime": 1701922096, + "blockTime": 1701755091, }, "id": 0, } ) ) + # Routes $2.50 to a single recipient w/ memo for track purchase mock_valid_track_purchase_single_recipient_pay_extra_tx = GetTransactionResp.from_json( json.dumps( diff --git a/packages/discovery-provider/integration_tests/tasks/test_index_payment_router.py b/packages/discovery-provider/integration_tests/tasks/test_index_payment_router.py index 12db3f470e0..f834760590c 100644 --- a/packages/discovery-provider/integration_tests/tasks/test_index_payment_router.py +++ b/packages/discovery-provider/integration_tests/tasks/test_index_payment_router.py @@ -426,6 +426,8 @@ def test_process_payment_router_tx_details_transfer_partial_recovery( ) # Recovery transaction was for half of the original amount, expect the difference assert transaction_record.change == -1000000 + # TODO: Need to also update the balance + # TODO: Consider for the case of recovering too much, we might just index it as a regular inbound transfer so that the balance is correct. # Recovery transaction doesn't match the most recent outbound transfer (different addresses). Should index diff --git a/packages/discovery-provider/integration_tests/tasks/test_index_user_bank.py b/packages/discovery-provider/integration_tests/tasks/test_index_user_bank.py index 106da76a811..0fd42c4a242 100644 --- a/packages/discovery-provider/integration_tests/tasks/test_index_user_bank.py +++ b/packages/discovery-provider/integration_tests/tasks/test_index_user_bank.py @@ -665,7 +665,7 @@ def test_process_user_bank_txs_details_ignore_payment_router_transfers(app): challenge_event_bus=challenge_event_bus, ) - # We do still expect the transfers to get indexed, but as regular transfers + # We expect no transactions to be logged for sender or receiver receiver_transaction_record = ( session.query(USDCTransactionsHistory) .filter(USDCTransactionsHistory.signature == tx_sig_str) diff --git a/packages/discovery-provider/src/tasks/index_user_bank.py b/packages/discovery-provider/src/tasks/index_user_bank.py index d77d317be78..c2451af860c 100644 --- a/packages/discovery-provider/src/tasks/index_user_bank.py +++ b/packages/discovery-provider/src/tasks/index_user_bank.py @@ -11,6 +11,7 @@ from solders.message import Message from solders.pubkey import Pubkey from solders.rpc.responses import GetTransactionResp +from solders.token.associated import get_associated_token_address from solders.transaction_status import UiTransactionStatusMeta from sqlalchemy import and_, desc from sqlalchemy.orm.session import Session @@ -78,17 +79,37 @@ # Populate values used in UserBank indexing from config USER_BANK_ADDRESS = shared_config["solana"]["user_bank_program_address"] +PAYMENT_ROUTER_ADDRESS = shared_config["solana"]["payment_router_program_address"] WAUDIO_MINT = shared_config["solana"]["waudio_mint"] USDC_MINT = shared_config["solana"]["usdc_mint"] + USER_BANK_KEY = Pubkey.from_string(USER_BANK_ADDRESS) if USER_BANK_ADDRESS else None WAUDIO_MINT_PUBKEY = Pubkey.from_string(WAUDIO_MINT) if WAUDIO_MINT else None USDC_MINT_PUBKEY = Pubkey.from_string(USDC_MINT) if USDC_MINT else None +PAYMENT_ROUTER_PUBKEY = ( + Pubkey.from_string(PAYMENT_ROUTER_ADDRESS) if PAYMENT_ROUTER_ADDRESS else None +) + # Transfer instructions don't have a mint acc arg but do have userbank authority. # So re-derive the claimable token PDAs for each mint here to help us determine mint later. WAUDIO_PDA, _ = get_base_address(WAUDIO_MINT_PUBKEY, USER_BANK_KEY) USDC_PDA, _ = get_base_address(USDC_MINT_PUBKEY, USER_BANK_KEY) +PAYMENT_ROUTER_PDA_PUBKEY, _ = get_base_address( + "payment_router".encode("UTF-8"), PAYMENT_ROUTER_PUBKEY +) +PAYMENT_ROUTER_USDC_ATA_ADDRESS = ( + str(get_associated_token_address(PAYMENT_ROUTER_PDA_PUBKEY, USDC_MINT_PUBKEY)) + if PAYMENT_ROUTER_PDA_PUBKEY and USDC_MINT_PUBKEY + else None +) +PAYMENT_ROUTER_WAUDIO_ATA_ADDRESS = ( + str(get_associated_token_address(PAYMENT_ROUTER_PDA_PUBKEY, WAUDIO_MINT_PUBKEY)) + if PAYMENT_ROUTER_PDA_PUBKEY and WAUDIO_MINT_PUBKEY + else None +) + # Used to limit tx history if needed MIN_SLOT = int(shared_config["solana"]["user_bank_min_slot"]) INITIAL_FETCH_SIZE = 10 @@ -558,6 +579,10 @@ def process_transfer_instruction( ) return + if receiver_account == PAYMENT_ROUTER_USDC_ATA_ADDRESS or receiver_account == PAYMENT_ROUTER_WAUDIO_ATA_ADDRESS: + logger.info(f"index_user_bank.py | Skipping payment router tx {tx_sig}") + return + user_id_accounts = [] if is_audio: From 34bf1b8eb83295bee878611cb6d631f1abf9d4f4 Mon Sep 17 00:00:00 2001 From: Randy Schott <1815175+schottra@users.noreply.github.com> Date: Mon, 22 Jan 2024 10:02:46 -0500 Subject: [PATCH 07/13] fix linting --- .../src/tasks/index_payment_router.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/packages/discovery-provider/src/tasks/index_payment_router.py b/packages/discovery-provider/src/tasks/index_payment_router.py index 0e8beecd2ee..94674fa2d78 100644 --- a/packages/discovery-provider/src/tasks/index_payment_router.py +++ b/packages/discovery-provider/src/tasks/index_payment_router.py @@ -395,7 +395,7 @@ def attempt_index_recovery_transfer( ) if len(receiver_user_accounts) != 1: raise Exception( - f"Recovery transfer must have exactly one receiver account. Received: {','.join([a.user_bank_account for a in receiver_user_accounts])}" + f"Recovery transfer must have exactly one receiver account. Received: {','.join([a['user_bank_account'] for a in receiver_user_accounts])}" ) receiver_user_account = receiver_user_accounts[0] @@ -500,8 +500,12 @@ def validate_and_index_usdc_transfers( tx_sig: str, ): """Checks if the transaction is a valid purchase and if so creates the purchase record. Otherwise, indexes a transfer.""" - if memo["type"] is RouteTransactionMemoType.purchase and validate_purchase( - purchase_metadata=memo["metadata"], balance_changes=balance_changes + if ( + memo["type"] is RouteTransactionMemoType.purchase + and memo["metadata"] is not None + and validate_purchase( + purchase_metadata=memo["metadata"], balance_changes=balance_changes + ) ): index_purchase( session=session, @@ -688,7 +692,10 @@ def process_route_instruction( ) # If the memo had purchase information, dispatch challenge events - if memo["type"] is RouteTransactionMemoType.purchase: + if ( + memo["type"] is RouteTransactionMemoType.purchase + and memo["metadata"] is not None + ): logger.info( f"index_payment_router.py | tx: {tx_sig} | Purchase memo found. Dispatching challenge events" ) From bcbfbb8ad5b89930b788ccce9a389c427625c554 Mon Sep 17 00:00:00 2001 From: Randy Schott <1815175+schottra@users.noreply.github.com> Date: Mon, 22 Jan 2024 10:47:56 -0500 Subject: [PATCH 08/13] update logic for partial and over-recovery --- .../tasks/test_index_payment_router.py | 75 +++++++++++++++++-- .../tasks/test_index_user_bank.py | 10 +-- .../src/tasks/index_payment_router.py | 18 +++-- 3 files changed, 85 insertions(+), 18 deletions(-) diff --git a/packages/discovery-provider/integration_tests/tasks/test_index_payment_router.py b/packages/discovery-provider/integration_tests/tasks/test_index_payment_router.py index f834760590c..e5f55071b4d 100644 --- a/packages/discovery-provider/integration_tests/tasks/test_index_payment_router.py +++ b/packages/discovery-provider/integration_tests/tasks/test_index_payment_router.py @@ -1,23 +1,22 @@ from datetime import datetime from unittest.mock import call, create_autospec -from payment_router_mock_transactions import ( +from integration_tests.tasks.payment_router_mock_transactions import ( mock_failed_track_purchase_single_recipient_tx, - mock_valid_track_purchase_from_user_bank_single_recipient_tx, mock_invalid_track_purchase_bad_PDA_account_single_recipient_tx, mock_invalid_track_purchase_insufficient_split_tx, mock_invalid_track_purchase_missing_split_tx, mock_non_route_transfer_purchase_single_recipient_tx, + mock_valid_track_purchase_from_user_bank_single_recipient_tx, mock_valid_track_purchase_multi_recipient_pay_extra_tx, mock_valid_track_purchase_multi_recipient_tx, mock_valid_track_purchase_single_recipient_pay_extra_tx, mock_valid_track_purchase_single_recipient_tx, - mock_valid_transfer_without_purchase_multi_recipient_tx, - mock_valid_transfer_without_purchase_single_recipient_tx, mock_valid_transfer_from_user_bank_without_purchase_single_recipient_tx, mock_valid_transfer_single_recipient_recovery_tx, + mock_valid_transfer_without_purchase_multi_recipient_tx, + mock_valid_transfer_without_purchase_single_recipient_tx, ) - from integration_tests.utils import populate_mock_db from src.challenges.challenge_event import ChallengeEvent from src.challenges.challenge_event_bus import ChallengeEventBus @@ -426,8 +425,70 @@ def test_process_payment_router_tx_details_transfer_partial_recovery( ) # Recovery transaction was for half of the original amount, expect the difference assert transaction_record.change == -1000000 - # TODO: Need to also update the balance - # TODO: Consider for the case of recovering too much, we might just index it as a regular inbound transfer so that the balance is correct. + assert transaction_record.balance == 1000000 + + +# Recovery transaction is for more than the original, should index as a new transfer +def test_process_payment_router_tx_details_transfer_over_recovery( + app, +): + tx_response = mock_valid_transfer_single_recipient_recovery_tx + with app.app_context(): + db = get_db() + + transaction = tx_response.value.transaction.transaction + + tx_sig_str = str(transaction.signatures[0]) + + challenge_event_bus = create_autospec(ChallengeEventBus) + + test_entries_with_transaction = test_entries.copy() + test_entries_with_transaction["usdc_transactions_history"] = [ + { + "user_bank": trackOwnerUserBank, + "signature": "existingWithdrawal", + "transaction_type": USDCTransactionType.transfer, + "method": USDCTransactionMethod.send, + "change": -500000, + "balance": 0, + "tx_metadata": transactionSenderUsdcAccount, + } + ] + + populate_mock_db(db, test_entries_with_transaction) + + with db.scoped_session() as session: + process_payment_router_tx_details( + session=session, + tx_info=tx_response, + tx_sig=tx_sig_str, + timestamp=datetime.now(), + challenge_event_bus=challenge_event_bus, + ) + + # Expect original transaction to be modified + existing_transaction_record = ( + session.query(USDCTransactionsHistory) + .filter(USDCTransactionsHistory.signature == "existingWithdrawal") + .filter(USDCTransactionsHistory.user_bank == trackOwnerUserBank) + .filter(USDCTransactionsHistory.tx_metadata == transactionSenderUsdcAccount) + .first() + ) + # Recovery transaction was for more than the original amount, expect original transaction to be unchanged + assert existing_transaction_record.change == -500000 + + # Expect new transaction to have been added + new_transaction_record = ( + session.query(USDCTransactionsHistory) + .filter(USDCTransactionsHistory.signature == tx_sig_str) + .filter(USDCTransactionsHistory.user_bank == trackOwnerUserBank) + .filter(USDCTransactionsHistory.tx_metadata == transactionSenderUsdcAccount) + .first() + ) + + assert new_transaction_record is not None + assert new_transaction_record.change == 1000000 + assert new_transaction_record.balance == 1000000 # Recovery transaction doesn't match the most recent outbound transfer (different addresses). Should index diff --git a/packages/discovery-provider/integration_tests/tasks/test_index_user_bank.py b/packages/discovery-provider/integration_tests/tasks/test_index_user_bank.py index 0fd42c4a242..668fa91133b 100644 --- a/packages/discovery-provider/integration_tests/tasks/test_index_user_bank.py +++ b/packages/discovery-provider/integration_tests/tasks/test_index_user_bank.py @@ -1,7 +1,10 @@ from datetime import datetime from unittest.mock import call, create_autospec -from user_bank_mock_transactions import ( +from integration_tests.tasks.payment_router_mock_transactions import ( + mock_valid_transfer_from_user_bank_without_purchase_single_recipient_tx, +) +from integration_tests.tasks.user_bank_mock_transactions import ( EXTERNAL_ACCOUNT_ADDRESS, RECIPIENT_ACCOUNT_ADDRESS, SENDER_ACCOUNT_ADDRESS, @@ -15,11 +18,6 @@ mock_valid_track_purchase_tx, mock_valid_transfer_without_purchase_tx, ) - -from payment_router_mock_transactions import ( - mock_valid_transfer_from_user_bank_without_purchase_single_recipient_tx, -) - from integration_tests.utils import populate_mock_db from src.challenges.challenge_event import ChallengeEvent from src.challenges.challenge_event_bus import ChallengeEventBus diff --git a/packages/discovery-provider/src/tasks/index_payment_router.py b/packages/discovery-provider/src/tasks/index_payment_router.py index 94674fa2d78..af14cd398eb 100644 --- a/packages/discovery-provider/src/tasks/index_payment_router.py +++ b/packages/discovery-provider/src/tasks/index_payment_router.py @@ -400,7 +400,8 @@ def attempt_index_recovery_transfer( receiver_user_account = receiver_user_accounts[0] receiver_bank_account = receiver_user_account["user_bank_account"] - receiver_balance_change = balance_changes[receiver_bank_account]["change"] + receiver_balance_change = balance_changes[receiver_bank_account] + receiver_balance_change_amount = receiver_balance_change["change"] # Find the previous transaction in the other direction (receiver -> sender) # So that we can modify it @@ -421,22 +422,29 @@ def attempt_index_recovery_transfer( f"Failed to find matching previous transcation from {receiver_bank_account} to {sender_account}" ) - # Previous transaction is completely undone by this recovery - if -previous_transaction.change <= receiver_balance_change: + difference = previous_transaction.change + receiver_balance_change_amount + + # Previous transaction is undone by this recovery + if difference == 0: logger.info( f"index_payment_router.py | tx: {tx_sig} | Removing previous transaction {previous_transaction.signature}." ) session.delete(previous_transaction) - else: + elif difference < 0: # Previous transaction is partially undone by this recovery # Modifying the object returned by the query will update the DB # when the session is flushed previous_transaction.change = ( - previous_transaction.change + receiver_balance_change + previous_transaction.change + receiver_balance_change_amount ) + previous_transaction.balance = Decimal(receiver_balance_change["post_balance"]) logger.info( f"index_payment_router.py | tx: {tx_sig} | Found partial recovery for {previous_transaction.signature}. New change value is {previous_transaction.change}." ) + else: + raise Exception( + f"Recovery transaction is for greater amount than original transaction ({receiver_balance_change_amount} > {-previous_transaction.change})" + ) def index_transfer( From 93cb9176ade0b2f2ba705083e84882ccbb12b3bd Mon Sep 17 00:00:00 2001 From: Randy Schott <1815175+schottra@users.noreply.github.com> Date: Mon, 22 Jan 2024 13:47:19 -0500 Subject: [PATCH 09/13] PR feedback --- packages/commands/src/route-tokens-to-user-bank.mjs | 4 +++- packages/commands/src/withdraw-tokens.mjs | 8 +++++--- packages/discovery-provider/.vscode/settings.json | 2 +- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/packages/commands/src/route-tokens-to-user-bank.mjs b/packages/commands/src/route-tokens-to-user-bank.mjs index a0b2dbc43fe..1e61658c367 100644 --- a/packages/commands/src/route-tokens-to-user-bank.mjs +++ b/packages/commands/src/route-tokens-to-user-bank.mjs @@ -114,7 +114,9 @@ program skipPreflight: true } ) - console.log(chalk.green(`Successfully created new ${mint} account`)) + console.log( + chalk.green(`Successfully created new ${mint} token account`) + ) console.log( chalk.yellow('Transaction Signature:'), accountCreationTxSignature diff --git a/packages/commands/src/withdraw-tokens.mjs b/packages/commands/src/withdraw-tokens.mjs index ea7f467dc05..b1cca274c1e 100644 --- a/packages/commands/src/withdraw-tokens.mjs +++ b/packages/commands/src/withdraw-tokens.mjs @@ -16,13 +16,13 @@ program .command('withdraw-tokens') .description('Send USDC from a user bank to an external address') .argument('', 'The solana address of the recipient') - .argument('', 'The amount of tokens to tip (in wei)') + .argument('', 'The amount of tokens to send (in wei)') .addOption( new Option('-m, --mint [mint]', 'The currency to use') .choices(['audio', 'usdc']) .default('usdc') ) - .option('-f, --from [from]', 'The account to tip from') + .option('-f, --from [from]', 'The account to send from') .action(async (recipientAccount, amount, { from, mint }) => { const audiusLibs = await initializeAudiusLibs(from) const { solanaWeb3Manager } = audiusLibs @@ -88,7 +88,9 @@ program skipPreflight: true } ) - console.log(chalk.green(`Successfully created new ${mint} account`)) + console.log( + chalk.green(`Successfully created new ${mint} token account`) + ) console.log( chalk.yellow('Transaction Signature:'), accountCreationTxSignature diff --git a/packages/discovery-provider/.vscode/settings.json b/packages/discovery-provider/.vscode/settings.json index 6a52dfd6fad..34c9a576326 100644 --- a/packages/discovery-provider/.vscode/settings.json +++ b/packages/discovery-provider/.vscode/settings.json @@ -15,7 +15,7 @@ "python.testing.unittestEnabled": false, "[python]": { "editor.codeActionsOnSave": { - "source.organizeImports": "explicit" + "source.organizeImports": "true" }, "editor.defaultFormatter": "ms-python.black-formatter" }, From 0894a27a61a6b2dbe62138c5dba7f373b2ec4891 Mon Sep 17 00:00:00 2001 From: Randy Schott <1815175+schottra@users.noreply.github.com> Date: Mon, 22 Jan 2024 13:47:48 -0500 Subject: [PATCH 10/13] remove settings migration --- .vscode/settings.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index ceefbc3fd9c..bc068921ab1 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -21,14 +21,14 @@ "python.testing.unittestEnabled": false, "[python]": { "editor.codeActionsOnSave": { - "source.organizeImports": "explicit" + "source.organizeImports": "true" }, "editor.defaultFormatter": "ms-python.black-formatter" }, "editor.find.addExtraSpaceOnTop": false, "eslint.workingDirectories": [{ "pattern": "./packages/*/" }], "editor.codeActionsOnSave": { - "source.fixAll": "explicit" + "source.fixAll": "true" }, "gitlens.advanced.fileHistoryFollowsRenames": true } From b7c089fefd1fc1ae7839bceaa2fd6f71c211abb8 Mon Sep 17 00:00:00 2001 From: Randy Schott <1815175+schottra@users.noreply.github.com> Date: Mon, 22 Jan 2024 13:51:48 -0500 Subject: [PATCH 11/13] actually revert config settings --- .vscode/settings.json | 4 ++-- packages/discovery-provider/.vscode/settings.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index bc068921ab1..f4af686f1b9 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -21,14 +21,14 @@ "python.testing.unittestEnabled": false, "[python]": { "editor.codeActionsOnSave": { - "source.organizeImports": "true" + "source.organizeImports": true }, "editor.defaultFormatter": "ms-python.black-formatter" }, "editor.find.addExtraSpaceOnTop": false, "eslint.workingDirectories": [{ "pattern": "./packages/*/" }], "editor.codeActionsOnSave": { - "source.fixAll": "true" + "source.fixAll": true }, "gitlens.advanced.fileHistoryFollowsRenames": true } diff --git a/packages/discovery-provider/.vscode/settings.json b/packages/discovery-provider/.vscode/settings.json index 34c9a576326..fc51bc3e1c6 100644 --- a/packages/discovery-provider/.vscode/settings.json +++ b/packages/discovery-provider/.vscode/settings.json @@ -15,7 +15,7 @@ "python.testing.unittestEnabled": false, "[python]": { "editor.codeActionsOnSave": { - "source.organizeImports": "true" + "source.organizeImports": true }, "editor.defaultFormatter": "ms-python.black-formatter" }, From 11da4194c1a3b5481e5cb93cdb826bc151beb2cc Mon Sep 17 00:00:00 2001 From: Randy Schott <1815175+schottra@users.noreply.github.com> Date: Mon, 22 Jan 2024 15:52:55 -0500 Subject: [PATCH 12/13] more PR feedback --- .../src/route-tokens-to-user-bank.mjs | 2 +- .../src/tasks/index_payment_router.py | 43 ++++++++----------- 2 files changed, 18 insertions(+), 27 deletions(-) diff --git a/packages/commands/src/route-tokens-to-user-bank.mjs b/packages/commands/src/route-tokens-to-user-bank.mjs index 1e61658c367..9de066ff419 100644 --- a/packages/commands/src/route-tokens-to-user-bank.mjs +++ b/packages/commands/src/route-tokens-to-user-bank.mjs @@ -76,7 +76,7 @@ program const senderAccountPublicKey = senderAccountKeypair.publicKey try { - console.log('checking for source usdc account') + console.log('checking for source account...') const senderTokenAccount = await solanaWeb3Manager.findAssociatedTokenAddress( senderAccountPublicKey.toString(), diff --git a/packages/discovery-provider/src/tasks/index_payment_router.py b/packages/discovery-provider/src/tasks/index_payment_router.py index af14cd398eb..5f0e1d54afc 100644 --- a/packages/discovery-provider/src/tasks/index_payment_router.py +++ b/packages/discovery-provider/src/tasks/index_payment_router.py @@ -130,7 +130,6 @@ class RouteTransactionMemoType(str, enum.Enum): purchase = "purchase" recovery = "recovery" unknown = "unknown" - none = "none" class RouteTransactionMemo(TypedDict): @@ -199,10 +198,10 @@ def get_tx_in_db(session: Session, tx_sig: str) -> bool: def parse_route_transaction_memo( session: Session, memos: List[str], timestamp: datetime -) -> RouteTransactionMemo: +) -> RouteTransactionMemo | None: """Checks the list of memos for one matching a format of a purchase's content_metadata, and then uses that content_metadata to find the stream_conditions associated with that content to get the price""" if len(memos) == 0: - return RouteTransactionMemo(type=RouteTransactionMemoType.none, metadata=None) + return None for memo in memos: if memo == RECOVERY_MEMO_STRING: return RouteTransactionMemo( @@ -502,14 +501,15 @@ def validate_and_index_usdc_transfers( receiver_user_accounts: List[UserIdBankAccount], receiver_accounts: List[str], balance_changes: dict[str, BalanceChange], - memo: RouteTransactionMemo, + memo: RouteTransactionMemo | None, slot: int, timestamp: datetime, tx_sig: str, ): """Checks if the transaction is a valid purchase and if so creates the purchase record. Otherwise, indexes a transfer.""" if ( - memo["type"] is RouteTransactionMemoType.purchase + memo is not None + and memo["type"] is RouteTransactionMemoType.purchase and memo["metadata"] is not None and validate_purchase( purchase_metadata=memo["metadata"], balance_changes=balance_changes @@ -528,7 +528,7 @@ def validate_and_index_usdc_transfers( ) # For invalid purchases or transfers not related to a purchase, we'll index # it as a regular transfer from the sender_account. - elif memo["type"] is RouteTransactionMemoType.recovery: + elif memo is not None and memo["type"] is RouteTransactionMemoType.recovery: try: attempt_index_recovery_transfer( session=session, @@ -567,23 +567,15 @@ def validate_and_index_usdc_transfers( ) -def find_sender_account_from_balance_changes( - balance_changes: dict[str, BalanceChange], - receiver_accounts: List[str], -): +def find_sender_account_from_balance_changes(balance_changes: dict[str, BalanceChange]): """Finds the sender account from the balance changes and receiver accounts""" - total_sent = 0 - for account in receiver_accounts: - if account in balance_changes: - total_sent += balance_changes[account]["change"] - return next( - ( - account - for account, balance_change in balance_changes.items() - if balance_change["change"] == -total_sent - ), - None, - ) + min_change = 0 + sender = None + for account, balance_change in balance_changes.items(): + if balance_change["change"] < min_change: + sender = account + min_change = balance_change["change"] + return sender def process_route_instruction( @@ -622,9 +614,7 @@ def process_route_instruction( # Detect the account which sent tokens _into_ payment router, as that's # our real source account sender_account = ( - find_sender_account_from_balance_changes( - balance_changes=balance_changes, receiver_accounts=receiver_accounts - ) + find_sender_account_from_balance_changes(balance_changes=balance_changes) or sender_pda_account ) @@ -701,7 +691,8 @@ def process_route_instruction( # If the memo had purchase information, dispatch challenge events if ( - memo["type"] is RouteTransactionMemoType.purchase + memo is not None + and memo["type"] is RouteTransactionMemoType.purchase and memo["metadata"] is not None ): logger.info( From 376a76bc8e5308091ecc09be1ccff9a71f657a7e Mon Sep 17 00:00:00 2001 From: Randy Schott <1815175+schottra@users.noreply.github.com> Date: Mon, 22 Jan 2024 16:32:28 -0500 Subject: [PATCH 13/13] fix doc string --- packages/commands/src/withdraw-tokens.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/commands/src/withdraw-tokens.mjs b/packages/commands/src/withdraw-tokens.mjs index b1cca274c1e..17971cee55f 100644 --- a/packages/commands/src/withdraw-tokens.mjs +++ b/packages/commands/src/withdraw-tokens.mjs @@ -14,7 +14,7 @@ import { program .command('withdraw-tokens') - .description('Send USDC from a user bank to an external address') + .description('Send tokens from a user bank to an external address') .argument('', 'The solana address of the recipient') .argument('', 'The amount of tokens to send (in wei)') .addOption(