diff --git a/packages/discovery-provider/plugins/pedalboard/apps/mri/src/config.ts b/packages/discovery-provider/plugins/pedalboard/apps/mri/src/config.ts index d489008bff2..96640b5e98f 100644 --- a/packages/discovery-provider/plugins/pedalboard/apps/mri/src/config.ts +++ b/packages/discovery-provider/plugins/pedalboard/apps/mri/src/config.ts @@ -62,7 +62,7 @@ export const readConfig = (): Config => { audius_mri_udr_external_endpoint: str({ default: 'http://localhost:4566' }), audius_mri_udr_external_access_key_id: str({ default: 'test' }), audius_mri_udr_external_secret_access_key: str({ default: 'test' }), - audius_mri_udr_external_bucket: str({ default: 'audius-clm-data' }), + audius_mri_udr_external_bucket: str({ default: 'audius-udr-data' }), audius_mri_udr_external_key_prefix: str({ default: 'Audius_UDR_' }) }) diff --git a/packages/discovery-provider/plugins/pedalboard/apps/mri/src/s3.ts b/packages/discovery-provider/plugins/pedalboard/apps/mri/src/s3.ts index 11f0a06dd52..7a12c7fea0e 100644 --- a/packages/discovery-provider/plugins/pedalboard/apps/mri/src/s3.ts +++ b/packages/discovery-provider/plugins/pedalboard/apps/mri/src/s3.ts @@ -1,8 +1,12 @@ -import { S3Client, CreateBucketCommand, PutObjectCommand } from "@aws-sdk/client-s3" -import { formatDate } from "./date" -import { readConfig } from "./config" -import { logger } from "./logger" -import { Logger } from "pino" +import { + S3Client, + CreateBucketCommand, + PutObjectCommand +} from '@aws-sdk/client-s3' +import { formatDate } from './date' +import { readConfig } from './config' +import { logger } from './logger' +import { Logger } from 'pino' export type S3Config = { s3: S3Client @@ -11,55 +15,84 @@ export type S3Config = { } const config = readConfig() -const isDev = config.env === "dev" +const isDev = config.env === 'dev' // creates s3 instance objects for later use, if in dev it will automatically create buckets -export const createS3Instances = async (): Promise<{clmS3s: S3Config[], udrS3s: S3Config[]}> => { +export const createS3Instances = async (): Promise<{ + clmS3s: S3Config[] + udrS3s: S3Config[] +}> => { try { - const clmS3s = await Promise.all(config.s3ClmConfigs.map(async ({ bucket, keyPrefix, region, endpoint, accessKeyId, secretAccessKey }) => { - const s3 = new S3Client({ - region, - endpoint, - credentials: { + const clmS3s = await Promise.all( + config.s3ClmConfigs.map( + async ({ + bucket, + keyPrefix, + region, + endpoint, accessKeyId, secretAccessKey - }, - forcePathStyle: true - }) + }) => { + const s3 = new S3Client({ + region, + endpoint, + credentials: { + accessKeyId, + secretAccessKey + }, + forcePathStyle: true + }) - if (isDev) { - const data = await s3.send(new CreateBucketCommand({ Bucket: bucket })); - logger.info(`Bucket "${bucket}" created successfully`, data); - } + if (isDev) { + const data = await s3.send( + new CreateBucketCommand({ Bucket: bucket }) + ) + logger.info(`Bucket "${bucket}" created successfully`, data) + } - return { - s3, - bucket, - keyPrefix - } - })) - const udrS3s = await Promise.all(config.s3ClmConfigs.map(async ({ bucket, keyPrefix, region, endpoint, accessKeyId, secretAccessKey }) => { - const s3 = new S3Client({ - region, - endpoint, - credentials: { + return { + s3, + bucket, + keyPrefix + } + } + ) + ) + const udrS3s = await Promise.all( + config.s3UdrConfigs.map( + async ({ + bucket, + keyPrefix, + region, + endpoint, accessKeyId, secretAccessKey - }, - forcePathStyle: true - }) + }) => { + const s3 = new S3Client({ + region, + endpoint, + credentials: { + accessKeyId, + secretAccessKey + }, + forcePathStyle: true + }) - if (isDev) { - const data = await s3.send(new CreateBucketCommand({ Bucket: bucket })); - logger.info(`Bucket "${bucket}" created successfully`, data); - } + if (isDev) { + const data = await s3.send( + new CreateBucketCommand({ Bucket: bucket }) + ) + logger.info(`Bucket "${bucket}" created successfully`, data) + } - return { - s3, - bucket, - keyPrefix - } - })) + return { + s3, + bucket, + keyPrefix + } + } + ) + ) return { clmS3s, udrS3s } } catch (e) { if (isDev) return { clmS3s: [], udrS3s: [] } @@ -67,24 +100,29 @@ export const createS3Instances = async (): Promise<{clmS3s: S3Config[], udrS3s: } } -export const publishToS3 = async (logger: Logger, config: S3Config, date: Date, csv: string): Promise => { +export const publishToS3 = async ( + logger: Logger, + config: S3Config, + date: Date, + csv: string +): Promise => { const { s3, keyPrefix, bucket } = config const key = `${keyPrefix}${formatDate(date)}.csv` const contentLength = Buffer.byteLength(csv, 'utf8') - logger.info({ contentLength, key, bucket }, "uploading") + logger.info({ contentLength, key, bucket }, 'uploading') if (contentLength === 0) { - logger.info({}, "no content to publish, no file being uploaded") - return "" + logger.info({}, 'no content to publish, no file being uploaded') + return '' } const uploadParams = { Bucket: bucket, Key: key, Body: csv, - ContentType: "text/csv", + ContentType: 'text/csv', ContentLength: contentLength }