Fix MRI clm / udr s3 configs

packages/discovery-provider/plugins/pedalboard/apps/mri/src/config.ts

@@ -62,7 +62,7 @@ export const readConfig = (): Config => {
     audius_mri_udr_external_endpoint: str({ default: 'http://localhost:4566' }),
     audius_mri_udr_external_access_key_id: str({ default: 'test' }),
     audius_mri_udr_external_secret_access_key: str({ default: 'test' }),
-    audius_mri_udr_external_bucket: str({ default: 'audius-clm-data' }),
+    audius_mri_udr_external_bucket: str({ default: 'audius-udr-data' }),
     audius_mri_udr_external_key_prefix: str({ default: 'Audius_UDR_' })
   })
 

packages/discovery-provider/plugins/pedalboard/apps/mri/src/s3.ts

@@ -1,8 +1,12 @@
-import { S3Client, CreateBucketCommand, PutObjectCommand } from "@aws-sdk/client-s3"
-import { formatDate } from "./date"
-import { readConfig } from "./config"
-import { logger } from "./logger"
-import { Logger } from "pino"
+import {
+  S3Client,
+  CreateBucketCommand,
+  PutObjectCommand
+} from '@aws-sdk/client-s3'
+import { formatDate } from './date'
+import { readConfig } from './config'
+import { logger } from './logger'
+import { Logger } from 'pino'
 
 export type S3Config = {
   s3: S3Client
@@ -11,80 +15,114 @@ export type S3Config = {
 }
 
 const config = readConfig()
-const isDev = config.env === "dev"
+const isDev = config.env === 'dev'
 
 // creates s3 instance objects for later use, if in dev it will automatically create buckets
-export const createS3Instances = async (): Promise<{clmS3s: S3Config[], udrS3s: S3Config[]}> => {
+export const createS3Instances = async (): Promise<{
+  clmS3s: S3Config[]
+  udrS3s: S3Config[]
+}> => {
   try {
-    const clmS3s = await Promise.all(config.s3ClmConfigs.map(async ({ bucket, keyPrefix, region, endpoint, accessKeyId, secretAccessKey }) => {
-      const s3 = new S3Client({
-        region,
-        endpoint,
-        credentials: {
+    const clmS3s = await Promise.all(
+      config.s3ClmConfigs.map(
+        async ({
+          bucket,
+          keyPrefix,
+          region,
+          endpoint,
           accessKeyId,
           secretAccessKey
-        },
-        forcePathStyle: true
-      })
+        }) => {
+          const s3 = new S3Client({
+            region,
+            endpoint,
+            credentials: {
+              accessKeyId,
+              secretAccessKey
+            },
+            forcePathStyle: true
+          })
 
-      if (isDev) {
-        const data = await s3.send(new CreateBucketCommand({ Bucket: bucket }));
-        logger.info(`Bucket "${bucket}" created successfully`, data);
-      }
+          if (isDev) {
+            const data = await s3.send(
+              new CreateBucketCommand({ Bucket: bucket })
+            )
+            logger.info(`Bucket "${bucket}" created successfully`, data)
+          }
 
-      return {
-        s3,
-        bucket,
-        keyPrefix
-      }
-    }))
-    const udrS3s = await Promise.all(config.s3ClmConfigs.map(async ({ bucket, keyPrefix, region, endpoint, accessKeyId, secretAccessKey }) => {
-      const s3 = new S3Client({
-        region,
-        endpoint,
-        credentials: {
+          return {
+            s3,
+            bucket,
+            keyPrefix
+          }
+        }
+      )
+    )
+    const udrS3s = await Promise.all(
+      config.s3UdrConfigs.map(
+        async ({
+          bucket,
+          keyPrefix,
+          region,
+          endpoint,
           accessKeyId,
           secretAccessKey
-        },
-        forcePathStyle: true
-      })
+        }) => {
+          const s3 = new S3Client({
+            region,
+            endpoint,
+            credentials: {
+              accessKeyId,
+              secretAccessKey
+            },
+            forcePathStyle: true
+          })
 
-      if (isDev) {
-        const data = await s3.send(new CreateBucketCommand({ Bucket: bucket }));
-        logger.info(`Bucket "${bucket}" created successfully`, data);
-      }
+          if (isDev) {
+            const data = await s3.send(
+              new CreateBucketCommand({ Bucket: bucket })
+            )
+            logger.info(`Bucket "${bucket}" created successfully`, data)
+          }
 
-      return {
-        s3,
-        bucket,
-        keyPrefix
-      }
-    }))
+          return {
+            s3,
+            bucket,
+            keyPrefix
+          }
+        }
+      )
+    )
     return { clmS3s, udrS3s }
   } catch (e) {
     if (isDev) return { clmS3s: [], udrS3s: [] }
     throw e
   }
 }
 
-export const publishToS3 = async (logger: Logger, config: S3Config, date: Date, csv: string): Promise<string> => {
+export const publishToS3 = async (
+  logger: Logger,
+  config: S3Config,
+  date: Date,
+  csv: string
+): Promise<string> => {
   const { s3, keyPrefix, bucket } = config
 
   const key = `${keyPrefix}${formatDate(date)}.csv`
   const contentLength = Buffer.byteLength(csv, 'utf8')
 
-  logger.info({ contentLength, key, bucket }, "uploading")
+  logger.info({ contentLength, key, bucket }, 'uploading')
 
   if (contentLength === 0) {
-    logger.info({}, "no content to publish, no file being uploaded")
-    return ""
+    logger.info({}, 'no content to publish, no file being uploaded')
+    return ''
   }
 
   const uploadParams = {
     Bucket: bucket,
     Key: key,
     Body: csv,
-    ContentType: "text/csv",
+    ContentType: 'text/csv',
     ContentLength: contentLength
   }