From 5b28cf8a460eb0ca98cfea8e96252ce4bb5ad4ee Mon Sep 17 00:00:00 2001
From: Michelle Brier <michelle.brier4@gmail.com>
Date: Mon, 5 Feb 2024 23:32:59 -0800
Subject: [PATCH 1/3] crawler + clean up envs during local dev + update README

---
 dev-tools/compose/docker-compose.ddex.yml     |  46 ++++---
 dev-tools/compose/docker-compose.yml          |  10 +-
 packages/ddex/.env.dev                        |  13 ++
 packages/ddex/{webapp/server => }/.env.stage  |   7 +
 packages/ddex/README.md                       |  43 ++++++
 packages/ddex/ingester/.air.toml              |   6 +-
 packages/ddex/ingester/README.md              |   1 +
 packages/ddex/ingester/cmd/main.go            |  38 ++++--
 packages/ddex/ingester/common/common.go       |  54 ++++++++
 packages/ddex/ingester/crawler/crawler.go     | 123 +++++++++++++++++-
 packages/ddex/ingester/go.mod                 |   3 +
 packages/ddex/ingester/go.sum                 |   8 ++
 packages/ddex/ingester/indexer/indexer.go     |  43 +++---
 packages/ddex/ingester/parser/parser.go       |  66 +++++-----
 packages/ddex/publisher/.env.stage            |   3 -
 packages/ddex/publisher/README.md             |   5 +-
 packages/ddex/publisher/package.json          |   1 +
 packages/ddex/publisher/src/index.ts          |   5 +-
 packages/ddex/webapp/README.md                |  39 +-----
 .../src/components/Collection/Collection.tsx  | 120 +++--------------
 packages/ddex/webapp/server/.env.dev          |   4 -
 packages/ddex/webapp/server/package.json      |   1 +
 packages/ddex/webapp/server/src/app.ts        |   3 +
 packages/ddex/webapp/server/src/index.ts      |   7 +-
 .../webapp/server/src/services/dbService.ts   |  19 ---
 .../ddex/webapp/server/src/services/s3.ts     |   2 +-
 26 files changed, 397 insertions(+), 273 deletions(-)
 create mode 100644 packages/ddex/.env.dev
 rename packages/ddex/{webapp/server => }/.env.stage (56%)
 create mode 100644 packages/ddex/ingester/common/common.go
 delete mode 100644 packages/ddex/publisher/.env.stage
 delete mode 100644 packages/ddex/webapp/server/.env.dev

diff --git a/dev-tools/compose/docker-compose.ddex.yml b/dev-tools/compose/docker-compose.ddex.yml
index 38b177b9d9d..287739a294a 100644
--- a/dev-tools/compose/docker-compose.ddex.yml
+++ b/dev-tools/compose/docker-compose.ddex.yml
@@ -1,24 +1,26 @@
 version: '3.9'
 
 services:
-  ddex-web:
-    container_name: ddex-web
-    build:
-      context: ${PROJECT_ROOT}
-      dockerfile: ${PROJECT_ROOT}/packages/ddex/webapp/Dockerfile
-      args:
-        app_name: ddex-web
-        TURBO_TEAM: '${TURBO_TEAM}'
-        TURBO_TOKEN: '${TURBO_TOKEN}'
-    environment:
-      - DDEX_PORT=9000
-      - DDEX_MONGODB_URL=mongodb://mongo:mongo@ddex-mongo:27017/ddex?authSource=admin&replicaSet=rs0
-    ports:
-      - "9000:9000"
-    networks:
-      - ddex-network
-    profiles:
-      - ddex
+  # ddex-web:
+  #   container_name: ddex-web
+  #   build:
+  #     context: ${PROJECT_ROOT}
+  #     dockerfile: ${PROJECT_ROOT}/packages/ddex/webapp/Dockerfile
+  #     args:
+  #       app_name: ddex-web
+  #       TURBO_TEAM: '${TURBO_TEAM}'
+  #       TURBO_TOKEN: '${TURBO_TOKEN}'
+  #   environment:
+  #     - DDEX_PORT=9000
+  #     - DDEX_MONGODB_URL=mongodb://mongo:mongo@ddex-mongo:27017/ddex?authSource=admin&replicaSet=rs0
+  #   env_file:
+  #     - ${PROJECT_ROOT}/packages/ddex/.env
+  #   ports:
+  #     - "9000:9000"
+  #   networks:
+  #     - ddex-network
+  #   profiles:
+  #     - ddex
 
   ddex-crawler:
     container_name: ddex-crawler
@@ -27,6 +29,8 @@ services:
       dockerfile: ${PROJECT_ROOT}/packages/ddex/ingester/Dockerfile
     environment:
       - DDEX_MONGODB_URL=mongodb://mongo:mongo@ddex-mongo:27017/ddex?authSource=admin&replicaSet=rs0
+    env_file:
+      - ${PROJECT_ROOT}/packages/ddex/.env
     depends_on:
       ddex-mongo:
         condition: service_healthy
@@ -43,6 +47,8 @@ services:
       dockerfile: ${PROJECT_ROOT}/packages/ddex/ingester/Dockerfile
     environment:
       - DDEX_MONGODB_URL=mongodb://mongo:mongo@ddex-mongo:27017/ddex?authSource=admin&replicaSet=rs0
+    env_file:
+      - ${PROJECT_ROOT}/packages/ddex/.env
     depends_on:
       ddex-mongo:
         condition: service_healthy
@@ -59,6 +65,8 @@ services:
       dockerfile: ${PROJECT_ROOT}/packages/ddex/ingester/Dockerfile
     environment:
       - DDEX_MONGODB_URL=mongodb://mongo:mongo@ddex-mongo:27017/ddex?authSource=admin&replicaSet=rs0
+    env_file:
+      - ${PROJECT_ROOT}/packages/ddex/.env
     depends_on:
       ddex-mongo:
         condition: service_healthy
@@ -79,6 +87,8 @@ services:
         TURBO_TOKEN: '${TURBO_TOKEN}'
     environment:
       - DDEX_MONGODB_URL=mongodb://mongo:mongo@ddex-mongo:27017/ddex?authSource=admin&replicaSet=rs0
+    env_file:
+      - ${PROJECT_ROOT}/packages/ddex/.env
     depends_on:
       ddex-mongo:
         condition: service_healthy
diff --git a/dev-tools/compose/docker-compose.yml b/dev-tools/compose/docker-compose.yml
index 0c40ab60287..2511ef022a8 100644
--- a/dev-tools/compose/docker-compose.yml
+++ b/dev-tools/compose/docker-compose.yml
@@ -184,11 +184,11 @@ services:
 
   # DDEX
 
-  ddex-web:
-    extends:
-      file: docker-compose.ddex.yml
-      service: ddex-web
-    <<: *common
+  # ddex-web:
+  #   extends:
+  #     file: docker-compose.ddex.yml
+  #     service: ddex-web
+  #   <<: *common
 
   ddex-crawler:
     extends:
diff --git a/packages/ddex/.env.dev b/packages/ddex/.env.dev
new file mode 100644
index 00000000000..8a89825b9ce
--- /dev/null
+++ b/packages/ddex/.env.dev
@@ -0,0 +1,13 @@
+NODE_ENV='dev'
+
+AWS_ACCESS_KEY_ID=
+AWS_SECRET_ACCESS_KEY=
+AWS_REGION=
+AWS_BUCKET_RAW=
+AWS_BUCKET_INDEXED=
+
+DDEX_KEY=
+DDEX_SECRET=
+
+# use stage optimizely
+OPTIMIZELY_SDK_KEY='MX4fYBgANQetvmBXGpuxzF'
diff --git a/packages/ddex/webapp/server/.env.stage b/packages/ddex/.env.stage
similarity index 56%
rename from packages/ddex/webapp/server/.env.stage
rename to packages/ddex/.env.stage
index 03713b9f835..d5c3cbeb2ce 100644
--- a/packages/ddex/webapp/server/.env.stage
+++ b/packages/ddex/.env.stage
@@ -1,4 +1,11 @@
 NODE_ENV='stage'
+
+AWS_ACCESS_KEY_ID=
+AWS_SECRET_ACCESS_KEY=
+AWS_REGION='us-west-2'
+AWS_BUCKET_RAW='ddex-dev-audius-raw'
+AWS_BUCKET_INDEXED='ddex-dev-audius-indexed'
+
 DDEX_KEY='49d5e13d355709b615b7cce7369174fb240b6b39'
 DDEX_SECRET='2b2c2b90d9a489234ae629a5284de84fb0633306257f17667aaebf2345d92152'
 OPTIMIZELY_SDK_KEY='MX4fYBgANQetvmBXGpuxzF'
diff --git a/packages/ddex/README.md b/packages/ddex/README.md
index 25192af3a85..64df2aa076d 100644
--- a/packages/ddex/README.md
+++ b/packages/ddex/README.md
@@ -5,6 +5,15 @@ Processes and uploads DDEX releases to Audius.
 ## Local Dev
 DDEX requires these services: `ddex-webapp`, `ddex-crawler`, `ddex-indexer`, `ddex-parser`, `ddex-publisher`, `ddex-mongo`.
 
+### Env configuration
+All services read from `.env`.
+
+To use stage envs: `cp .env.stage .env`
+
+To use dev envs: `cp .env.dev .env`
+
+Fill in all missing values. See the `Creating a bucket in S3` section below for how to set up S3.
+
 ### Setup
 1. (At the monorepo root) Generate a keyfile for mongodb:
 ```
@@ -24,3 +33,37 @@ To access the ddex db via the mongo shell: `docker exec -it ddex-mongo mongosh -
 
 ### Develop with hot reloading
 Each service can be run independently as long as `ddex-mongo` is up. See the respective subdirectories' READMEs.
+
+### Creating a bucket in S3
+1. Create a new bucket in the S3 console with the name `ddex-[dev|staging]-<label/distributor>-raw`. Use all the defaults, including "ACLs disabled"
+2. Do the same for a bucket named `ddex-[dev|staging]-<label/distributor>-indexed`. Use all the defaults, including "ACLs disabled"
+3. Create an IAM Policy (here](https://us-east-1.console.aws.amazon.com/iamv2/home?region=us-west-2#/policies/create) (or search IAM and click Policies > Create Policy). Select S3.
+    * Under `Read` choose `GetObject` and `GetObjectAttributes`.
+    * Under `Write` choose `DeleteObject` and `PutObject`.
+    * Under `List` choose `ListBucket`.
+    * Click `Add Arn` for object actions, enter the bucket name ending with `raw`, and check the box for `Any object name`.
+    * Click `Add Arn` for object actions again, enter the bucket name ending with `indexed`, and check the box for `Any object name`.
+    * Click `Add Arn` for bucket actions and enter the bucket name ending with `raw`.
+    * Click `Add Arn` for bucket actions again and enter the bucket name ending with `indexed`.
+    * Click Next, and then name the policy `ddex-[dev|staging]-<label/distributor>-policy`.
+4. Create an IAM User [here](https://us-east-1.console.aws.amazon.com/iamv2/home?region=us-west-2#/users/create) (or search IAM and click Users > Create User).
+    * Name the user `ddex-[dev|staging]-<label/distributor>-user` and press Next.
+    * Select "Attach policies directly," and search for the policy you created (`ddex-[dev|staging]-<label/distributor>-policy`). Check the box next to it and press Next and then Create User.
+5. Search for your new user and press "Create access key" and then "Third-party service." Copy the access key and secret access key into your .env file (assuming you've already done `cp .env.dev .env`).
+6. Go back to the bucket ending with `raw`, and add CORS at the bottom of the Permissions tab. Here's an example for dev, but for a prod environment you'll wnat to replace "*" in "AllowedOrigins" with the DNS that the frontend will be served from:
+```json
+[
+    {
+        "AllowedHeaders": [
+            "*"
+        ],
+        "AllowedMethods": [
+            "PUT"
+        ],
+        "AllowedOrigins": [
+            "*"
+        ],
+        "ExposeHeaders": []
+    }
+]
+```
diff --git a/packages/ddex/ingester/.air.toml b/packages/ddex/ingester/.air.toml
index d9e1804c565..77d37048601 100644
--- a/packages/ddex/ingester/.air.toml
+++ b/packages/ddex/ingester/.air.toml
@@ -16,7 +16,7 @@ tmp_dir = "tmp"
   include_dir = []
   include_ext = ["go", "tpl", "tmpl", "html"]
   include_file = []
-  kill_delay = "0s"
+  kill_delay = "5s"
   log = "build-errors.log"
   poll = false
   poll_interval = 0
@@ -24,7 +24,7 @@ tmp_dir = "tmp"
   pre_cmd = []
   rerun = false
   rerun_delay = 500
-  send_interrupt = false
+  send_interrupt = true
   stop_on_error = false
 
 [color]
@@ -39,7 +39,7 @@ tmp_dir = "tmp"
   time = false
 
 [misc]
-  clean_on_exit = false
+  clean_on_exit = true
 
 [screen]
   clear_on_rebuild = false
diff --git a/packages/ddex/ingester/README.md b/packages/ddex/ingester/README.md
index bd40aea7c57..52913d24e82 100644
--- a/packages/ddex/ingester/README.md
+++ b/packages/ddex/ingester/README.md
@@ -7,4 +7,5 @@ Indexes and parses new DDEX uploads.
 
 To run an ingester service locally with hot reloading:
 1. Make sure you can connect to mongo at `mongodb://mongo:mongo@localhost:27017/ddex`. See `packages/ddex/README.md` on how to spin up `ddex-mongo` and the other ddex containers.
+2. Make sure you've configured your `packages/ddex/.env` and S3 buckets according to the toplevel DDEX README.
 2. `air -c .air.toml -- --service [crawler|indexer|parser]`
diff --git a/packages/ddex/ingester/cmd/main.go b/packages/ddex/ingester/cmd/main.go
index a1f12b7fdda..83edb66451a 100644
--- a/packages/ddex/ingester/cmd/main.go
+++ b/packages/ddex/ingester/cmd/main.go
@@ -1,33 +1,55 @@
 package main
 
 import (
+	"context"
 	"flag"
 	"fmt"
 	"ingester/crawler"
 	"ingester/indexer"
 	"ingester/parser"
+	"log"
 	"os"
 	"os/signal"
 	"syscall"
+
+	"github.com/joho/godotenv"
 )
 
 func main() {
 	service := flag.String("service", "", "Specify the service to run: crawler, indexer, or parser")
 	flag.Parse()
 
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	go func() {
+		sigChan := make(chan os.Signal, 1)
+		signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
+		sig := <-sigChan
+		fmt.Printf("Received signal: %v, shutting down...\n", sig)
+		cancel()
+	}()
+
+	err := godotenv.Load("../.env")
+	if err != nil {
+		if os.IsNotExist(err) {
+			log.Println("No .env file found, proceeding with existing environment variables")
+		} else {
+			log.Fatalf("Error loading .env file: %v", err)
+		}
+	}
+
 	switch *service {
 	case "crawler":
-		crawler.Run()
+		go crawler.Run(ctx)
 	case "indexer":
-		indexer.Run()
+		go indexer.Run(ctx)
 	case "parser":
-		parser.Run()
+		go parser.Run(ctx)
 	default:
 		fmt.Println("Unknown service: " + *service)
-		// sleep
-		sigChan := make(chan os.Signal, 1)
-		signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
-		sig := <-sigChan
-		fmt.Printf("Received signal: %v, shutting down...\n", sig)
 	}
+
+	<-ctx.Done() // Wait until the context is canceled
+	fmt.Println("Service stopped")
 }
diff --git a/packages/ddex/ingester/common/common.go b/packages/ddex/ingester/common/common.go
new file mode 100644
index 00000000000..7b1c9c3f8b3
--- /dev/null
+++ b/packages/ddex/ingester/common/common.go
@@ -0,0 +1,54 @@
+package common
+
+import (
+	"context"
+	"log"
+	"os"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"go.mongodb.org/mongo-driver/mongo"
+	"go.mongodb.org/mongo-driver/mongo/options"
+)
+
+func InitMongoClient(ctx context.Context) *mongo.Client {
+	mongoUrl := os.Getenv("DDEX_MONGODB_URL")
+	if mongoUrl == "" {
+		mongoUrl = "mongodb://mongo:mongo@localhost:27017/ddex?authSource=admin&replicaSet=rs0"
+	}
+	client, err := mongo.Connect(ctx, options.Client().ApplyURI(mongoUrl))
+	if err != nil {
+		panic(err)
+	}
+	log.Println("Connected to mongo")
+	return client
+}
+
+func InitS3Client() *s3.S3 {
+	awsRegion := MustGetenv("AWS_REGION")
+	awsKey := MustGetenv("AWS_ACCESS_KEY_ID")
+	awsSecret := MustGetenv("AWS_SECRET_ACCESS_KEY")
+	sess, err := session.NewSession(&aws.Config{
+		Region:      aws.String(awsRegion),
+		Credentials: credentials.NewStaticCredentials(awsKey, awsSecret, ""),
+	})
+	if err != nil {
+		panic(err)
+	}
+	log.Println("Connected to s3")
+	return s3.New(sess)
+}
+
+func MustGetenv(key string) string {
+	val := os.Getenv(key)
+	if val == "" {
+		log.Println("Missing required env variable: ", key, " sleeping ...")
+		// If config is incorrect, sleep a bit to prevent container from restarting constantly
+		time.Sleep(time.Hour)
+		log.Fatal("Missing required env variable: ", key)
+	}
+	return val
+}
diff --git a/packages/ddex/ingester/crawler/crawler.go b/packages/ddex/ingester/crawler/crawler.go
index 4c2d8636cb0..a114aa715d0 100644
--- a/packages/ddex/ingester/crawler/crawler.go
+++ b/packages/ddex/ingester/crawler/crawler.go
@@ -1,9 +1,126 @@
 package crawler
 
 import (
-	"fmt"
+	"context"
+	"ingester/common"
+	"log"
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/mongo"
+	"go.mongodb.org/mongo-driver/mongo/options"
 )
 
-func Run() {
-	fmt.Printf("Crawler: TODO")
+func Run(ctx context.Context) {
+	s3Client := common.InitS3Client()
+	mongoClient := common.InitMongoClient(ctx)
+	defer mongoClient.Disconnect(ctx)
+
+	bucketName := common.MustGetenv("AWS_BUCKET_RAW")
+	// ticker := time.NewTicker(3 * time.Minute)
+	ticker := time.NewTicker(3 * time.Second)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+			lastPolledTime, err := getLastPolledTime(mongoClient, ctx)
+			if err != nil {
+				panic(err)
+			}
+
+			uploads, err := pollS3Bucket(s3Client, bucketName, lastPolledTime)
+			if err != nil {
+				log.Println("Error polling S3 bucket:", err)
+				continue
+			}
+
+			if len(uploads) > 0 {
+				err = persistUploads(mongoClient, bucketName, uploads, ctx)
+				if err != nil {
+					log.Println("Error inserting into mongodb:", err)
+				}
+				log.Printf("Processed %d new uploads", len(uploads))
+			}
+
+			err = updateLastPolledTime(mongoClient, time.Now(), ctx)
+			if err != nil {
+				log.Printf("Failed to update s3 raw bucket's last polled time: %v\n", err)
+			}
+
+			if ctx.Err() != nil {
+				return
+			}
+		}
+	}
+}
+
+func updateLastPolledTime(client *mongo.Client, lastPolledTime time.Time, ctx context.Context) error {
+	collection := client.Database("ddex").Collection("cursors")
+
+	filter := bson.M{"service": "crawler"}
+	update := bson.M{"$set": bson.M{"s3RawLastPolledTime": lastPolledTime}}
+	opts := options.Update().SetUpsert(true)
+
+	_, err := collection.UpdateOne(ctx, filter, update, opts)
+	return err
+}
+
+func getLastPolledTime(client *mongo.Client, ctx context.Context) (time.Time, error) {
+	collection := client.Database("ddex").Collection("cursors")
+
+	var result struct {
+		LastPolledTime time.Time `bson:"s3RawLastPolledTime"`
+	}
+	filter := bson.M{"service": "crawler"}
+
+	err := collection.FindOne(ctx, filter).Decode(&result)
+	if err != nil {
+		if err == mongo.ErrNoDocuments {
+			// No record found, return zero time
+			return time.Time{}, nil
+		}
+		return time.Time{}, err
+	}
+
+	return result.LastPolledTime, nil
+}
+
+func pollS3Bucket(s3Client *s3.S3, bucketName string, lastPolledTime time.Time) ([]*s3.Object, error) {
+	resp, err := s3Client.ListObjectsV2(&s3.ListObjectsV2Input{
+		Bucket: aws.String(bucketName),
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	var newUploads []*s3.Object
+	for _, item := range resp.Contents {
+		if item.LastModified.After(lastPolledTime) {
+			newUploads = append(newUploads, item)
+		}
+	}
+	return newUploads, nil
+}
+
+func persistUploads(client *mongo.Client, bucket string, uploads []*s3.Object, ctx context.Context) error {
+	uploadsColl := client.Database("ddex").Collection("uploads")
+	for _, upload := range uploads {
+		path := "s3://" + bucket + "/" + *upload.Key
+		etag := strings.Trim(*upload.ETag, "\"")
+		// Only insert if a document doesn't already exist with this path and etag
+		filter := bson.M{"path": path, "delivery_etag": etag}
+		update := bson.M{"$setOnInsert": bson.M{"path": path, "delivery_etag": etag}}
+		opts := options.Update().SetUpsert(true)
+		_, err := uploadsColl.UpdateOne(ctx, filter, update, opts)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
 }
diff --git a/packages/ddex/ingester/go.mod b/packages/ddex/ingester/go.mod
index 22084e6d4c8..a312b366a97 100644
--- a/packages/ddex/ingester/go.mod
+++ b/packages/ddex/ingester/go.mod
@@ -4,6 +4,7 @@ go 1.21.5
 
 require (
 	dario.cat/mergo v1.0.0 // indirect
+	github.com/aws/aws-sdk-go v1.50.11 // indirect
 	github.com/bep/godartsass v1.2.0 // indirect
 	github.com/bep/godartsass/v2 v2.0.0 // indirect
 	github.com/bep/golibsass v1.1.1 // indirect
@@ -14,6 +15,8 @@ require (
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/gohugoio/hugo v0.122.0 // indirect
 	github.com/golang/snappy v0.0.1 // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
+	github.com/joho/godotenv v1.5.1 // indirect
 	github.com/klauspost/compress v1.13.6 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
diff --git a/packages/ddex/ingester/go.sum b/packages/ddex/ingester/go.sum
index 0952587aa07..126ad9acc8f 100644
--- a/packages/ddex/ingester/go.sum
+++ b/packages/ddex/ingester/go.sum
@@ -2,6 +2,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT
 dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
 dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/aws/aws-sdk-go v1.50.11 h1:BxUqMbkzKzzQ6FXLlZFNFkFjHnv6utbc4PItVs5SvaE=
+github.com/aws/aws-sdk-go v1.50.11/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
 github.com/bep/godartsass v1.2.0 h1:E2VvQrxAHAFwbjyOIExAMmogTItSKodoKuijNrGm5yU=
 github.com/bep/godartsass v1.2.0/go.mod h1:6LvK9RftsXMxGfsA0LDV12AGc4Jylnu6NgHL+Q5/pE8=
 github.com/bep/godartsass/v2 v2.0.0 h1:Ruht+BpBWkpmW+yAM2dkp7RSSeN0VLaTobyW0CiSP3Y=
@@ -49,6 +51,11 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
+github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
+github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
 github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc=
 github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
 github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
@@ -174,6 +181,7 @@ google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHh
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
diff --git a/packages/ddex/ingester/indexer/indexer.go b/packages/ddex/ingester/indexer/indexer.go
index 41951345b5c..7f996f76453 100644
--- a/packages/ddex/ingester/indexer/indexer.go
+++ b/packages/ddex/ingester/indexer/indexer.go
@@ -2,41 +2,34 @@ package indexer
 
 import (
 	"context"
-	"go.mongodb.org/mongo-driver/bson"
-	"go.mongodb.org/mongo-driver/mongo"
-	"go.mongodb.org/mongo-driver/mongo/options"
+	"ingester/common"
 	"ingester/constants"
 	"log"
-	"os"
+
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/mongo"
 )
 
-func Run() {
-	mongoUrl := os.Getenv("DDEX_MONGODB_URL")
-	if mongoUrl == "" {
-		mongoUrl = "mongodb://mongo:mongo@localhost:27017/ddex?authSource=admin&replicaSet=rs0"
-	}
-	client, err := mongo.Connect(context.Background(), options.Client().ApplyURI(mongoUrl))
-	if err != nil {
-		panic(err)
-	}
-	log.Println("Indexer: connected to mongo")
-	defer client.Disconnect(context.Background())
+func Run(ctx context.Context) {
+	mongoClient := common.InitMongoClient(ctx)
+	defer mongoClient.Disconnect(ctx)
 
-	uploadsColl := client.Database("ddex").Collection("uploads")
-	changeStream, err := uploadsColl.Watch(context.Background(), mongo.Pipeline{})
+	uploadsColl := mongoClient.Database("ddex").Collection("uploads")
+	pipeline := mongo.Pipeline{bson.D{{"$match", bson.D{{"operationType", "insert"}}}}}
+	changeStream, err := uploadsColl.Watch(ctx, pipeline)
 	if err != nil {
 		panic(err)
 	}
-	log.Println("Indexer: watching collection 'uploads'")
-	defer changeStream.Close(context.Background())
+	log.Println("Watching collection 'uploads'")
+	defer changeStream.Close(ctx)
 
-	for changeStream.Next(context.Background()) {
+	for changeStream.Next(ctx) {
 		var changeDoc bson.M
 		if err := changeStream.Decode(&changeDoc); err != nil {
 			log.Fatal(err)
 		}
 		fullDocument, _ := changeDoc["fullDocument"].(bson.M)
-		indexUpload(client, fullDocument)
+		indexUpload(mongoClient, fullDocument, ctx)
 	}
 
 	if err := changeStream.Err(); err != nil {
@@ -44,8 +37,8 @@ func Run() {
 	}
 }
 
-func indexUpload(client *mongo.Client, fullDocument bson.M) {
-	log.Printf("Indexed: Processing new upload: %v\n", fullDocument)
+func indexUpload(client *mongo.Client, fullDocument bson.M, ctx context.Context) {
+	log.Printf("Processing new upload: %v\n", fullDocument)
 	indexedColl := client.Database("ddex").Collection("indexed")
 
 	delete(fullDocument, "_id")
@@ -60,10 +53,10 @@ func indexUpload(client *mongo.Client, fullDocument bson.M) {
 	// TODO download xml from bucket
 	// fullDocument["delivery_xml"] = ...
 
-	result, err := indexedColl.InsertOne(context.Background(), fullDocument)
+	result, err := indexedColl.InsertOne(ctx, fullDocument)
 	if err != nil {
 		log.Fatal(err)
 	}
 
-	log.Println("Indexer: New indexed doc ID: ", result.InsertedID)
+	log.Println("New indexed doc ID: ", result.InsertedID)
 }
diff --git a/packages/ddex/ingester/parser/parser.go b/packages/ddex/ingester/parser/parser.go
index 65eee49a1ca..45db16b6dc5 100644
--- a/packages/ddex/ingester/parser/parser.go
+++ b/packages/ddex/ingester/parser/parser.go
@@ -2,44 +2,36 @@ package parser
 
 import (
 	"context"
-	"go.mongodb.org/mongo-driver/bson"
-	"go.mongodb.org/mongo-driver/bson/primitive"
-	"go.mongodb.org/mongo-driver/mongo"
-	"go.mongodb.org/mongo-driver/mongo/options"
+	"ingester/common"
 	"ingester/constants"
 	"log"
-	"os"
 	"time"
+
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"go.mongodb.org/mongo-driver/mongo"
 )
 
-func Run() {
-	mongoUrl := os.Getenv("DDEX_MONGODB_URL")
-	if mongoUrl == "" {
-		mongoUrl = "mongodb://mongo:mongo@localhost:27017/ddex?authSource=admin&replicaSet=rs0"
-	}
-	client, err := mongo.Connect(context.Background(), options.Client().ApplyURI(mongoUrl))
-	if err != nil {
-		panic(err)
-	}
-	log.Println("Parser: connected to mongo")
-	defer client.Disconnect(context.Background())
+func Run(ctx context.Context) {
+	mongoClient := common.InitMongoClient(ctx)
+	defer mongoClient.Disconnect(ctx)
 
-	indexedColl := client.Database("ddex").Collection("indexed")
+	indexedColl := mongoClient.Database("ddex").Collection("indexed")
 	pipeline := mongo.Pipeline{bson.D{{"$match", bson.D{{"operationType", "insert"}}}}}
-	changeStream, err := indexedColl.Watch(context.Background(), pipeline)
+	changeStream, err := indexedColl.Watch(ctx, pipeline)
 	if err != nil {
 		panic(err)
 	}
-	log.Println("Parser: watching collection 'indexed'")
-	defer changeStream.Close(context.Background())
+	log.Println("Watching collection 'indexed'")
+	defer changeStream.Close(ctx)
 
-	for changeStream.Next(context.Background()) {
+	for changeStream.Next(ctx) {
 		var changeDoc bson.M
 		if err := changeStream.Decode(&changeDoc); err != nil {
 			log.Fatal(err)
 		}
 		fullDocument, _ := changeDoc["fullDocument"].(bson.M)
-		parseIndexed(client, fullDocument)
+		parseIndexed(mongoClient, fullDocument, ctx)
 	}
 
 	if err := changeStream.Err(); err != nil {
@@ -47,8 +39,8 @@ func Run() {
 	}
 }
 
-func parseIndexed(client *mongo.Client, fullDocument bson.M) {
-	log.Printf("Parser: Processing new indexed document: %v\n", fullDocument)
+func parseIndexed(client *mongo.Client, fullDocument bson.M, ctx context.Context) {
+	log.Printf("Processing new indexed document: %v\n", fullDocument)
 	indexedColl := client.Database("ddex").Collection("indexed")
 	parsedColl := client.Database("ddex").Collection("parsed")
 
@@ -57,25 +49,25 @@ func parseIndexed(client *mongo.Client, fullDocument bson.M) {
 
 	session, err := client.StartSession()
 	if err != nil {
-		failAndUpdateStatus(err, indexedColl, context.Background(), fullDocument["_id"].(primitive.ObjectID))
+		failAndUpdateStatus(err, indexedColl, ctx, fullDocument["_id"].(primitive.ObjectID))
 	}
-	err = mongo.WithSession(context.Background(), session, func(sessionContext mongo.SessionContext) error {
+	err = mongo.WithSession(ctx, session, func(sessionContext mongo.SessionContext) error {
 		if err := session.StartTransaction(); err != nil {
 			return err
 		}
 
 		// 2. Write each release in "delivery_xml" in the indexed doc as a bson doc in the 'parsed' collection
 		parsedDoc := bson.M{
-			"delivery_id":  fullDocument["delivery_id"],
-			"entity":       "track",
-			"publish_date": time.Now(),
+			"delivery_etag": fullDocument["delivery_etag"],
+			"entity":        "track",
+			"publish_date":  time.Now(),
 		}
-		result, err := parsedColl.InsertOne(context.Background(), parsedDoc)
+		result, err := parsedColl.InsertOne(ctx, parsedDoc)
 		if err != nil {
 			session.AbortTransaction(sessionContext)
 			return err
 		}
-		log.Println("Parser: New parsed release doc ID: ", result.InsertedID)
+		log.Println("New parsed release doc ID: ", result.InsertedID)
 
 		// 3. Set delivery status for delivery in 'indexed' collection
 		err = setDeliveryStatus(indexedColl, sessionContext, fullDocument["_id"].(primitive.ObjectID), constants.DeliveryStatusAwaitingPublishing)
@@ -88,19 +80,19 @@ func parseIndexed(client *mongo.Client, fullDocument bson.M) {
 	})
 
 	if err != nil {
-		failAndUpdateStatus(err, indexedColl, context.Background(), fullDocument["_id"].(primitive.ObjectID))
+		failAndUpdateStatus(err, indexedColl, ctx, fullDocument["_id"].(primitive.ObjectID))
 	}
 
-	session.EndSession(context.Background())
+	session.EndSession(ctx)
 }
 
-func setDeliveryStatus(collection *mongo.Collection, context context.Context, documentId primitive.ObjectID, status string) error {
+func setDeliveryStatus(collection *mongo.Collection, ctx context.Context, documentId primitive.ObjectID, status string) error {
 	update := bson.M{"$set": bson.M{"delivery_status": status}}
-	_, err := collection.UpdateByID(context, documentId, update)
+	_, err := collection.UpdateByID(ctx, documentId, update)
 	return err
 }
 
-func failAndUpdateStatus(err error, collection *mongo.Collection, context context.Context, documentId primitive.ObjectID) {
-	setDeliveryStatus(collection, context, documentId, constants.DeliveryStatusError)
+func failAndUpdateStatus(err error, collection *mongo.Collection, ctx context.Context, documentId primitive.ObjectID) {
+	setDeliveryStatus(collection, ctx, documentId, constants.DeliveryStatusError)
 	log.Fatal(err)
 }
diff --git a/packages/ddex/publisher/.env.stage b/packages/ddex/publisher/.env.stage
deleted file mode 100644
index 873d8e46186..00000000000
--- a/packages/ddex/publisher/.env.stage
+++ /dev/null
@@ -1,3 +0,0 @@
-NODE_ENV='stage'
-DDEX_KEY='49d5e13d355709b615b7cce7369174fb240b6b39'
-DDEX_SECRET='2b2c2b90d9a489234ae629a5284de84fb0633306257f17667aaebf2345d92152'
diff --git a/packages/ddex/publisher/README.md b/packages/ddex/publisher/README.md
index f2a2ab8f9f6..652efbff585 100644
--- a/packages/ddex/publisher/README.md
+++ b/packages/ddex/publisher/README.md
@@ -3,7 +3,10 @@
 Server that publishes DDEX entities queued for release by the DDEX ingester.
 
 ### Local Dev
+Setup:
+Make sure you've configured your `packages/ddex/.env` and S3 buckets according to the toplevel DDEX README.
+
 Run the server:
 1. Make sure you can connect to mongo at `mongodb://mongo:mongo@localhost:27017/ddex`. See `packages/ddex/README.md` on how to spin up `ddex-mongo` and the other ddex containers.
 2. At the monorepo root: `npm i`
-3. At packages/ddex/publisher: `npm run dev:[stage|prod]`
+3. At packages/ddex/publisher: `npm run dev:[dev|stage|prod]`
diff --git a/packages/ddex/publisher/package.json b/packages/ddex/publisher/package.json
index 773492e49c5..11c329f48d1 100644
--- a/packages/ddex/publisher/package.json
+++ b/packages/ddex/publisher/package.json
@@ -7,6 +7,7 @@
     "build": "tsc",
     "start:prod": "NODE_ENV=production node dist/index.js",
     "start": "nodemon src/index.ts",
+    "dev:dev": "NODE_ENV=dev turbo run start --filter=@audius/ddex-publisher",
     "dev:stage": "NODE_ENV=stage turbo run start --filter=@audius/ddex-publisher",
     "dev:prod": "NODE_ENV=production turbo run start --filter=@audius/ddex-publisher",
     "lint": "eslint src --ext ts,tsx --report-unused-disable-directives --max-warnings 0",
diff --git a/packages/ddex/publisher/src/index.ts b/packages/ddex/publisher/src/index.ts
index 7c472a0975b..a7615f2fa21 100644
--- a/packages/ddex/publisher/src/index.ts
+++ b/packages/ddex/publisher/src/index.ts
@@ -1,9 +1,8 @@
 import dotenv from 'dotenv'
 import path from 'path'
 
-// Load env vars based on NODE_ENV
-const envFile = process.env.NODE_ENV === 'stage' ? '.env.stage' : '.env'
-dotenv.config({ path: path.resolve(process.cwd(), envFile) })
+// Load env vars from ddex package root
+dotenv.config({ path: path.join(__dirname, '..', '..', '..', '.env') })
 
 import createApp from './app'
 import { dialDb } from './services/dbService'
diff --git a/packages/ddex/webapp/README.md b/packages/ddex/webapp/README.md
index a24d248ef10..b2b37e3a2b0 100644
--- a/packages/ddex/webapp/README.md
+++ b/packages/ddex/webapp/README.md
@@ -4,49 +4,16 @@ tRPC server and client for the Audius DDEX control layer (UI for suppliers to do
 
 ### Local Dev
 Setup:
-1. `cp .env.dev .env`
-2. (only required to upload to S3 from the UI) Follow the instructions below to create a bucket in S3 and add the credentials to your .env file
+Make sure you've configured your `packages/ddex/.env` and S3 buckets according to the toplevel DDEX README.
 
 Run the server:
 1. Make sure you can connect to mongo at `mongodb://mongo:mongo@localhost:27017/ddex`. See `packages/ddex/README.md` on how to spin up `ddex-mongo` and the other ddex containers.
 2. At the monorepo root: `npm i`
-3. At packages/ddex/webapp/server: `npm run dev:[stage|prod]`  
+3. At packages/ddex/webapp/server: `npm run dev:[dev|stage|prod]`
 
 Run the client:
 If you want to run the frontend locally, you'll need to:
-1. At packages/ddex/webapp/client: `npm run start:[stage|prod]`
+1. At packages/ddex/webapp/client: `npm run start:[dev|stage|prod]`
 
 Notes:
 * When running on stage or prod, the backend serves the frontend as static assets at the root path
-
-### Creating a bucket in S3
-1. Create a new bucket in the S3 console with the name `ddex-[dev|staging]-<label/distributor>-raw`. Use all the defaults, including "ACLs disabled"
-2. Do the same for a bucket named `ddex-[dev|staging]-<label/distributor>-indexed`. Use all the defaults, including "ACLs disabled"
-3. Create an IAM Policy (here](https://us-east-1.console.aws.amazon.com/iamv2/home?region=us-west-2#/policies/create) (or search IAM and click Policies > Create Policy). Select S3.
-    * Under `Read` choose `GetObject` and `GetObjectAttributes`.
-    * Under `Write` choose `DeleteObject` and `PutObject`.
-    * Under `List` choose `ListBucket`.
-    * Click `Add Arn`, enter the bucket name ending with `raw`, and check the box for `Any object name`.
-    * Click `Add Arn` again, enter the bucket name ending with `indexed`, and check the box for `Any object name`.
-    * Click Next, and then name the policy `ddex-[dev|staging]-<label/distributor>-policy`.
-4. Create an IAM User [here](https://us-east-1.console.aws.amazon.com/iamv2/home?region=us-west-2#/users/create) (or search IAM and click Users > Create User).
-    * Name the user `ddex-[dev|staging]-<label/distributor>-user` and press Next.
-    * Select "Attach policies directly," and search for the policy you created (`ddex-[dev|staging]-<label/distributor>-policy`). Check the box next to it and press Next and then Create User.
-5. Search for your new user and press "Create access key" and then "Third-party service." Copy the access key and secret access key into your .env file (assuming you've already done `cp .env.dev .env`).
-6. Go back to the bucket ending with `raw`, and add CORS at the bottom of the Permissions tab. Here's an example for dev, but for a prod environment you'll wnat to replace "*" in "AllowedOrigins" with the DNS that the frontend will be served from:
-```json
-[
-    {
-        "AllowedHeaders": [
-            "*"
-        ],
-        "AllowedMethods": [
-            "PUT"
-        ],
-        "AllowedOrigins": [
-            "*"
-        ],
-        "ExposeHeaders": []
-    }
-]
-```
diff --git a/packages/ddex/webapp/client/src/components/Collection/Collection.tsx b/packages/ddex/webapp/client/src/components/Collection/Collection.tsx
index db23ec76a45..3b513725db6 100644
--- a/packages/ddex/webapp/client/src/components/Collection/Collection.tsx
+++ b/packages/ddex/webapp/client/src/components/Collection/Collection.tsx
@@ -8,107 +8,23 @@ import styles from './Collection.module.css'
 
 type CollectionT = 'uploads' | 'indexed' | 'parsed' | 'published'
 
-const Table = ({
-  collection,
-  data
-}: {
-  collection: CollectionT
-  data: any
-}) => {
-  switch (collection) {
-    case 'uploads':
-      return (
-        <table className={styles.styledTable}>
-          <thead>
-            <tr>
-              <th>ID</th>
-              <th>Uploaded By</th>
-              <th>Uploaded At</th>
-              <th>Status</th>
-            </tr>
-          </thead>
-          <tbody>
-            {data.items.map((item: any) => (
-              <tr key={item.id}>
-                <td>TODO</td>
-                <td>TODO</td>
-                <td>TODO</td>
-                <td>TODO</td>
-              </tr>
-            ))}
-          </tbody>
-        </table>
-      )
-    case 'indexed':
-      return (
-        <table className={styles.styledTable}>
-          <thead>
-            <tr>
-              <th>Upload ID</th>
-              <th>TODO</th>
-              <th>TODO</th>
-              <th>TODO</th>
-            </tr>
-          </thead>
-          <tbody>
-            {data.items.map((item: any) => (
-              <tr key={item.id}>
-                <td>TODO</td>
-                <td>TODO</td>
-                <td>TODO</td>
-                <td>TODO</td>
-              </tr>
-            ))}
-          </tbody>
-        </table>
-      )
-    case 'parsed':
-      return (
-        <table className={styles.styledTable}>
-          <thead>
-            <tr>
-              <th>Upload ID</th>
-              <th>TODO</th>
-              <th>TODO</th>
-              <th>TODO</th>
-            </tr>
-          </thead>
-          <tbody>
-            {data.items.map((item: any) => (
-              <tr key={item.id}>
-                <td>TODO</td>
-                <td>TODO</td>
-                <td>TODO</td>
-                <td>TODO</td>
-              </tr>
-            ))}
-          </tbody>
-        </table>
-      )
-    case 'published':
-      return (
-        <table className={styles.styledTable}>
-          <thead>
-            <tr>
-              <th>Upload ID</th>
-              <th>TODO</th>
-              <th>TODO</th>
-              <th>TODO</th>
-            </tr>
-          </thead>
-          <tbody>
-            {data.items.map((item: any) => (
-              <tr key={item.id}>
-                <td>TODO</td>
-                <td>TODO</td>
-                <td>TODO</td>
-                <td>TODO</td>
-              </tr>
-            ))}
-          </tbody>
-        </table>
-      )
-  }
+const Table = ({ data }: { data: any }) => {
+  return (
+    <table className={styles.styledTable}>
+      <thead>
+        <tr>
+          <th>Items</th>
+        </tr>
+      </thead>
+      <tbody>
+        {data.items.map((item: any) => (
+          <tr key={item._id}>
+            <td>{JSON.stringify(item)}</td>
+          </tr>
+        ))}
+      </tbody>
+    </table>
+  )
 }
 
 export const Collection = ({ collection }: { collection: CollectionT }) => {
@@ -167,7 +83,7 @@ export const Collection = ({ collection }: { collection: CollectionT }) => {
         <Text variant='body' color='default'>
           {isLoading && <div>Loading...</div>}
           {error && <div>Error: {error.message}</div>}
-          {data && <Table collection={collection} data={data} />}
+          {data && <Table data={data} />}
 
           <Flex justifyContent='space-between'>
             <Button
diff --git a/packages/ddex/webapp/server/.env.dev b/packages/ddex/webapp/server/.env.dev
deleted file mode 100644
index b67c11183c0..00000000000
--- a/packages/ddex/webapp/server/.env.dev
+++ /dev/null
@@ -1,4 +0,0 @@
-AWS_ACCESS_KEY_ID='see README'
-AWS_SECRET_ACCESS_KEY='see README'
-AWS_REGION='us-west-2'
-AWS_BUCKET='ddex-dev-audius-raw'
\ No newline at end of file
diff --git a/packages/ddex/webapp/server/package.json b/packages/ddex/webapp/server/package.json
index 9ab0c95e7fd..3e13e4f6f99 100644
--- a/packages/ddex/webapp/server/package.json
+++ b/packages/ddex/webapp/server/package.json
@@ -7,6 +7,7 @@
     "build": "tsc",
     "start:prod": "NODE_ENV=production node dist/index.js",
     "start": "IS_DEV=true nodemon src/index.ts",
+    "dev:dev": "NODE_ENV=dev turbo run start --filter=@audius/ddex-webapp-server",
     "dev:stage": "NODE_ENV=stage turbo run start --filter=@audius/ddex-webapp-server",
     "dev:prod": "NODE_ENV=production turbo run start --filter=@audius/ddex-webapp-server",
     "lint": "eslint src --ext ts,tsx --report-unused-disable-directives --max-warnings 0",
diff --git a/packages/ddex/webapp/server/src/app.ts b/packages/ddex/webapp/server/src/app.ts
index 6b49fe3cd8d..34b870e9e82 100644
--- a/packages/ddex/webapp/server/src/app.ts
+++ b/packages/ddex/webapp/server/src/app.ts
@@ -15,6 +15,9 @@ export default function createApp() {
     const envData = {
       data: {
         env: process.env.NODE_ENV,
+        awsAccessKeyId: process.env.AWS_ACCESS_KEY_ID,
+        awsBucketRaw: process.env.AWS_BUCKET_RAW,
+        awsBucketIndexed: process.env.AWS_BUCKET_INDEXED,
         ddexKey: process.env.DDEX_KEY,
         optimizelySdkKey: process.env.OPTIMIZELY_SDK_KEY,
       },
diff --git a/packages/ddex/webapp/server/src/index.ts b/packages/ddex/webapp/server/src/index.ts
index 2f0ba56e67b..43f5039e74f 100644
--- a/packages/ddex/webapp/server/src/index.ts
+++ b/packages/ddex/webapp/server/src/index.ts
@@ -2,11 +2,8 @@ import dotenv from 'dotenv'
 import path from 'path'
 import createS3 from './services/s3'
 
-// Load env vars based on NODE_ENV
-if (process.env.NODE_ENV === 'stage') {
-  dotenv.config({ path: path.resolve(process.cwd(), '.env.stage') })
-}
-dotenv.config({ path: path.resolve(process.cwd(), '.env') })
+// Load env vars from ddex package root
+dotenv.config({ path: path.join(__dirname, '..', '..', '..', '.env') })
 
 import createApp from './app'
 import { dialDb } from './services/dbService'
diff --git a/packages/ddex/webapp/server/src/services/dbService.ts b/packages/ddex/webapp/server/src/services/dbService.ts
index f2d58e25307..0dd46437f9e 100644
--- a/packages/ddex/webapp/server/src/services/dbService.ts
+++ b/packages/ddex/webapp/server/src/services/dbService.ts
@@ -1,25 +1,6 @@
 import mongoose from 'mongoose'
-import { collections } from '../controllers/collectionController'
-
-const runMigrations = async () => {
-  console.log('Running migrations...')
-  const existingCollections = await mongoose.connection.db
-    .listCollections()
-    .toArray()
-  const existingCollectionNames = existingCollections.map((c) => c.name)
-  for (const collection of collections) {
-    if (!existingCollectionNames.includes(collection)) {
-      await mongoose.connection.db.createCollection(collection)
-      console.log(`Created collection ${collection}!`)
-    } else {
-      console.log(`Collection ${collection} already exists.`)
-    }
-  }
-}
 
 export const dialDb = async (dbUrl: string) => {
   await mongoose.connect(dbUrl)
   console.log('MongoDB connected...')
-
-  await runMigrations()
 }
diff --git a/packages/ddex/webapp/server/src/services/s3.ts b/packages/ddex/webapp/server/src/services/s3.ts
index 946eddb33e4..35dda451e76 100644
--- a/packages/ddex/webapp/server/src/services/s3.ts
+++ b/packages/ddex/webapp/server/src/services/s3.ts
@@ -13,7 +13,7 @@ export default function createS3() {
     },
     region: process.env.AWS_REGION,
   }
-  const bucket = process.env.AWS_BUCKET
+  const bucket = process.env.AWS_BUCKET_RAW
   const client = new S3Client(config)
 
   async function getSignedFileUrl(

From 94b2fb1cb4aea70250ef7d179f302312b239b673 Mon Sep 17 00:00:00 2001
From: Michelle Brier <michelle.brier4@gmail.com>
Date: Mon, 5 Feb 2024 23:43:01 -0800
Subject: [PATCH 2/3] cleanup

---
 dev-tools/compose/docker-compose.ddex.yml | 40 +++++++++++------------
 dev-tools/compose/docker-compose.yml      | 10 +++---
 packages/ddex/ingester/cmd/main.go        |  2 +-
 packages/ddex/ingester/crawler/crawler.go |  9 ++---
 4 files changed, 31 insertions(+), 30 deletions(-)

diff --git a/dev-tools/compose/docker-compose.ddex.yml b/dev-tools/compose/docker-compose.ddex.yml
index 287739a294a..9c4fd683aef 100644
--- a/dev-tools/compose/docker-compose.ddex.yml
+++ b/dev-tools/compose/docker-compose.ddex.yml
@@ -1,26 +1,26 @@
 version: '3.9'
 
 services:
-  # ddex-web:
-  #   container_name: ddex-web
-  #   build:
-  #     context: ${PROJECT_ROOT}
-  #     dockerfile: ${PROJECT_ROOT}/packages/ddex/webapp/Dockerfile
-  #     args:
-  #       app_name: ddex-web
-  #       TURBO_TEAM: '${TURBO_TEAM}'
-  #       TURBO_TOKEN: '${TURBO_TOKEN}'
-  #   environment:
-  #     - DDEX_PORT=9000
-  #     - DDEX_MONGODB_URL=mongodb://mongo:mongo@ddex-mongo:27017/ddex?authSource=admin&replicaSet=rs0
-  #   env_file:
-  #     - ${PROJECT_ROOT}/packages/ddex/.env
-  #   ports:
-  #     - "9000:9000"
-  #   networks:
-  #     - ddex-network
-  #   profiles:
-  #     - ddex
+  ddex-web:
+    container_name: ddex-web
+    build:
+      context: ${PROJECT_ROOT}
+      dockerfile: ${PROJECT_ROOT}/packages/ddex/webapp/Dockerfile
+      args:
+        app_name: ddex-web
+        TURBO_TEAM: '${TURBO_TEAM}'
+        TURBO_TOKEN: '${TURBO_TOKEN}'
+    environment:
+      - DDEX_PORT=9000
+      - DDEX_MONGODB_URL=mongodb://mongo:mongo@ddex-mongo:27017/ddex?authSource=admin&replicaSet=rs0
+    env_file:
+      - ${PROJECT_ROOT}/packages/ddex/.env
+    ports:
+      - "9000:9000"
+    networks:
+      - ddex-network
+    profiles:
+      - ddex
 
   ddex-crawler:
     container_name: ddex-crawler
diff --git a/dev-tools/compose/docker-compose.yml b/dev-tools/compose/docker-compose.yml
index 2511ef022a8..0c40ab60287 100644
--- a/dev-tools/compose/docker-compose.yml
+++ b/dev-tools/compose/docker-compose.yml
@@ -184,11 +184,11 @@ services:
 
   # DDEX
 
-  # ddex-web:
-  #   extends:
-  #     file: docker-compose.ddex.yml
-  #     service: ddex-web
-  #   <<: *common
+  ddex-web:
+    extends:
+      file: docker-compose.ddex.yml
+      service: ddex-web
+    <<: *common
 
   ddex-crawler:
     extends:
diff --git a/packages/ddex/ingester/cmd/main.go b/packages/ddex/ingester/cmd/main.go
index 83edb66451a..9a3207cc039 100644
--- a/packages/ddex/ingester/cmd/main.go
+++ b/packages/ddex/ingester/cmd/main.go
@@ -35,7 +35,7 @@ func main() {
 		if os.IsNotExist(err) {
 			log.Println("No .env file found, proceeding with existing environment variables")
 		} else {
-			log.Fatalf("Error loading .env file: %v", err)
+			log.Printf("Error loading .env file: %v\n", err)
 		}
 	}
 
diff --git a/packages/ddex/ingester/crawler/crawler.go b/packages/ddex/ingester/crawler/crawler.go
index a114aa715d0..3739e1b3fa5 100644
--- a/packages/ddex/ingester/crawler/crawler.go
+++ b/packages/ddex/ingester/crawler/crawler.go
@@ -20,8 +20,7 @@ func Run(ctx context.Context) {
 	defer mongoClient.Disconnect(ctx)
 
 	bucketName := common.MustGetenv("AWS_BUCKET_RAW")
-	// ticker := time.NewTicker(3 * time.Minute)
-	ticker := time.NewTicker(3 * time.Second)
+	ticker := time.NewTicker(3 * time.Minute)
 	defer ticker.Stop()
 
 	for {
@@ -31,7 +30,8 @@ func Run(ctx context.Context) {
 		case <-ticker.C:
 			lastPolledTime, err := getLastPolledTime(mongoClient, ctx)
 			if err != nil {
-				panic(err)
+				log.Println("Failed to retrieve s3 raw bucket's last polled time:", err)
+				continue
 			}
 
 			uploads, err := pollS3Bucket(s3Client, bucketName, lastPolledTime)
@@ -44,13 +44,14 @@ func Run(ctx context.Context) {
 				err = persistUploads(mongoClient, bucketName, uploads, ctx)
 				if err != nil {
 					log.Println("Error inserting into mongodb:", err)
+					continue
 				}
 				log.Printf("Processed %d new uploads", len(uploads))
 			}
 
 			err = updateLastPolledTime(mongoClient, time.Now(), ctx)
 			if err != nil {
-				log.Printf("Failed to update s3 raw bucket's last polled time: %v\n", err)
+				log.Println("Failed to update s3 raw bucket's last polled time:", err)
 			}
 
 			if ctx.Err() != nil {

From 0d321e780658379a698747a431aac2fcc6decc1b Mon Sep 17 00:00:00 2001
From: Michelle Brier <michelle.brier4@gmail.com>
Date: Mon, 5 Feb 2024 23:47:17 -0800
Subject: [PATCH 3/3] cleanup more

---
 packages/ddex/ingester/cmd/main.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/packages/ddex/ingester/cmd/main.go b/packages/ddex/ingester/cmd/main.go
index 9a3207cc039..b51ed4b5ce7 100644
--- a/packages/ddex/ingester/cmd/main.go
+++ b/packages/ddex/ingester/cmd/main.go
@@ -26,7 +26,7 @@ func main() {
 		sigChan := make(chan os.Signal, 1)
 		signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
 		sig := <-sigChan
-		fmt.Printf("Received signal: %v, shutting down...\n", sig)
+		log.Printf("Received signal: %v, shutting down...\n", sig)
 		cancel()
 	}()
 
@@ -35,7 +35,7 @@ func main() {
 		if os.IsNotExist(err) {
 			log.Println("No .env file found, proceeding with existing environment variables")
 		} else {
-			log.Printf("Error loading .env file: %v\n", err)
+			log.Println("Error loading .env file:", err)
 		}
 	}
 
@@ -51,5 +51,5 @@ func main() {
 	}
 
 	<-ctx.Done() // Wait until the context is canceled
-	fmt.Println("Service stopped")
+	log.Println("Service stopped")
 }