From 741805fc571df5336f258ba08a9d8bc4ee8cb1d8 Mon Sep 17 00:00:00 2001 From: Steve Perkins Date: Thu, 16 May 2024 13:55:55 -0400 Subject: [PATCH 1/6] it's yaml time --- .circleci/config.yml | 1 + .circleci/src/@continue_config.yml | 3 +++ .circleci/src/commands/@docker-commands.yml | 10 +++++++++- .circleci/src/workflows/ddex-processor.yml | 3 +++ .circleci/src/workflows/ddex-stage.yml | 7 +++++++ .circleci/src/workflows/discovery.yml | 8 ++++++++ .circleci/src/workflows/release.yml | 6 ++++++ dev-tools/compose/docker-compose.ddex.yml | 17 +++++++++++++++++ dev-tools/compose/docker-compose.yml | 9 +++++++++ 9 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 .circleci/src/workflows/ddex-processor.yml diff --git a/.circleci/config.yml b/.circleci/config.yml index a16c3e5b224..6ed818d64c3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -564,6 +564,7 @@ workflows: packages/ddex/webapp/.* run-ddex-webapp-workflow true packages/ddex/webapp/.* run-ddex-e2e-workflow true packages/ddex/ingester/.* run-ddex-e2e-workflow true + packages/ddex/processor/.* run-ddex-processor-workflow true packages/ddex/publisher/.* run-ddex-publisher-workflow true packages/create-audius-app/.* run-create-audius-app-workflow true protocol-dashboard/.* run-protocol-dashboard-workflow true diff --git a/.circleci/src/@continue_config.yml b/.circleci/src/@continue_config.yml index 92d4ab19dd8..99e5247421a 100644 --- a/.circleci/src/@continue_config.yml +++ b/.circleci/src/@continue_config.yml @@ -70,6 +70,9 @@ parameters: run-ddex-webapp-workflow: type: boolean default: false + run-ddex-processor-workflow: + type: boolean + default: false run-ddex-publisher-workflow: type: boolean default: false diff --git a/.circleci/src/commands/@docker-commands.yml b/.circleci/src/commands/@docker-commands.yml index a5928e798a8..b450565aada 100644 --- a/.circleci/src/commands/@docker-commands.yml +++ b/.circleci/src/commands/@docker-commands.yml @@ -19,7 +19,14 @@ docker-tag-images: enum: ['prerelease', 'edge', 'current', '__version__'] service: type: enum - enum: ['discovery-provider', 'creator-node', 'identity-service', 'ddex', 'all'] + enum: + [ + 'discovery-provider', + 'creator-node', + 'identity-service', + 'ddex', + 'all' + ] steps: - run: name: Get crane @@ -58,6 +65,7 @@ docker-tag-images: ddex=( ddex-ingester ddex-publisher + ddex-processor ddex-web ) diff --git a/.circleci/src/workflows/ddex-processor.yml b/.circleci/src/workflows/ddex-processor.yml new file mode 100644 index 00000000000..36c54dc61ce --- /dev/null +++ b/.circleci/src/workflows/ddex-processor.yml @@ -0,0 +1,3 @@ +when: << pipeline.parameters.run-ddex-processor-workflow >> +jobs: + - ddex-processor-init diff --git a/.circleci/src/workflows/ddex-stage.yml b/.circleci/src/workflows/ddex-stage.yml index 5f0eae42db0..11b3c658e46 100644 --- a/.circleci/src/workflows/ddex-stage.yml +++ b/.circleci/src/workflows/ddex-stage.yml @@ -26,6 +26,13 @@ jobs: filters: branches: only: main + - push-docker-image: + name: push-ddex-processor + context: [Vercel, dockerhub] + service: ddex-processor + filters: + branches: + only: main - push-docker-image: name: push-ddex-publisher context: [Vercel, dockerhub] diff --git a/.circleci/src/workflows/discovery.yml b/.circleci/src/workflows/discovery.yml index d14599962e0..6b91372e008 100644 --- a/.circleci/src/workflows/discovery.yml +++ b/.circleci/src/workflows/discovery.yml @@ -112,6 +112,13 @@ jobs: filters: branches: only: main + - push-docker-image: + name: push-ddex-processor + context: [Vercel, dockerhub] + service: ddex-processor + filters: + branches: + only: main - push-docker-image: name: push-ddex-publisher context: [Vercel, dockerhub] @@ -306,6 +313,7 @@ jobs: - push-healthz - push-ddex-web - push-ddex-ingester + - push-ddex-processor - push-ddex-publisher # uncomment when arm builds are stable # - push-discovery-provider-arm diff --git a/.circleci/src/workflows/release.yml b/.circleci/src/workflows/release.yml index f88a9339f76..774dd4ac804 100644 --- a/.circleci/src/workflows/release.yml +++ b/.circleci/src/workflows/release.yml @@ -90,6 +90,11 @@ jobs: context: [GCP, dockerhub, slack-secrets] service: ddex-ingester notify_slack_on_failure: true + - push-docker-image: + name: push-ddex-processor + context: [GCP, dockerhub, slack-secrets] + service: ddex-processor + notify_slack_on_failure: true - push-docker-image: name: push-ddex-publisher context: [GCP, dockerhub, slack-secrets] @@ -227,6 +232,7 @@ jobs: - push-uptime - push-ddex-web - push-ddex-ingester + - push-ddex-processor - push-ddex-publisher # uncomment these when arm builds are stable # - push-identity-service-arm diff --git a/dev-tools/compose/docker-compose.ddex.yml b/dev-tools/compose/docker-compose.ddex.yml index 483b11f3374..334424efee6 100644 --- a/dev-tools/compose/docker-compose.ddex.yml +++ b/dev-tools/compose/docker-compose.ddex.yml @@ -1,6 +1,23 @@ version: '3.9' services: + ddex-processor: + container_name: ddex-processor + build: + context: ${PROJECT_ROOT} + dockerfile: ${PROJECT_ROOT}/packages/ddex/processor/Dockerfile + args: + app_name: ddex-processor + TURBO_TEAM: '${TURBO_TEAM}' + TURBO_TOKEN: '${TURBO_TOKEN}' + env_file: .env + ports: + - "8889:8989" + networks: + - ddex-network + profiles: + - ddex + ddex-web: container_name: ddex-web build: diff --git a/dev-tools/compose/docker-compose.yml b/dev-tools/compose/docker-compose.yml index 5fd2f19ca03..21c6d93f85c 100644 --- a/dev-tools/compose/docker-compose.yml +++ b/dev-tools/compose/docker-compose.yml @@ -225,6 +225,15 @@ services: profiles: - ddex + ddex-processor: + extends: + file: docker-compose.ddex.yml + service: ddex-processor + container_name: ddex-processor + <<: *common + profiles: + - ddex + ddex-publisher: extends: file: docker-compose.ddex.yml From 74747259e9e4cee47a217b068008c9545b810399 Mon Sep 17 00:00:00 2001 From: Steve Perkins Date: Thu, 16 May 2024 15:59:14 -0400 Subject: [PATCH 2/6] yaml --- .circleci/config.yml | 3 +++ .circleci/src/jobs/@ddex-jobs.yml | 29 +++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6ed818d64c3..0c094c3daf7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -475,6 +475,7 @@ workflows: packages/ddex/webapp/.* run-ddex-webapp-workflow true packages/ddex/webapp/.* run-ddex-e2e-workflow true packages/ddex/ingester/.* run-ddex-e2e-workflow true + packages/ddex/processor/.* run-ddex-processor-workflow true packages/ddex/publisher/.* run-ddex-publisher-workflow true eth-contracts/.* run-eth-contracts-workflow true monitoring/healthz/.* run-healthz-workflow true @@ -487,6 +488,7 @@ workflows: .circleci/.* run-protocol-dashboard-workflow true .circleci/.* run-ddex-webapp-workflow true .circleci/.* run-ddex-e2e-workflow true + .circleci/.* run-ddex-processor-workflow true .circleci/.* run-ddex-publisher-workflow true .circleci/.* run-healthz-workflow true .circleci/.* run-contracts-workflow true @@ -515,6 +517,7 @@ workflows: packages/libs/.* run-identity-workflow true packages/libs/.* run-protocol-dashboard-workflow true packages/libs/.* run-ddex-webapp-workflow true + packages/libs/.* run-ddex-processor-workflow true packages/libs/.* run-ddex-publisher-workflow true packages/libs/.* run-sdk-workflow true packages/libs/.* run-web-workflow true diff --git a/.circleci/src/jobs/@ddex-jobs.yml b/.circleci/src/jobs/@ddex-jobs.yml index 585688b1880..125057e4548 100644 --- a/.circleci/src/jobs/@ddex-jobs.yml +++ b/.circleci/src/jobs/@ddex-jobs.yml @@ -55,3 +55,32 @@ ddex-publisher-init: - node_modules - packages/ddex/node_modules - packages/ddex/publisher/node_modules + +ddex-processor-init: + working_directory: ~/audius-protocol + docker: + - image: cimg/node:18.17 + steps: + - checkout + - attach_workspace: + at: ./ + + # Download and cache dependencies + - create_concatenated_patch_file: + filename: combined-patch-file.txt + + - restore_cache: + keys: + - cache-{{ checksum "package-lock.json" }}-{{ checksum "combined-patch-file.txt" }} + + - run: + name: lint & typecheck + command: | + npx turbo run verify --filter=@audius/ddex-processor + + - persist_to_workspace: + root: ./ + paths: + - node_modules + - packages/ddex/node_modules + - packages/ddex/processor/node_modules From 6e97b64209c259c238814e5eb440e04450322417 Mon Sep 17 00:00:00 2001 From: Steve Perkins Date: Thu, 16 May 2024 16:27:39 -0400 Subject: [PATCH 3/6] Update Dockerfile --- packages/ddex/processor/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ddex/processor/Dockerfile b/packages/ddex/processor/Dockerfile index 268166c098c..03fcf5ca13c 100644 --- a/packages/ddex/processor/Dockerfile +++ b/packages/ddex/processor/Dockerfile @@ -54,7 +54,7 @@ USER nodejs WORKDIR /app COPY --from=app-builder --chown=nodejs:nodejs /app . -WORKDIR /app/packages/ddex/publisher +WORKDIR /app/packages/ddex/processor EXPOSE 9001 CMD ["npm", "run", "start:prod"] From c9bf1e2134f2f0780bf15f5d5d972c97da8bb699 Mon Sep 17 00:00:00 2001 From: Steve Perkins Date: Thu, 16 May 2024 22:35:59 -0400 Subject: [PATCH 4/6] skip file-type --- packages/ddex/processor/src/server.ts | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/packages/ddex/processor/src/server.ts b/packages/ddex/processor/src/server.ts index 40aa0f0eed7..430f4a673b5 100644 --- a/packages/ddex/processor/src/server.ts +++ b/packages/ddex/processor/src/server.ts @@ -1,7 +1,6 @@ import 'dotenv/config' import { serve } from '@hono/node-server' -import { fileTypeFromBuffer } from 'file-type' import { Context, Hono } from 'hono' import { deleteCookie, getSignedCookie, setSignedCookie } from 'hono/cookie' import { html } from 'hono/html' @@ -390,12 +389,14 @@ app.get('/release/:key/:type/:ref/:name?', async (c) => { // some mime stuff if (asset.fileName.endsWith('flac')) { c.header('Content-Type', 'audio/flac') - } else { - const ft = await fileTypeFromBuffer(ok.buffer) - if (ft) { - c.header('Content-Type', ft.mime) - } } + // skip because of some file-type lint weirdness + // else { + // const ft = await fileTypeFromBuffer(ok.buffer) + // if (ft) { + // c.header('Content-Type', ft.mime) + // } + // } return c.body(ok.buffer) }) From f12df2c93a79bc78dc2e39a996cbaeb610b32cf0 Mon Sep 17 00:00:00 2001 From: Steve Perkins Date: Fri, 17 May 2024 10:14:48 -0400 Subject: [PATCH 5/6] fix ci lint weirdness --- packages/ddex/processor/package.json | 2 +- packages/ddex/processor/src/server.ts | 13 ++++++------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/packages/ddex/processor/package.json b/packages/ddex/processor/package.json index c8e825abd24..7ef552990a0 100644 --- a/packages/ddex/processor/package.json +++ b/packages/ddex/processor/package.json @@ -10,7 +10,7 @@ "start": "tsx watch cli.ts server", "lint": "eslint src --ext ts,tsx --report-unused-disable-directives --max-warnings 0", "lint:fix": "eslint --fix src --ext ts,tsx --report-unused-disable-directives --max-warnings 0", - "verify": "concurrently \"npm:build\" \"npm:lint\"", + "verify": "tsc --noEmit", "test": "SQLITE_URL=:memory: vitest run", "test:watch": "SQLITE_URL=:memory: vitest" }, diff --git a/packages/ddex/processor/src/server.ts b/packages/ddex/processor/src/server.ts index 430f4a673b5..40aa0f0eed7 100644 --- a/packages/ddex/processor/src/server.ts +++ b/packages/ddex/processor/src/server.ts @@ -1,6 +1,7 @@ import 'dotenv/config' import { serve } from '@hono/node-server' +import { fileTypeFromBuffer } from 'file-type' import { Context, Hono } from 'hono' import { deleteCookie, getSignedCookie, setSignedCookie } from 'hono/cookie' import { html } from 'hono/html' @@ -389,14 +390,12 @@ app.get('/release/:key/:type/:ref/:name?', async (c) => { // some mime stuff if (asset.fileName.endsWith('flac')) { c.header('Content-Type', 'audio/flac') + } else { + const ft = await fileTypeFromBuffer(ok.buffer) + if (ft) { + c.header('Content-Type', ft.mime) + } } - // skip because of some file-type lint weirdness - // else { - // const ft = await fileTypeFromBuffer(ok.buffer) - // if (ft) { - // c.header('Content-Type', ft.mime) - // } - // } return c.body(ok.buffer) }) From 4b63ba9b57c5f1b0cc98666aaee0d9c2233d7746 Mon Sep 17 00:00:00 2001 From: Steve Perkins Date: Fri, 17 May 2024 14:17:38 -0400 Subject: [PATCH 6/6] align file-type versions --- package-lock.json | 81 +-------------------------- packages/ddex/processor/package.json | 2 +- packages/ddex/processor/src/server.ts | 2 +- 3 files changed, 3 insertions(+), 82 deletions(-) diff --git a/package-lock.json b/package-lock.json index ad7d493f9c4..9eaa9f8c525 100644 --- a/package-lock.json +++ b/package-lock.json @@ -113828,7 +113828,7 @@ "commander": "12.0.0", "decompress": "4.2.1", "dotenv": "16.3.1", - "file-type": "19.0.0", + "file-type": "16.5.3", "hono": "4.2.5", "web3": "4.3.0" }, @@ -115788,22 +115788,6 @@ "fxparser": "src/cli/cli.js" } }, - "packages/ddex/processor/node_modules/file-type": { - "version": "19.0.0", - "resolved": "https://registry.npmjs.org/file-type/-/file-type-19.0.0.tgz", - "integrity": "sha512-s7cxa7/leUWLiXO78DVVfBVse+milos9FitauDLG1pI7lNaJ2+5lzPnr2N24ym+84HVwJL6hVuGfgVE+ALvU8Q==", - "dependencies": { - "readable-web-to-node-stream": "^3.0.2", - "strtok3": "^7.0.0", - "token-types": "^5.0.1" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sindresorhus/file-type?sponsor=1" - } - }, "packages/ddex/processor/node_modules/find-up": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", @@ -115882,25 +115866,6 @@ "node": ">=16.17.0" } }, - "packages/ddex/processor/node_modules/ieee754": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", - "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ] - }, "packages/ddex/processor/node_modules/is-stream": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-3.0.0.tgz", @@ -116080,18 +116045,6 @@ "node": ">=8" } }, - "packages/ddex/processor/node_modules/peek-readable": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/peek-readable/-/peek-readable-5.0.0.tgz", - "integrity": "sha512-YtCKvLUOvwtMGmrniQPdO7MwPjgkFBtFIrmfSbYmYuq3tKDV/mcfAhBth1+C3ru7uXIZasc/pHnb+YDYNkkj4A==", - "engines": { - "node": ">=14.16" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/Borewit" - } - }, "packages/ddex/processor/node_modules/prettier": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.1.1.tgz", @@ -116211,22 +116164,6 @@ "url": "https://github.com/sponsors/antfu" } }, - "packages/ddex/processor/node_modules/strtok3": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/strtok3/-/strtok3-7.0.0.tgz", - "integrity": "sha512-pQ+V+nYQdC5H3Q7qBZAz/MO6lwGhoC2gOAjuouGf/VO0m7vQRh8QNMl2Uf6SwAtzZ9bOw3UIeBukEGNJl5dtXQ==", - "dependencies": { - "@tokenizer/token": "^0.3.0", - "peek-readable": "^5.0.0" - }, - "engines": { - "node": ">=14.16" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/Borewit" - } - }, "packages/ddex/processor/node_modules/supports-color": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", @@ -116264,22 +116201,6 @@ "node": ">=14.0.0" } }, - "packages/ddex/processor/node_modules/token-types": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/token-types/-/token-types-5.0.1.tgz", - "integrity": "sha512-Y2fmSnZjQdDb9W4w4r1tswlMHylzWIeOKpx0aZH9BgGtACHhrk3OkT52AzwcuqTRBZtvvnTjDBh8eynMulu8Vg==", - "dependencies": { - "@tokenizer/token": "^0.3.0", - "ieee754": "^1.2.1" - }, - "engines": { - "node": ">=14.16" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/Borewit" - } - }, "packages/ddex/processor/node_modules/tr46": { "version": "0.0.3", "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", diff --git a/packages/ddex/processor/package.json b/packages/ddex/processor/package.json index 7ef552990a0..1c1ba3d08cd 100644 --- a/packages/ddex/processor/package.json +++ b/packages/ddex/processor/package.json @@ -27,7 +27,7 @@ "commander": "12.0.0", "decompress": "4.2.1", "dotenv": "16.3.1", - "file-type": "19.0.0", + "file-type": "16.5.3", "hono": "4.2.5", "web3": "4.3.0" }, diff --git a/packages/ddex/processor/src/server.ts b/packages/ddex/processor/src/server.ts index 40aa0f0eed7..2ae1e97ad7b 100644 --- a/packages/ddex/processor/src/server.ts +++ b/packages/ddex/processor/src/server.ts @@ -1,7 +1,7 @@ import 'dotenv/config' import { serve } from '@hono/node-server' -import { fileTypeFromBuffer } from 'file-type' +import { fromBuffer as fileTypeFromBuffer } from 'file-type' import { Context, Hono } from 'hono' import { deleteCookie, getSignedCookie, setSignedCookie } from 'hono/cookie' import { html } from 'hono/html'