From 5767e296181b1e857bf533aa236c364f330fe6d7 Mon Sep 17 00:00:00 2001 From: Dheeraj Manjunath Date: Mon, 27 Jun 2022 17:27:50 -0400 Subject: [PATCH 1/9] Try out this new health check pattern --- .../healthCheck/healthCheckController.js | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/creator-node/src/components/healthCheck/healthCheckController.js b/creator-node/src/components/healthCheck/healthCheckController.js index a33540b261f..5939e7560a3 100644 --- a/creator-node/src/components/healthCheck/healthCheckController.js +++ b/creator-node/src/components/healthCheck/healthCheckController.js @@ -26,6 +26,7 @@ const { } = require('../../fileManager') const config = require('../../config') +const { ensureStorageMiddleware } = require('middlewares') const router = express.Router() @@ -204,6 +205,15 @@ const healthCheckFileUploadController = async (req) => { if (err) { return errorResponseServerError(err) } + + await AsyncProcessingQueue.addTranscodeAndSegmentTask({ + logContext: req.logContext, + req: { + fileName: req.fileName, + fileDir: req.fileDir, + uuid: req.logContext.requestID + } + }) return successResponse({ success: true }) } @@ -228,8 +238,15 @@ router.get( router.post( '/health_check/fileupload', healthCheckVerifySignature, + ensureStorageMiddleware, handleTrackContentUpload, - handleResponseWithHeartbeat(healthCheckFileUploadController) + healthCheckFileUploadController +) +router.post( + '/health_check/fileupload/cleanup', + healthCheckVerifySignature, + ensureStorageMiddleware, + handleTrackContentUpload, + healthCheckFileUploadController ) - module.exports = router From 4c6b0e1e8f8bdf96968c606cdb8973034139be59 Mon Sep 17 00:00:00 2001 From: Dheeraj Manjunath Date: Mon, 27 Jun 2022 21:45:00 +0000 Subject: [PATCH 2/9] Import fix --- .../src/components/healthCheck/healthCheckController.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/creator-node/src/components/healthCheck/healthCheckController.js b/creator-node/src/components/healthCheck/healthCheckController.js index 5939e7560a3..dbe3a588ced 100644 --- a/creator-node/src/components/healthCheck/healthCheckController.js +++ b/creator-node/src/components/healthCheck/healthCheckController.js @@ -26,7 +26,7 @@ const { } = require('../../fileManager') const config = require('../../config') -const { ensureStorageMiddleware } = require('middlewares') +const { ensureStorageMiddleware } = require('../../middlewares') const router = express.Router() @@ -244,7 +244,7 @@ router.post( ) router.post( '/health_check/fileupload/cleanup', - healthCheckVerifySignature, + // healthCheckVerifySignature, ensureStorageMiddleware, handleTrackContentUpload, healthCheckFileUploadController From ca24b8438e13bd78f92473a66e4d10e2847a4f8e Mon Sep 17 00:00:00 2001 From: Dheeraj Manjunath Date: Wed, 29 Jun 2022 17:56:51 +0000 Subject: [PATCH 3/9] WIP --- .../healthCheck/healthCheckController.js | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/creator-node/src/components/healthCheck/healthCheckController.js b/creator-node/src/components/healthCheck/healthCheckController.js index dbe3a588ced..26f47185ef4 100644 --- a/creator-node/src/components/healthCheck/healthCheckController.js +++ b/creator-node/src/components/healthCheck/healthCheckController.js @@ -69,6 +69,7 @@ const healthCheckVerifySignature = (req, res, next) => { ) ) } + // todo - also allow other registered nodes to test this const delegateOwnerWallet = config.get('delegateOwnerWallet').toLowerCase() if (recoveredPublicWallet !== delegateOwnerWallet) { return sendResponse( @@ -200,12 +201,15 @@ const healthCheckVerboseController = async (req) => { const healthCheckFileUploadController = async (req) => { const err = req.fileFilterError || - req.fileSizeError || - (await removeTrackFolder(req, req.fileDir)) + req.fileSizeError if (err) { + await removeTrackFolder(req, req.fileDir) return errorResponseServerError(err) } + const AsyncProcessingQueue = + req.app.get('serviceRegistry').asyncProcessingQueue + req.logger.info("about to add addTranscodeAndSegmentTask") await AsyncProcessingQueue.addTranscodeAndSegmentTask({ logContext: req.logContext, req: { @@ -214,7 +218,8 @@ const healthCheckFileUploadController = async (req) => { uuid: req.logContext.requestID } }) - return successResponse({ success: true }) + req.logger.info("finished adding addTranscodeAndSegmentTask") + return successResponse({ uuid: req.logContext.requestID }) } // Routes @@ -223,12 +228,12 @@ router.get('/health_check', handleResponse(healthCheckController)) router.get('/health_check/sync', handleResponse(syncHealthCheckController)) router.get( '/health_check/duration', - healthCheckVerifySignature, + // healthCheckVerifySignature, handleResponse(healthCheckDurationController) ) router.get( '/health_check/duration/heartbeat', - healthCheckVerifySignature, + // healthCheckVerifySignature, handleResponseWithHeartbeat(healthCheckDurationController) ) router.get( @@ -237,16 +242,16 @@ router.get( ) router.post( '/health_check/fileupload', - healthCheckVerifySignature, + // healthCheckVerifySignature, ensureStorageMiddleware, handleTrackContentUpload, - healthCheckFileUploadController + handleResponse(healthCheckFileUploadController) ) router.post( '/health_check/fileupload/cleanup', // healthCheckVerifySignature, ensureStorageMiddleware, handleTrackContentUpload, - healthCheckFileUploadController + handleResponse(healthCheckFileUploadController) ) module.exports = router From bafc49ef5e32d5ec55f9476b3fce6c896a6823e4 Mon Sep 17 00:00:00 2001 From: Dheeraj Manjunath Date: Wed, 29 Jun 2022 22:13:18 +0000 Subject: [PATCH 4/9] Should be working content node side --- creator-node/package-lock.json | 4 +- .../healthCheck/healthCheckController.js | 47 +------------------ creator-node/src/routes/tracks.js | 13 ++++- 3 files changed, 16 insertions(+), 48 deletions(-) diff --git a/creator-node/package-lock.json b/creator-node/package-lock.json index 7fa276f3d4e..46e85aae98e 100644 --- a/creator-node/package-lock.json +++ b/creator-node/package-lock.json @@ -3939,7 +3939,7 @@ "bmp-js": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/bmp-js/-/bmp-js-0.1.0.tgz", - "integrity": "sha512-vHdS19CnY3hwiNdkaqk93DvjVLfbEcI8mys4UjuWrlX1haDmroo8o4xCzh4wD6DGV6HxRCyauwhHRqMTfERtjw==" + "integrity": "sha1-4Fpj95amwf8l9Hcex62twUjAcjM=" }, "bn.js": { "version": "4.12.0", @@ -4197,7 +4197,7 @@ "buffer-equal": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/buffer-equal/-/buffer-equal-0.0.1.tgz", - "integrity": "sha512-RgSV6InVQ9ODPdLWJ5UAqBqJBOg370Nz6ZQtRzpt6nUjc8v0St97uJ4PYC6NztqIScrAXafKM3mZPMygSe1ggA==" + "integrity": "sha1-kbx0sR6kBbyRa8aqkI+q+ltKrEs=" }, "buffer-layout": { "version": "1.2.2", diff --git a/creator-node/src/components/healthCheck/healthCheckController.js b/creator-node/src/components/healthCheck/healthCheckController.js index cd89001f411..ba5bad46640 100644 --- a/creator-node/src/components/healthCheck/healthCheckController.js +++ b/creator-node/src/components/healthCheck/healthCheckController.js @@ -247,65 +247,22 @@ const healthCheckVerboseController = async (req) => { }) } -/** - * Controller for `health_check/fileupload` route * - * Perform a file upload health check limited to configured delegateOwnerWallet. - * This prunes the disc artifacts created by the process after. - */ -const healthCheckFileUploadController = async (req) => { - const err = - req.fileFilterError || - req.fileSizeError - if (err) { - await removeTrackFolder(req, req.fileDir) - return errorResponseServerError(err) - } - - const AsyncProcessingQueue = - req.app.get('serviceRegistry').asyncProcessingQueue - req.logger.info("about to add addTranscodeAndSegmentTask") - await AsyncProcessingQueue.addTranscodeAndSegmentTask({ - logContext: req.logContext, - req: { - fileName: req.fileName, - fileDir: req.fileDir, - uuid: req.logContext.requestID - } - }) - req.logger.info("finished adding addTranscodeAndSegmentTask") - return successResponse({ uuid: req.logContext.requestID }) -} - // Routes router.get('/health_check', handleResponse(healthCheckController)) router.get('/health_check/sync', handleResponse(syncHealthCheckController)) router.get( '/health_check/duration', - // healthCheckVerifySignature, + healthCheckVerifySignature, handleResponse(healthCheckDurationController) ) router.get( '/health_check/duration/heartbeat', - // healthCheckVerifySignature, + healthCheckVerifySignature, handleResponseWithHeartbeat(healthCheckDurationController) ) router.get( '/health_check/verbose', handleResponse(healthCheckVerboseController) ) -router.post( - '/health_check/fileupload', - // healthCheckVerifySignature, - ensureStorageMiddleware, - handleTrackContentUpload, - handleResponse(healthCheckFileUploadController) -) -router.post( - '/health_check/fileupload/cleanup', - // healthCheckVerifySignature, - ensureStorageMiddleware, - handleTrackContentUpload, - handleResponse(healthCheckFileUploadController) -) module.exports = router diff --git a/creator-node/src/routes/tracks.js b/creator-node/src/routes/tracks.js index 128bd0ea763..aa55260117f 100644 --- a/creator-node/src/routes/tracks.js +++ b/creator-node/src/routes/tracks.js @@ -127,7 +127,7 @@ module.exports = function (app) { * Given that the request is coming from a valid SP, serve the corresponding file * from the transcode handoff * - * This route is used on an available SP when the primary requests the transcoded files after + * This route is called from the primary to request the transcoded files after * sending the first request for the transcode handoff. This route does not run on the primary. */ app.get( @@ -172,6 +172,17 @@ module.exports = function (app) { } ) + app.post( + 'transcode_and_segment/clear_filesystem', + ensureValidSPMiddleware, + handleResponse(async (req, res) => { + const fileDir = req.body.fileDir + removeTrackFolder({ logContext: req.logContext }, fileDir) + + return successResponse() + }) + ) + /** * Given track metadata object, save metadata to disk. Return metadata multihash if successful. * If metadata is for a downloadable track, ensures transcoded master record exists in DB From a2d48f9f177ec18115691dbc8641e818899def5b Mon Sep 17 00:00:00 2001 From: Dheeraj Manjunath Date: Wed, 29 Jun 2022 22:31:02 +0000 Subject: [PATCH 5/9] lint --- .../src/components/healthCheck/healthCheckController.js | 9 +++------ creator-node/src/routes/tracks.js | 2 +- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/creator-node/src/components/healthCheck/healthCheckController.js b/creator-node/src/components/healthCheck/healthCheckController.js index ba5bad46640..e05b68247d8 100644 --- a/creator-node/src/components/healthCheck/healthCheckController.js +++ b/creator-node/src/components/healthCheck/healthCheckController.js @@ -20,13 +20,8 @@ const { getMonitors } = require('../../monitors/monitors') const TranscodingQueue = require('../../TranscodingQueue') const { recoverWallet } = require('../../apiSigning') -const { - handleTrackContentUpload, - removeTrackFolder -} = require('../../fileManager') const config = require('../../config') -const { ensureStorageMiddleware } = require('../../middlewares') const router = express.Router() @@ -44,6 +39,7 @@ const FIND_REPLICA_SET_UPDATES_JOB_MAX_LAST_SUCCESSFUL_RUN_DELAY_MS = config.get('findReplicaSetUpdatesJobLastSuccessfulRunDelayMs') // Helper Functions + /** * Verifies that the request is made by the delegate Owner */ @@ -65,6 +61,7 @@ const healthCheckVerifySignature = (req, res, next) => { const recoveredTimestampDate = new Date(timestamp) const currentTimestampDate = new Date() const requestAge = currentTimestampDate - recoveredTimestampDate + if (requestAge >= MAX_HEALTH_CHECK_TIMESTAMP_AGE_MS) { return sendResponse( req, @@ -74,7 +71,7 @@ const healthCheckVerifySignature = (req, res, next) => { ) ) } - // todo - also allow other registered nodes to test this + const delegateOwnerWallet = config.get('delegateOwnerWallet').toLowerCase() if (recoveredPublicWallet !== delegateOwnerWallet) { return sendResponse( diff --git a/creator-node/src/routes/tracks.js b/creator-node/src/routes/tracks.js index aa55260117f..074ef7db709 100644 --- a/creator-node/src/routes/tracks.js +++ b/creator-node/src/routes/tracks.js @@ -177,7 +177,7 @@ module.exports = function (app) { ensureValidSPMiddleware, handleResponse(async (req, res) => { const fileDir = req.body.fileDir - removeTrackFolder({ logContext: req.logContext }, fileDir) + await removeTrackFolder({ logContext: req.logContext }, fileDir) return successResponse() }) From 3d66e349e9d6458c470d250ab756ca96d08bb651 Mon Sep 17 00:00:00 2001 From: Dheeraj Manjunath Date: Wed, 29 Jun 2022 23:38:15 +0000 Subject: [PATCH 6/9] Gate to tmp_track_artifacts --- creator-node/src/routes/tracks.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/creator-node/src/routes/tracks.js b/creator-node/src/routes/tracks.js index 074ef7db709..c49e5a46e37 100644 --- a/creator-node/src/routes/tracks.js +++ b/creator-node/src/routes/tracks.js @@ -177,6 +177,9 @@ module.exports = function (app) { ensureValidSPMiddleware, handleResponse(async (req, res) => { const fileDir = req.body.fileDir + if (!fileDir.includes('tmp_track_artifacts')){ + return errorResponseBadRequest('Cannot remove track folder outside temporary track artifacts') + } await removeTrackFolder({ logContext: req.logContext }, fileDir) return successResponse() From 22f16811d45c76efde3d9499e10e119ac596333c Mon Sep 17 00:00:00 2001 From: Dheeraj Manjunath Date: Wed, 29 Jun 2022 23:41:35 +0000 Subject: [PATCH 7/9] lint --- creator-node/src/routes/tracks.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/creator-node/src/routes/tracks.js b/creator-node/src/routes/tracks.js index c49e5a46e37..f89e609cb1a 100644 --- a/creator-node/src/routes/tracks.js +++ b/creator-node/src/routes/tracks.js @@ -177,8 +177,10 @@ module.exports = function (app) { ensureValidSPMiddleware, handleResponse(async (req, res) => { const fileDir = req.body.fileDir - if (!fileDir.includes('tmp_track_artifacts')){ - return errorResponseBadRequest('Cannot remove track folder outside temporary track artifacts') + if (!fileDir.includes('tmp_track_artifacts')) { + return errorResponseBadRequest( + 'Cannot remove track folder outside temporary track artifacts' + ) } await removeTrackFolder({ logContext: req.logContext }, fileDir) From 3df27169a0c51cf5700515e7283979ceeb8c47f7 Mon Sep 17 00:00:00 2001 From: Dheeraj Manjunath Date: Fri, 1 Jul 2022 17:47:55 +0000 Subject: [PATCH 8/9] Final middleware cleanup --- .../healthCheck/healthCheckController.js | 54 ++----------------- creator-node/src/routes/tracks.js | 37 +++++++------ 2 files changed, 24 insertions(+), 67 deletions(-) diff --git a/creator-node/src/components/healthCheck/healthCheckController.js b/creator-node/src/components/healthCheck/healthCheckController.js index e05b68247d8..214027019ca 100644 --- a/creator-node/src/components/healthCheck/healthCheckController.js +++ b/creator-node/src/components/healthCheck/healthCheckController.js @@ -19,7 +19,7 @@ const { sequelize } = require('../../models') const { getMonitors } = require('../../monitors/monitors') const TranscodingQueue = require('../../TranscodingQueue') -const { recoverWallet } = require('../../apiSigning') +const { ensureValidSPMiddleware } = require('../../middlewares') const config = require('../../config') @@ -38,55 +38,7 @@ const FIND_SYNC_REQUESTS_JOB_MAX_LAST_SUCCESSFUL_RUN_DELAY_MS = config.get( const FIND_REPLICA_SET_UPDATES_JOB_MAX_LAST_SUCCESSFUL_RUN_DELAY_MS = config.get('findReplicaSetUpdatesJobLastSuccessfulRunDelayMs') -// Helper Functions -/** - * Verifies that the request is made by the delegate Owner - */ -const healthCheckVerifySignature = (req, res, next) => { - const { timestamp, randomBytes, signature } = req.query - if (!timestamp || !randomBytes || !signature) { - return sendResponse( - req, - res, - errorResponseBadRequest('Missing required query parameters') - ) - } - - const recoveryObject = { randomBytesToSign: randomBytes, timestamp } - const recoveredPublicWallet = recoverWallet( - recoveryObject, - signature - ).toLowerCase() - const recoveredTimestampDate = new Date(timestamp) - const currentTimestampDate = new Date() - const requestAge = currentTimestampDate - recoveredTimestampDate - - if (requestAge >= MAX_HEALTH_CHECK_TIMESTAMP_AGE_MS) { - return sendResponse( - req, - res, - errorResponseBadRequest( - `Submitted timestamp=${recoveredTimestampDate}, current timestamp=${currentTimestampDate}. Maximum age =${MAX_HEALTH_CHECK_TIMESTAMP_AGE_MS}` - ) - ) - } - - const delegateOwnerWallet = config.get('delegateOwnerWallet').toLowerCase() - if (recoveredPublicWallet !== delegateOwnerWallet) { - return sendResponse( - req, - res, - errorResponseBadRequest( - "Requester's public key does does not match Creator Node's delegate owner wallet." - ) - ) - } - - next() -} - -// Controllers /** * Controller for `health_check` route, calls @@ -250,12 +202,12 @@ router.get('/health_check', handleResponse(healthCheckController)) router.get('/health_check/sync', handleResponse(syncHealthCheckController)) router.get( '/health_check/duration', - healthCheckVerifySignature, + ensureValidSPMiddleware, handleResponse(healthCheckDurationController) ) router.get( '/health_check/duration/heartbeat', - healthCheckVerifySignature, + ensureValidSPMiddleware, handleResponseWithHeartbeat(healthCheckDurationController) ) router.get( diff --git a/creator-node/src/routes/tracks.js b/creator-node/src/routes/tracks.js index f89e609cb1a..52e82127b93 100644 --- a/creator-node/src/routes/tracks.js +++ b/creator-node/src/routes/tracks.js @@ -94,6 +94,27 @@ module.exports = function (app) { }) ) + /** + * Delete all temporary transcode artifacts from track transcode handoff flow. + * This is called on the node that was handed off the transcode to clear the state from disk + */ + app.post( + '/clear_transcode_and_segment_artifacts', + ensureValidSPMiddleware, + handleResponse(async (req, res) => { + const fileDir = req.body.fileDir + req.logger.info("Clearing filesystem fileDir", fileDir) + if (!fileDir.includes('tmp_track_artifacts')) { + return errorResponseBadRequest( + 'Cannot remove track folder outside temporary track artifacts' + ) + } + await removeTrackFolder({ logContext: req.logContext }, fileDir) + + return successResponse() + }) + ) + /** * Given that the requester is a valid SP, the current Content Node has enough storage, * upload the track to the current node and add a transcode and segmenting job to the queue. @@ -172,22 +193,6 @@ module.exports = function (app) { } ) - app.post( - 'transcode_and_segment/clear_filesystem', - ensureValidSPMiddleware, - handleResponse(async (req, res) => { - const fileDir = req.body.fileDir - if (!fileDir.includes('tmp_track_artifacts')) { - return errorResponseBadRequest( - 'Cannot remove track folder outside temporary track artifacts' - ) - } - await removeTrackFolder({ logContext: req.logContext }, fileDir) - - return successResponse() - }) - ) - /** * Given track metadata object, save metadata to disk. Return metadata multihash if successful. * If metadata is for a downloadable track, ensures transcoded master record exists in DB From 61670d71eb23b48a8a8c1a0462f180494f072df0 Mon Sep 17 00:00:00 2001 From: Dheeraj Manjunath Date: Fri, 1 Jul 2022 20:09:51 +0000 Subject: [PATCH 9/9] Lint --- .../components/healthCheck/healthCheckController.js | 2 -- creator-node/src/routes/tracks.js | 10 +++++----- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/creator-node/src/components/healthCheck/healthCheckController.js b/creator-node/src/components/healthCheck/healthCheckController.js index 214027019ca..124b9492856 100644 --- a/creator-node/src/components/healthCheck/healthCheckController.js +++ b/creator-node/src/components/healthCheck/healthCheckController.js @@ -38,8 +38,6 @@ const FIND_SYNC_REQUESTS_JOB_MAX_LAST_SUCCESSFUL_RUN_DELAY_MS = config.get( const FIND_REPLICA_SET_UPDATES_JOB_MAX_LAST_SUCCESSFUL_RUN_DELAY_MS = config.get('findReplicaSetUpdatesJobLastSuccessfulRunDelayMs') - - /** * Controller for `health_check` route, calls * `healthCheckComponentService`. diff --git a/creator-node/src/routes/tracks.js b/creator-node/src/routes/tracks.js index 52e82127b93..3fda9b372cd 100644 --- a/creator-node/src/routes/tracks.js +++ b/creator-node/src/routes/tracks.js @@ -94,16 +94,16 @@ module.exports = function (app) { }) ) - /** - * Delete all temporary transcode artifacts from track transcode handoff flow. - * This is called on the node that was handed off the transcode to clear the state from disk - */ + /** + * Delete all temporary transcode artifacts from track transcode handoff flow. + * This is called on the node that was handed off the transcode to clear the state from disk + */ app.post( '/clear_transcode_and_segment_artifacts', ensureValidSPMiddleware, handleResponse(async (req, res) => { const fileDir = req.body.fileDir - req.logger.info("Clearing filesystem fileDir", fileDir) + req.logger.info('Clearing filesystem fileDir', fileDir) if (!fileDir.includes('tmp_track_artifacts')) { return errorResponseBadRequest( 'Cannot remove track folder outside temporary track artifacts'