From c28efde9ef30899610fe52865a7390c0adba708e Mon Sep 17 00:00:00 2001 From: Raymond Jacobson Date: Mon, 9 May 2022 21:47:40 -0700 Subject: [PATCH 01/10] Separate prod / stage environments --- audius-cli | 37 ++++++++++++++++------- creator-node/docker-compose.yml | 2 +- creator-node/{env => prod.env} | 0 creator-node/stage.env | 28 ++++++++++++++++++ discovery-provider/docker-compose.yml | 4 +-- discovery-provider/{env => prod.env} | 0 discovery-provider/stage.env | 42 +++++++++++++++++++++++++++ 7 files changed, 100 insertions(+), 13 deletions(-) rename creator-node/{env => prod.env} (100%) create mode 100644 creator-node/stage.env rename discovery-provider/{env => prod.env} (100%) create mode 100644 discovery-provider/stage.env diff --git a/audius-cli b/audius-cli index bc622af7c..5c7b628f4 100755 --- a/audius-cli +++ b/audius-cli @@ -23,9 +23,21 @@ SERVICE_PORTS = { } service_type = click.Choice(["creator-node", "discovery-provider"]) +environment = click.Choice(["prod", "stage"]) container_type = click.Choice(["backend", "cache", "db"]) +def update_base_env(ctx, environment): + """ + Sets the base env in .env for each service and + returns the name of the base env file to use. + """ + for service in ["creator-node", "discovery-provider"]: + env_file = ctx.obj["manifests_path"] / service / ".env" + dotenv.set_key(env_file, "ENV", environment) + return f"{environment}.env" + + @click.group() @click.pass_context def cli(ctx): @@ -38,10 +50,11 @@ def cli(ctx): @cli.command() @click.argument("service", type=service_type) +@click.option("--environment", type=environment, default="prod") @click.pass_context -def check_config(ctx, service): +def check_config(ctx, service, environment): """Check the config for a service""" - env = ctx.obj["manifests_path"] / service / "env" + env = ctx.obj["manifests_path"] / service / update_base_env(ctx, environment) override_env = ctx.obj["manifests_path"] / service / "override.env" env_data = dotenv.dotenv_values(env) @@ -122,11 +135,14 @@ def health_check(ctx, service): @cli.command() -@click.option("--seed-job", is_flag=True) @click.argument("service", type=service_type) +@click.option("--seed-job", is_flag=True) +@click.option("--environment", type=environment, default="prod") @click.pass_context -def launch(ctx, seed_job, service): +def launch(ctx, service, seed_job, environment): """Launch the service""" + update_base_env(ctx, environment) + try: ctx.invoke(check_config, service=service) except SystemExit: @@ -246,19 +262,20 @@ def restart(ctx, service, containers): @cli.command() -@click.option("--unset", is_flag=True) -@click.option("--required", is_flag=True) @click.argument("service", type=service_type) @click.argument("key", required=False) @click.argument("value", required=False) +@click.option("--unset", is_flag=True) +@click.option("--required", is_flag=True) +@click.option("--environment", type=environment, default="prod") @click.pass_context -def set_config(ctx, service, unset, required, key, value): +def set_config(ctx, service, unset, required, key, value, environment): """Set a config value""" if required and (key or value): click.secho("--required cannot be used when key or value is set", fg="red") sys.exit(1) - env = ctx.obj["manifests_path"] / service / "env" + env = ctx.obj["manifests_path"] / service / update_base_env(ctx, environment) env_data = dotenv.dotenv_values(env) override_env = ctx.obj["manifests_path"] / service / "override.env" @@ -290,8 +307,8 @@ def set_config(ctx, service, unset, required, key, value): @cli.command() -@click.option("--unset", is_flag=True) @click.argument("tag", required=False) +@click.option("--unset", is_flag=True) @click.pass_context def set_tag(ctx, unset, tag): """Set a config value""" @@ -334,8 +351,8 @@ def upgrade(ctx, branch): @cli.command() -@click.option("--remove", is_flag=True) @click.argument("cron-expression", default="0 */6 * * *") +@click.option("--remove", is_flag=True) @click.pass_context def auto_upgrade(ctx, remove, cron_expression): """Setup auto upgrade with a cron job""" diff --git a/creator-node/docker-compose.yml b/creator-node/docker-compose.yml index ea69f55e4..2bba2a98e 100644 --- a/creator-node/docker-compose.yml +++ b/creator-node/docker-compose.yml @@ -29,7 +29,7 @@ services: ports: - "4000:4000" env_file: - - env + - ${ENV}.env - override.env networks: - creator-node-network diff --git a/creator-node/env b/creator-node/prod.env similarity index 100% rename from creator-node/env rename to creator-node/prod.env diff --git a/creator-node/stage.env b/creator-node/stage.env new file mode 100644 index 000000000..cb880d358 --- /dev/null +++ b/creator-node/stage.env @@ -0,0 +1,28 @@ +dbUrl=postgres://postgres:postgres@db:5432/postgres +dbConnectionPoolMax=500 + +redisHost=cache +redisPort=6379 + +dataNetworkId=77 +dataProviderUrl=https://poa-gateway.staging.audius.co +dataRegistryAddress=0x793373aBF96583d5eb71a15d86fFE732CD04D452 +ethNetworkId=3 +ethOwnerWallet=0xcccc7428648c4AdC0ae262D3547584dDAE25c465 +ethProviderUrl=https://eth-ropsten.alchemyapi.io/v2/EEhfbhEFErMbKOp_XQJyvvWKouQjvSss +ethRegistryAddress=0xe39b1cA04fc06c416c4eaBd188Cb1330b8FED781 +ethTokenAddress=0x74f24429ec3708fc21381e017194A5711E93B751 +headersTimeout=60000 +identityService=https://identityservice.staging.audius.co +keepAliveTimeout=5000 +logLevel=info +maxAudioFileSizeBytes=1000000000 +pinAddCIDs=Qma5FGErNqHA32cyEjgp8hnwCFoqyYcBXU1ySKPA6MUJS5,QmYPy9A3zxLeBBuk1vpPNeG5aoMfa5qLNjJgh473Z9zV9b +setTimeout=3600000 +timeout=3600000 + +creatorNodeEndpoint= +delegateOwnerWallet= +delegatePrivateKey= +discoveryProviderWhitelist= +spOwnerWallet= diff --git a/discovery-provider/docker-compose.yml b/discovery-provider/docker-compose.yml index ea7b62706..ee5ca8ac3 100644 --- a/discovery-provider/docker-compose.yml +++ b/discovery-provider/docker-compose.yml @@ -57,7 +57,7 @@ services: ports: - "5000:5000" env_file: - - env + - ${ENV}.env - override.env networks: - discovery-provider-network @@ -66,7 +66,7 @@ services: image: audius/discovery-provider:${TAG:-257b60f2ff0b990138d08e618b4e364e3b27b6d7} command: bash -c " curl https://audius-pgdump.s3-us-west-2.amazonaws.com/discProvProduction.dump -O && pg_restore -d $$audius_db_url discProvProduction.dump " env_file: - - env + - ${ENV}.env - override.env depends_on: discovery-provider-db: diff --git a/discovery-provider/env b/discovery-provider/prod.env similarity index 100% rename from discovery-provider/env rename to discovery-provider/prod.env diff --git a/discovery-provider/stage.env b/discovery-provider/stage.env new file mode 100644 index 000000000..263555d45 --- /dev/null +++ b/discovery-provider/stage.env @@ -0,0 +1,42 @@ +audius_redis_url=redis://cache:6379/00 + +audius_db_url=postgresql://postgres:postgres@db:5432/postgres +audius_db_url_read_replica=postgresql://postgres:postgres@db:5432/postgres + +audius_contracts_registry=0x793373aBF96583d5eb71a15d86fFE732CD04D452 +audius_cors_allow_all=true +audius_discprov_blacklist_block_processing_window=1200 +audius_discprov_block_processing_window=100 +audius_discprov_env=stage +audius_discprov_identity_service_url=https://identityservice.staging.audius.co +audius_discprov_notifications_max_block_diff=10 +audius_discprov_start_block=0x65a3243860511ed28a933c3a113dea7df368ad53f721cc9d0034c0c75f996afb +audius_discprov_user_metadata_service_url=https://usermetadata.staging.audius.co +audius_eth_contracts_registry=0xe39b1cA04fc06c416c4eaBd188Cb1330b8FED781 +audius_gunicorn_worker_class=eventlet +audius_gunicorn_workers=16 +audius_solana_endpoint=https://audius.rpcpool.com/0b7f8d4970676bd78dbe6fb97e8f +audius_solana_rewards_manager_account=71hWFVYokLaN1PNYzTAWi13EfJ7Xt9VbSWUKsXUT8mxE +audius_solana_rewards_manager_min_slot=0 +audius_solana_rewards_manager_program_address=CDpzvz7DfgbF95jSSCHLX3ERkugyfgn9Fw8ypNZ1hfXp +audius_solana_signer_group_address=DP5ymiCCQurW9jTeDwHAUs3jmfmnqqypGgNF573wje5e +audius_solana_track_listen_count_address=ApR7QbouRviwoE6nLL83omE6GdtM6yUJAsuXw5sPDCQV +audius_solana_user_bank_min_slot=0 +audius_solana_user_bank_program_address=2sjQNmUfkV6yKKi4dPR8gWRgtyma5aiymE3aXL2RAZww +audius_solana_waudio_mint=BELGiMZQ34SDE6x2FUaML2UHDAgBLS64xvhXjX5tBBZo +audius_web3_eth_provider_url=https://eth-ropsten.alchemyapi.io/v2/EEhfbhEFErMbKOp_XQJyvvWKouQjvSss +audius_web3_host=poa-gateway.staging.audius.co +audius_web3_port=443 + +audius_delegate_owner_wallet= +audius_delegate_private_key= + + + + + + + + + + From 3b7f7529fb86b57fc0d601ff8ef577555931e49b Mon Sep 17 00:00:00 2001 From: Raymond Jacobson Date: Mon, 9 May 2022 22:07:21 -0700 Subject: [PATCH 02/10] Update README --- .gitignore | 4 ++++ README.md | 18 +++++++++++------- audius-cli | 8 ++++---- discovery-provider/docker-compose.yml | 4 ++-- 4 files changed, 21 insertions(+), 13 deletions(-) diff --git a/.gitignore b/.gitignore index fc113e4dd..f84d4b89e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,6 @@ override.env auto-upgrade.log + +# .env files for system (tag, env management) +discovery-provider/.env +creator-node/.env diff --git a/README.md b/README.md index b3ae9abbe..cccd17155 100644 --- a/README.md +++ b/README.md @@ -4,13 +4,14 @@ Launch audius services via docker compose Actively under development, not recommended for production use. -### Migration from Kubernetes +## Migration from Kubernetes For existing machines running audius services via kube, first run the following commands, ```sh audius-cli auto-upgrade --remove kubectl delete --all-namespaces --all deployments kubectl delete --all-namespaces --all pods +sudo kubeadm reset git clone https://github.com/AudiusProject/audius-docker-compose.git ~/audius-docker-compose cd ~/audius-docker-compose @@ -23,15 +24,13 @@ cat audius-k8s-manifests/config.yaml ``` and set them similarly to before with audius-cli -### Single Click Install +## Single Click Install ```sh bash <(curl https://raw.githubusercontent.com/AudiusProject/audius-docker-compose/main/install.sh) ``` -### Launching an Audius Service - -Currently, this has been tested on Ubuntu 20.04 LTS machines. +## Launching an Audius Service ```sh bash setup.sh @@ -43,6 +42,7 @@ For every key do ```sh audius-cli set-config ``` + #### Creator Node The full list of variables and explanations can be found on the wiki [here](https://github.com/AudiusProject/audius-protocol/wiki/Content-Node:-Configuration-Details#required-environment-variables). @@ -108,6 +108,10 @@ Make sure that your service exposes all the required environment variables. See ### Launch ```sh audius-cli launch -``` -Launch with `--seed-job` for first time discovery provider setup +# Options: +# --seed +# Seeds the database from a snapshot. Required for first-time discovery setup. +# --environment [stage, prod] +# Uses environment variables for the relevant environment. +``` diff --git a/audius-cli b/audius-cli index 5c7b628f4..aace3cd07 100755 --- a/audius-cli +++ b/audius-cli @@ -136,10 +136,10 @@ def health_check(ctx, service): @cli.command() @click.argument("service", type=service_type) -@click.option("--seed-job", is_flag=True) +@click.option("--seed", is_flag=True) @click.option("--environment", type=environment, default="prod") @click.pass_context -def launch(ctx, service, seed_job, environment): +def launch(ctx, service, seed, environment): """Launch the service""" update_base_env(ctx, environment) @@ -161,7 +161,7 @@ def launch(ctx, service, seed_job, environment): click.confirm(click.style("Do you want to continue?", bold=True), abort=True) - if seed_job and service == "discovery-provider": + if seed and service == "discovery-provider": # make sure backend is not running subprocess.run( [ @@ -182,7 +182,7 @@ def launch(ctx, service, seed_job, environment): "--project-directory", ctx.obj["manifests_path"] / "discovery-provider", "run", - "seed-job", + "seed", ], ) diff --git a/discovery-provider/docker-compose.yml b/discovery-provider/docker-compose.yml index ee5ca8ac3..cc930ea06 100644 --- a/discovery-provider/docker-compose.yml +++ b/discovery-provider/docker-compose.yml @@ -62,7 +62,7 @@ services: networks: - discovery-provider-network - seed-job: + seed: image: audius/discovery-provider:${TAG:-257b60f2ff0b990138d08e618b4e364e3b27b6d7} command: bash -c " curl https://audius-pgdump.s3-us-west-2.amazonaws.com/discProvProduction.dump -O && pg_restore -d $$audius_db_url discProvProduction.dump " env_file: @@ -74,7 +74,7 @@ services: networks: - discovery-provider-network profiles: - - seed-job + - seed networks: discovery-provider-network: From 4dfcebf0c78c865519aa94f24bb793632fa8c83f Mon Sep 17 00:00:00 2001 From: Raymond Jacobson Date: Mon, 9 May 2022 22:56:29 -0700 Subject: [PATCH 03/10] Give nice container names --- creator-node/docker-compose.yml | 3 +++ discovery-provider/docker-compose.yml | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/creator-node/docker-compose.yml b/creator-node/docker-compose.yml index 2bba2a98e..4b7bf5ed6 100644 --- a/creator-node/docker-compose.yml +++ b/creator-node/docker-compose.yml @@ -2,6 +2,7 @@ version: "3.9" services: db: + container_name: postgres extends: file: ../common-services.yml service: base-db @@ -11,6 +12,7 @@ services: - creator-node-network cache: + container_name: redis extends: file: ../common-services.yml service: base-cache @@ -18,6 +20,7 @@ services: - creator-node-network backend: + container_name: server image: audius/creator-node:${TAG:-257b60f2ff0b990138d08e618b4e364e3b27b6d7} depends_on: db: diff --git a/discovery-provider/docker-compose.yml b/discovery-provider/docker-compose.yml index cc930ea06..f6a322d43 100644 --- a/discovery-provider/docker-compose.yml +++ b/discovery-provider/docker-compose.yml @@ -2,6 +2,7 @@ version: "3.9" services: db: + container_name: postgres extends: file: ../common-services.yml service: base-db @@ -11,6 +12,7 @@ services: - discovery-provider-network cache: + container_name: redis extends: file: ../common-services.yml service: base-cache @@ -18,8 +20,8 @@ services: - discovery-provider-network elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.1.0 container_name: elasticsearch + image: docker.elastic.co/elasticsearch/elasticsearch:8.1.0 environment: - network.host=0.0.0.0 - discovery.type=single-node @@ -46,6 +48,7 @@ services: timeout: 5s backend: + container_name: server image: audius/discovery-provider:${TAG:-257b60f2ff0b990138d08e618b4e364e3b27b6d7} depends_on: db: From 6acaa8501541d4f92635013ac9be249732564f74 Mon Sep 17 00:00:00 2001 From: Raymond Jacobson Date: Mon, 9 May 2022 22:59:08 -0700 Subject: [PATCH 04/10] Fix bug in seed job --- discovery-provider/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/discovery-provider/docker-compose.yml b/discovery-provider/docker-compose.yml index f6a322d43..9e2925408 100644 --- a/discovery-provider/docker-compose.yml +++ b/discovery-provider/docker-compose.yml @@ -72,7 +72,7 @@ services: - ${ENV}.env - override.env depends_on: - discovery-provider-db: + db: condition: service_healthy networks: - discovery-provider-network From 72beffc2af8385725489a20e99880e1e57a65bc5 Mon Sep 17 00:00:00 2001 From: Raymond Jacobson Date: Mon, 9 May 2022 23:18:46 -0700 Subject: [PATCH 05/10] Fix seed --- discovery-provider/docker-compose.yml | 4 +++- discovery-provider/seed.sh | 14 ++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 discovery-provider/seed.sh diff --git a/discovery-provider/docker-compose.yml b/discovery-provider/docker-compose.yml index 9e2925408..baf87e611 100644 --- a/discovery-provider/docker-compose.yml +++ b/discovery-provider/docker-compose.yml @@ -67,10 +67,12 @@ services: seed: image: audius/discovery-provider:${TAG:-257b60f2ff0b990138d08e618b4e364e3b27b6d7} - command: bash -c " curl https://audius-pgdump.s3-us-west-2.amazonaws.com/discProvProduction.dump -O && pg_restore -d $$audius_db_url discProvProduction.dump " + command: bash -c /seed.sh env_file: - ${ENV}.env - override.env + volumes: + - ./seed.sh:/seed.sh depends_on: db: condition: service_healthy diff --git a/discovery-provider/seed.sh b/discovery-provider/seed.sh new file mode 100644 index 000000000..deb12a1b8 --- /dev/null +++ b/discovery-provider/seed.sh @@ -0,0 +1,14 @@ +#!/usr/bin/bash + +if [[ $ENV == 'prod' ]]; then + curl https://audius-pgdump.s3-us-west-2.amazonaws.com/discProvProduction.dump -O + pg_restore -d $audius_db_url discProvProduction.dump + +elif [[ $ENV == 'stage']]; then + curl https://audius-pgdump.s3-us-west-2.amazonaws.com/discProvStaging.dump -O + pg_restore -d $audius_db_url discProvStaging.dump + +else + echo "Invalid env: $ENV" + exit 1 +fi \ No newline at end of file From 15368f65d8750a21d376dd189bb1c2719ad11c81 Mon Sep 17 00:00:00 2001 From: Raymond Jacobson Date: Mon, 9 May 2022 23:20:19 -0700 Subject: [PATCH 06/10] Fix invocation location --- audius-cli | 10 +++++----- discovery-provider/docker-compose.yml | 4 ++-- discovery-provider/seed.sh | 16 +++++++++++----- 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/audius-cli b/audius-cli index aace3cd07..63e5ccb99 100755 --- a/audius-cli +++ b/audius-cli @@ -144,20 +144,20 @@ def launch(ctx, service, seed, environment): update_base_env(ctx, environment) try: - ctx.invoke(check_config, service=service) + ctx.invoke(check_config, service=service, environment=environment) except SystemExit: pass total_memory = psutil.virtual_memory().total // (1024 ** 3) cpu_count = psutil.cpu_count() - click.echo(f"CPUs:\t{cpu_count}\t(recommended: {RECOMMENDED_CPU_COUNT})") - click.echo(f"Memory:\t{total_memory}GB\t(recommended: {RECOMMENDED_MEMORY}GB)") + click.echo(f"CPUs:\t{cpu_count}\t(required: {RECOMMENDED_CPU_COUNT})") + click.echo(f"Memory:\t{total_memory}GB\t(required: {RECOMMENDED_MEMORY}GB)") if cpu_count < RECOMMENDED_CPU_COUNT or total_memory < RECOMMENDED_MEMORY: - click.secho("System does not meet recommended requirements", fg="red") + click.secho("System does not meet requirements", fg="red") else: - click.secho("System meets recommended requirements", fg="green") + click.secho("System meets requirements", fg="green") click.confirm(click.style("Do you want to continue?", bold=True), abort=True) diff --git a/discovery-provider/docker-compose.yml b/discovery-provider/docker-compose.yml index baf87e611..6697d4dee 100644 --- a/discovery-provider/docker-compose.yml +++ b/discovery-provider/docker-compose.yml @@ -67,12 +67,12 @@ services: seed: image: audius/discovery-provider:${TAG:-257b60f2ff0b990138d08e618b4e364e3b27b6d7} - command: bash -c /seed.sh + command: bash /usr/share/seed.sh ${ENV} env_file: - ${ENV}.env - override.env volumes: - - ./seed.sh:/seed.sh + - ./seed.sh:/usr/share/seed.sh depends_on: db: condition: service_healthy diff --git a/discovery-provider/seed.sh b/discovery-provider/seed.sh index deb12a1b8..c860d2674 100644 --- a/discovery-provider/seed.sh +++ b/discovery-provider/seed.sh @@ -1,14 +1,20 @@ #!/usr/bin/bash -if [[ $ENV == 'prod' ]]; then +ENV=$1 + +if [ "$ENV" == 'prod' ] +then + echo "Downloading $ENV database..." curl https://audius-pgdump.s3-us-west-2.amazonaws.com/discProvProduction.dump -O + echo "Restoring $ENV database to $audius_db_url..." pg_restore -d $audius_db_url discProvProduction.dump - -elif [[ $ENV == 'stage']]; then +elif [ "$ENV" == 'stage' ] +then + echo "Downloading $ENV database..." curl https://audius-pgdump.s3-us-west-2.amazonaws.com/discProvStaging.dump -O + echo "Restoring $ENV database to $audius_db_url..." pg_restore -d $audius_db_url discProvStaging.dump - else echo "Invalid env: $ENV" exit 1 -fi \ No newline at end of file +fi From 261baa2011c2172594f544f4b15dc161b526acb6 Mon Sep 17 00:00:00 2001 From: Raymond Jacobson Date: Mon, 9 May 2022 23:51:11 -0700 Subject: [PATCH 07/10] Wipe db --- discovery-provider/seed.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/discovery-provider/seed.sh b/discovery-provider/seed.sh index c860d2674..6c7268e27 100644 --- a/discovery-provider/seed.sh +++ b/discovery-provider/seed.sh @@ -7,13 +7,13 @@ then echo "Downloading $ENV database..." curl https://audius-pgdump.s3-us-west-2.amazonaws.com/discProvProduction.dump -O echo "Restoring $ENV database to $audius_db_url..." - pg_restore -d $audius_db_url discProvProduction.dump + pg_restore -d $audius_db_url --clean --if-exists discProvProduction.dump elif [ "$ENV" == 'stage' ] then echo "Downloading $ENV database..." curl https://audius-pgdump.s3-us-west-2.amazonaws.com/discProvStaging.dump -O echo "Restoring $ENV database to $audius_db_url..." - pg_restore -d $audius_db_url discProvStaging.dump + pg_restore -d $audius_db_url --clean --if-exists discProvStaging.dump else echo "Invalid env: $ENV" exit 1 From 83711ded87d71dd69857b2c97f37ce114d674396 Mon Sep 17 00:00:00 2001 From: Raymond Jacobson Date: Tue, 10 May 2022 00:09:07 -0700 Subject: [PATCH 08/10] Use verbose restore --- discovery-provider/seed.sh | 4 ++-- discovery-provider/stage.env | 10 ---------- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/discovery-provider/seed.sh b/discovery-provider/seed.sh index 6c7268e27..6d6bf5026 100644 --- a/discovery-provider/seed.sh +++ b/discovery-provider/seed.sh @@ -7,13 +7,13 @@ then echo "Downloading $ENV database..." curl https://audius-pgdump.s3-us-west-2.amazonaws.com/discProvProduction.dump -O echo "Restoring $ENV database to $audius_db_url..." - pg_restore -d $audius_db_url --clean --if-exists discProvProduction.dump + pg_restore -d $audius_db_url --clean --if-exists --verbose discProvProduction.dump elif [ "$ENV" == 'stage' ] then echo "Downloading $ENV database..." curl https://audius-pgdump.s3-us-west-2.amazonaws.com/discProvStaging.dump -O echo "Restoring $ENV database to $audius_db_url..." - pg_restore -d $audius_db_url --clean --if-exists discProvStaging.dump + pg_restore -d $audius_db_url --clean --if-exists --verbose discProvStaging.dump else echo "Invalid env: $ENV" exit 1 diff --git a/discovery-provider/stage.env b/discovery-provider/stage.env index 263555d45..5451506d4 100644 --- a/discovery-provider/stage.env +++ b/discovery-provider/stage.env @@ -30,13 +30,3 @@ audius_web3_port=443 audius_delegate_owner_wallet= audius_delegate_private_key= - - - - - - - - - - From 1fbfcd6a8a227153948384fa45f4f3feb0a3dee9 Mon Sep 17 00:00:00 2001 From: Cheran Senthilkumar Date: Tue, 10 May 2022 14:12:26 +0530 Subject: [PATCH 09/10] Rename ENV to NETWORK and use env file over args --- audius-cli | 53 ++++++++++++++++----------- creator-node/docker-compose.yml | 2 +- discovery-provider/docker-compose.yml | 6 +-- discovery-provider/seed.sh | 18 ++++----- 4 files changed, 43 insertions(+), 36 deletions(-) diff --git a/audius-cli b/audius-cli index 63e5ccb99..701fef3cf 100755 --- a/audius-cli +++ b/audius-cli @@ -23,19 +23,15 @@ SERVICE_PORTS = { } service_type = click.Choice(["creator-node", "discovery-provider"]) -environment = click.Choice(["prod", "stage"]) +network_type = click.Choice(["prod", "stage"]) container_type = click.Choice(["backend", "cache", "db"]) -def update_base_env(ctx, environment): - """ - Sets the base env in .env for each service and - returns the name of the base env file to use. - """ - for service in ["creator-node", "discovery-provider"]: - env_file = ctx.obj["manifests_path"] / service / ".env" - dotenv.set_key(env_file, "ENV", environment) - return f"{environment}.env" +def get_network(ctx, service): + """Returns the network name for the given service.""" + return dotenv.dotenv_values(ctx.obj["manifests_path"] / service / ".env").get( + "NETWORK", "prod" + ) @click.group() @@ -50,11 +46,10 @@ def cli(ctx): @cli.command() @click.argument("service", type=service_type) -@click.option("--environment", type=environment, default="prod") @click.pass_context -def check_config(ctx, service, environment): +def check_config(ctx, service): """Check the config for a service""" - env = ctx.obj["manifests_path"] / service / update_base_env(ctx, environment) + env = ctx.obj["manifests_path"] / service / f"{get_network(ctx, service)}.env" override_env = ctx.obj["manifests_path"] / service / "override.env" env_data = dotenv.dotenv_values(env) @@ -137,14 +132,11 @@ def health_check(ctx, service): @cli.command() @click.argument("service", type=service_type) @click.option("--seed", is_flag=True) -@click.option("--environment", type=environment, default="prod") @click.pass_context -def launch(ctx, service, seed, environment): +def launch(ctx, service, seed): """Launch the service""" - update_base_env(ctx, environment) - try: - ctx.invoke(check_config, service=service, environment=environment) + ctx.invoke(check_config, service=service) except SystemExit: pass @@ -168,7 +160,7 @@ def launch(ctx, service, seed, environment): "docker", "compose", "--project-directory", - ctx.obj["manifests_path"] / service, + ctx.obj["manifests_path"] / "discovery-provider", "down", "backend", ], @@ -267,15 +259,14 @@ def restart(ctx, service, containers): @click.argument("value", required=False) @click.option("--unset", is_flag=True) @click.option("--required", is_flag=True) -@click.option("--environment", type=environment, default="prod") @click.pass_context -def set_config(ctx, service, unset, required, key, value, environment): +def set_config(ctx, service, unset, required, key, value): """Set a config value""" if required and (key or value): click.secho("--required cannot be used when key or value is set", fg="red") sys.exit(1) - env = ctx.obj["manifests_path"] / service / update_base_env(ctx, environment) + env = ctx.obj["manifests_path"] / service / f"{get_network(ctx, service)}.env" env_data = dotenv.dotenv_values(env) override_env = ctx.obj["manifests_path"] / service / "override.env" @@ -324,6 +315,24 @@ def set_tag(ctx, unset, tag): dotenv.set_key(env_file, "TAG", tag) +@cli.command() +@click.argument("network", type=network_type, required=False) +@click.option("--unset", is_flag=True) +@click.pass_context +def set_network(ctx, unset, network): + """Set a config value""" + if not unset and network is None: + network = click.prompt(click.style("Network", bold=True)) + + for service in ["creator-node", "discovery-provider"]: + env_file = ctx.obj["manifests_path"] / service / ".env" + + if unset: + dotenv.unset_key(env_file, "NETWORK") + else: + dotenv.set_key(env_file, "NETWORK", network) + + @cli.command() @click.argument("branch", required=False) @click.pass_context diff --git a/creator-node/docker-compose.yml b/creator-node/docker-compose.yml index 4b7bf5ed6..d286585ac 100644 --- a/creator-node/docker-compose.yml +++ b/creator-node/docker-compose.yml @@ -32,7 +32,7 @@ services: ports: - "4000:4000" env_file: - - ${ENV}.env + - ${NETWORK}.env - override.env networks: - creator-node-network diff --git a/discovery-provider/docker-compose.yml b/discovery-provider/docker-compose.yml index 6697d4dee..e0c7b1503 100644 --- a/discovery-provider/docker-compose.yml +++ b/discovery-provider/docker-compose.yml @@ -60,16 +60,16 @@ services: ports: - "5000:5000" env_file: - - ${ENV}.env + - ${NETWORK}.env - override.env networks: - discovery-provider-network seed: image: audius/discovery-provider:${TAG:-257b60f2ff0b990138d08e618b4e364e3b27b6d7} - command: bash /usr/share/seed.sh ${ENV} + command: bash /usr/share/seed.sh ${NETWORK} env_file: - - ${ENV}.env + - ${NETWORK}.env - override.env volumes: - ./seed.sh:/usr/share/seed.sh diff --git a/discovery-provider/seed.sh b/discovery-provider/seed.sh index 6d6bf5026..f8a298934 100644 --- a/discovery-provider/seed.sh +++ b/discovery-provider/seed.sh @@ -1,20 +1,18 @@ #!/usr/bin/bash -ENV=$1 +NETWORK=$1 -if [ "$ENV" == 'prod' ] -then - echo "Downloading $ENV database..." +if [ "$NETWORK" == 'prod' ]; then + echo "Downloading $NETWORK database..." curl https://audius-pgdump.s3-us-west-2.amazonaws.com/discProvProduction.dump -O - echo "Restoring $ENV database to $audius_db_url..." + echo "Restoring $NETWORK database to $audius_db_url..." pg_restore -d $audius_db_url --clean --if-exists --verbose discProvProduction.dump -elif [ "$ENV" == 'stage' ] -then - echo "Downloading $ENV database..." +elif [ "$NETWORK" == 'stage' ]; then + echo "Downloading $NETWORK database..." curl https://audius-pgdump.s3-us-west-2.amazonaws.com/discProvStaging.dump -O - echo "Restoring $ENV database to $audius_db_url..." + echo "Restoring $NETWORK database to $audius_db_url..." pg_restore -d $audius_db_url --clean --if-exists --verbose discProvStaging.dump else - echo "Invalid env: $ENV" + echo "Invalid network: $NETWORK" exit 1 fi From bee2f5298453051f8e8c1dda9e8d78a50945d083 Mon Sep 17 00:00:00 2001 From: Cheran Senthilkumar Date: Tue, 10 May 2022 14:34:30 +0530 Subject: [PATCH 10/10] Minor fixes --- README.md | 2 -- discovery-provider/seed.sh | 5 +++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index cccd17155..6b52dc612 100644 --- a/README.md +++ b/README.md @@ -112,6 +112,4 @@ audius-cli launch # Options: # --seed # Seeds the database from a snapshot. Required for first-time discovery setup. -# --environment [stage, prod] -# Uses environment variables for the relevant environment. ``` diff --git a/discovery-provider/seed.sh b/discovery-provider/seed.sh index f8a298934..85b227f0b 100644 --- a/discovery-provider/seed.sh +++ b/discovery-provider/seed.sh @@ -1,13 +1,14 @@ #!/usr/bin/bash +set -e NETWORK=$1 -if [ "$NETWORK" == 'prod' ]; then +if [[ "$NETWORK" == "prod" ]]; then echo "Downloading $NETWORK database..." curl https://audius-pgdump.s3-us-west-2.amazonaws.com/discProvProduction.dump -O echo "Restoring $NETWORK database to $audius_db_url..." pg_restore -d $audius_db_url --clean --if-exists --verbose discProvProduction.dump -elif [ "$NETWORK" == 'stage' ]; then +elif [[ "$NETWORK" == "stage" ]]; then echo "Downloading $NETWORK database..." curl https://audius-pgdump.s3-us-west-2.amazonaws.com/discProvStaging.dump -O echo "Restoring $NETWORK database to $audius_db_url..."