From a7a535fdd4f0e8e46cc5a7ac751f945fdb42311b Mon Sep 17 00:00:00 2001 From: Dheeraj Manjunath Date: Tue, 17 May 2022 13:25:18 -0400 Subject: [PATCH 01/19] Few more auto ugprade + storage space fixes --- audius-cli | 21 +++++++++++++++++++-- setup.sh | 3 ++- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/audius-cli b/audius-cli index 72d315b24..da55c095f 100755 --- a/audius-cli +++ b/audius-cli @@ -3,6 +3,8 @@ import json import os import pathlib +import random +import shutil import subprocess import sys import urllib.error @@ -15,6 +17,8 @@ import psutil RECOMMENDED_CPU_COUNT = 8 RECOMMENDED_MEMORY = 16 +RECOMMENDED_DISCOVERY_STORAGE_GB = 250 +RECOMMENDED_CONTENT_STORAGE_GB = 2000 SERVICE_PORTS = { "discovery-provider": "5000", @@ -142,11 +146,24 @@ def launch(ctx, service, seed): total_memory = psutil.virtual_memory().total // (1024 ** 3) cpu_count = psutil.cpu_count() + total_storage, _, _ = shutil.disk_usage("/var/k8s") + total_storage_gb = (total_storage // (2**30)) + meets_storage_requirements = False + click.echo(f"CPUs:\t{cpu_count}\t(required: {RECOMMENDED_CPU_COUNT})") click.echo(f"Memory:\t{total_memory}GB\t(required: {RECOMMENDED_MEMORY}GB)") - if cpu_count < RECOMMENDED_CPU_COUNT or total_memory < RECOMMENDED_MEMORY: + if service == "discovery-provider": + click.echo(f"Storage:\t{total_storage_gb}GB\t(required: {RECOMMENDED_DISCOVERY_STORAGE_GB}GB)") + if total_storage_gb > RECOMMENDED_DISCOVERY_STORAGE_GB: + meets_storage_requirements = True + else: + click.echo(f"Storage:\t{total_storage_gb}GB\t(required: {RECOMMENDED_CONTENT_STORAGE_GB}GB)") + if total_storage_gb > RECOMMENDED_DISCOVERY_STORAGE_GB: + meets_storage_requirements = True + + if cpu_count < RECOMMENDED_CPU_COUNT or total_memory < RECOMMENDED_MEMORY or not meets_storage_requirements: click.secho("System does not meet requirements", fg="red") else: click.secho("System meets requirements", fg="green") @@ -360,7 +377,7 @@ def upgrade(ctx, branch): @cli.command() -@click.argument("cron-expression", default="0 */6 * * *") +@click.argument("cron-expression", default=f"{random.randint(0, 60)} * * * *") # random so nodes in the network stagger upgrades @click.option("--remove", is_flag=True) @click.pass_context def auto_upgrade(ctx, remove, cron_expression): diff --git a/setup.sh b/setup.sh index d4a215706..9e121f8e5 100755 --- a/setup.sh +++ b/setup.sh @@ -57,6 +57,7 @@ touch discovery-provider/.env # setup service if [[ "$1" != "" ]]; then + audius-cli auto-upgrade audius-cli set-config --required "$1" read -p "Are you using an externally managed Postgres? [Y/n] " -n 1 -r @@ -79,7 +80,7 @@ if [[ "$1" != "" ]]; then echo if [[ "$REPLY" =~ ^([Yy]|)$ ]]; then if [[ "$1" == "discovery-provider" ]]; then - read -p "Run seed job? [Y/n] " -n 1 -r + read -p "Seed discovery db from snapshot (takes ~1 hour)? [Y/n] " -n 1 -r echo if [[ "$REPLY" =~ ^([Yy]|)$ ]]; then extra_args="--seed" From 43489f4b9a78dad885ed120910716516dff8fa1c Mon Sep 17 00:00:00 2001 From: Dheeraj Manjunath Date: Tue, 17 May 2022 17:22:00 -0400 Subject: [PATCH 02/19] Another comment --- audius-cli | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/audius-cli b/audius-cli index da55c095f..7568fd81b 100755 --- a/audius-cli +++ b/audius-cli @@ -373,7 +373,7 @@ def pull(ctx, branch): def upgrade(ctx, branch): """Pull and restarts all running services""" ctx.forward(pull) - ctx.invoke(restart) + ctx.invoke(restart) # i think this should call launch (or just do docker compose up) @cli.command() From 90839bada0c6c8a739269b06af3e071b7844b60c Mon Sep 17 00:00:00 2001 From: Cheran Senthilkumar Date: Wed, 18 May 2022 21:13:13 +0530 Subject: [PATCH 03/19] Address some comments --- audius-cli | 44 +++++++++++++++++++++++--------------------- setup.sh | 7 ++++++- 2 files changed, 29 insertions(+), 22 deletions(-) diff --git a/audius-cli b/audius-cli index 7568fd81b..94535a3e4 100755 --- a/audius-cli +++ b/audius-cli @@ -15,10 +15,14 @@ import crontab import dotenv import psutil +GB = 1024 ** 3 + RECOMMENDED_CPU_COUNT = 8 -RECOMMENDED_MEMORY = 16 -RECOMMENDED_DISCOVERY_STORAGE_GB = 250 -RECOMMENDED_CONTENT_STORAGE_GB = 2000 +RECOMMENDED_MEMORY = 16 * GB +RECOMMENDED_STORAGE = { + "creator-node": 2048 * GB, + "discovery-provider": 256 * GB, +} SERVICE_PORTS = { "discovery-provider": "5000", @@ -144,26 +148,23 @@ def launch(ctx, service, seed): except SystemExit: pass - total_memory = psutil.virtual_memory().total // (1024 ** 3) + total_memory = psutil.virtual_memory().total cpu_count = psutil.cpu_count() - total_storage, _, _ = shutil.disk_usage("/var/k8s") - total_storage_gb = (total_storage // (2**30)) - meets_storage_requirements = False - + total_storage = shutil.disk_usage("/var/k8s").total click.echo(f"CPUs:\t{cpu_count}\t(required: {RECOMMENDED_CPU_COUNT})") - click.echo(f"Memory:\t{total_memory}GB\t(required: {RECOMMENDED_MEMORY}GB)") - - if service == "discovery-provider": - click.echo(f"Storage:\t{total_storage_gb}GB\t(required: {RECOMMENDED_DISCOVERY_STORAGE_GB}GB)") - if total_storage_gb > RECOMMENDED_DISCOVERY_STORAGE_GB: - meets_storage_requirements = True - else: - click.echo(f"Storage:\t{total_storage_gb}GB\t(required: {RECOMMENDED_CONTENT_STORAGE_GB}GB)") - if total_storage_gb > RECOMMENDED_DISCOVERY_STORAGE_GB: - meets_storage_requirements = True + click.echo( + f"Memory:\t{total_memory // GB}GB\t(required: {RECOMMENDED_MEMORY // GB}GB)" + ) + click.echo( + f"Stroage:\t{total_storage // GB}GB\t(required: {RECOMMENDED_STORAGE[service] // GB}GB)" + ) - if cpu_count < RECOMMENDED_CPU_COUNT or total_memory < RECOMMENDED_MEMORY or not meets_storage_requirements: + if ( + cpu_count < RECOMMENDED_CPU_COUNT + or total_memory < RECOMMENDED_MEMORY + or total_storage < RECOMMENDED_STORAGE[service] + ): click.secho("System does not meet requirements", fg="red") else: click.secho("System meets requirements", fg="green") @@ -373,11 +374,12 @@ def pull(ctx, branch): def upgrade(ctx, branch): """Pull and restarts all running services""" ctx.forward(pull) - ctx.invoke(restart) # i think this should call launch (or just do docker compose up) + ctx.invoke(restart) @cli.command() -@click.argument("cron-expression", default=f"{random.randint(0, 60)} * * * *") # random so nodes in the network stagger upgrades +# random so nodes in the network stagger upgrades +@click.argument("cron-expression", default=f"{random.randint(0, 59)} */6 * * *") @click.option("--remove", is_flag=True) @click.pass_context def auto_upgrade(ctx, remove, cron_expression): diff --git a/setup.sh b/setup.sh index 9e121f8e5..6b75ab036 100755 --- a/setup.sh +++ b/setup.sh @@ -57,7 +57,12 @@ touch discovery-provider/.env # setup service if [[ "$1" != "" ]]; then - audius-cli auto-upgrade + read -p "Setup auto upgrade? [Y/n] " -n 1 -r + echo + if [[ "$REPLY" =~ ^([Yy]|)$ ]]; then + audius-cli auto-upgrade + fi + audius-cli set-config --required "$1" read -p "Are you using an externally managed Postgres? [Y/n] " -n 1 -r From f8ccdf478c7b24d5cf17fd88ded416a873d3cd7b Mon Sep 17 00:00:00 2001 From: Cheran Senthilkumar Date: Wed, 18 May 2022 22:56:37 +0530 Subject: [PATCH 04/19] Change back to hourly --- audius-cli | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/audius-cli b/audius-cli index 94535a3e4..95567efa4 100755 --- a/audius-cli +++ b/audius-cli @@ -379,7 +379,7 @@ def upgrade(ctx, branch): @cli.command() # random so nodes in the network stagger upgrades -@click.argument("cron-expression", default=f"{random.randint(0, 59)} */6 * * *") +@click.argument("cron-expression", default=f"{random.randint(0, 59)} * * * *") @click.option("--remove", is_flag=True) @click.pass_context def auto_upgrade(ctx, remove, cron_expression): From ef78787aab7398c21a1cf02eea49718c7de9c231 Mon Sep 17 00:00:00 2001 From: Cheran Senthilkumar Date: Wed, 18 May 2022 23:12:10 +0530 Subject: [PATCH 05/19] Update commit hash --- creator-node/docker-compose.yml | 2 +- discovery-provider/docker-compose.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/creator-node/docker-compose.yml b/creator-node/docker-compose.yml index 7401ae2dc..e6995307b 100644 --- a/creator-node/docker-compose.yml +++ b/creator-node/docker-compose.yml @@ -21,7 +21,7 @@ services: backend: container_name: server - image: audius/creator-node:${TAG:-257b60f2ff0b990138d08e618b4e364e3b27b6d7} + image: audius/creator-node:${TAG:-23d350aabedfe7e4c3a4c3bd060c2f550f695068} depends_on: db: condition: service_healthy diff --git a/discovery-provider/docker-compose.yml b/discovery-provider/docker-compose.yml index 7a83c6326..4e6ee53e0 100644 --- a/discovery-provider/docker-compose.yml +++ b/discovery-provider/docker-compose.yml @@ -52,7 +52,7 @@ services: backend: container_name: server - image: audius/discovery-provider:${TAG:-257b60f2ff0b990138d08e618b4e364e3b27b6d7} + image: audius/discovery-provider:${TAG:-23d350aabedfe7e4c3a4c3bd060c2f550f695068} restart: always depends_on: db: @@ -72,7 +72,7 @@ services: - discovery-provider-network seed: - image: audius/discovery-provider:${TAG:-257b60f2ff0b990138d08e618b4e364e3b27b6d7} + image: audius/discovery-provider:${TAG:-23d350aabedfe7e4c3a4c3bd060c2f550f695068} command: bash /usr/share/seed.sh ${NETWORK:-prod} env_file: - ${NETWORK:-prod}.env From 9713f1c8bf40cc7400e979f2d7775f8317509ae6 Mon Sep 17 00:00:00 2001 From: Cheran Senthilkumar Date: Wed, 18 May 2022 23:43:35 +0530 Subject: [PATCH 06/19] Change restart to launch --- audius-cli | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/audius-cli b/audius-cli index 95567efa4..797e14c8d 100755 --- a/audius-cli +++ b/audius-cli @@ -369,20 +369,22 @@ def pull(ctx, branch): @cli.command() +@click.argument("service", type=service_type) @click.argument("branch", required=False) @click.pass_context -def upgrade(ctx, branch): +def upgrade(ctx, service, branch): """Pull and restarts all running services""" ctx.forward(pull) - ctx.invoke(restart) + ctx.invoke(launch) @cli.command() +@click.argument("service", type=service_type) # random so nodes in the network stagger upgrades @click.argument("cron-expression", default=f"{random.randint(0, 59)} * * * *") @click.option("--remove", is_flag=True) @click.pass_context -def auto_upgrade(ctx, remove, cron_expression): +def auto_upgrade(ctx, remove, service, cron_expression): """Setup auto upgrade with a cron job""" with crontab.CronTab(user=True) as cron: for job in cron.find_comment("audius-cli auto-upgrade"): @@ -391,7 +393,7 @@ def auto_upgrade(ctx, remove, cron_expression): if not remove: log_file = ctx.obj["manifests_path"] / "auto-upgrade.log" job = cron.new( - f"date >> {log_file}; /usr/local/bin/audius-cli upgrade >> {log_file} 2>&1", + f"date >> {log_file}; /usr/local/bin/audius-cli upgrade {service} >> {log_file} 2>&1", "audius-cli auto-upgrade", ) From 8d358239a7dd6ddab80a8e18c8940cec15530d37 Mon Sep 17 00:00:00 2001 From: Cheran Senthilkumar Date: Wed, 18 May 2022 23:49:13 +0530 Subject: [PATCH 07/19] Replace upgrade with pull and launch --- audius-cli | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/audius-cli b/audius-cli index 797e14c8d..2dae4f38c 100755 --- a/audius-cli +++ b/audius-cli @@ -369,22 +369,20 @@ def pull(ctx, branch): @cli.command() -@click.argument("service", type=service_type) @click.argument("branch", required=False) @click.pass_context -def upgrade(ctx, service, branch): +def upgrade(ctx, branch): """Pull and restarts all running services""" ctx.forward(pull) - ctx.invoke(launch) + ctx.invoke(restart) @cli.command() -@click.argument("service", type=service_type) # random so nodes in the network stagger upgrades @click.argument("cron-expression", default=f"{random.randint(0, 59)} * * * *") @click.option("--remove", is_flag=True) @click.pass_context -def auto_upgrade(ctx, remove, service, cron_expression): +def auto_upgrade(ctx, remove, cron_expression): """Setup auto upgrade with a cron job""" with crontab.CronTab(user=True) as cron: for job in cron.find_comment("audius-cli auto-upgrade"): @@ -393,7 +391,7 @@ def auto_upgrade(ctx, remove, service, cron_expression): if not remove: log_file = ctx.obj["manifests_path"] / "auto-upgrade.log" job = cron.new( - f"date >> {log_file}; /usr/local/bin/audius-cli upgrade {service} >> {log_file} 2>&1", + f"date >> {log_file}; /usr/local/bin/audius-cli pull >> {log_file}; /usr/local/bin/audius-cli launch >> {log_file} 2>&1", "audius-cli auto-upgrade", ) From 97f1e87b22ea9f1337cac798bd7898705f4f03c0 Mon Sep 17 00:00:00 2001 From: Cheran Senthilkumar Date: Wed, 18 May 2022 23:51:09 +0530 Subject: [PATCH 08/19] Bugfix for launch in auto-upgrade --- audius-cli | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/audius-cli b/audius-cli index 2dae4f38c..12ae3c355 100755 --- a/audius-cli +++ b/audius-cli @@ -378,6 +378,7 @@ def upgrade(ctx, branch): @cli.command() +@click.argument("service", type=service_type) # random so nodes in the network stagger upgrades @click.argument("cron-expression", default=f"{random.randint(0, 59)} * * * *") @click.option("--remove", is_flag=True) @@ -391,7 +392,7 @@ def auto_upgrade(ctx, remove, cron_expression): if not remove: log_file = ctx.obj["manifests_path"] / "auto-upgrade.log" job = cron.new( - f"date >> {log_file}; /usr/local/bin/audius-cli pull >> {log_file}; /usr/local/bin/audius-cli launch >> {log_file} 2>&1", + f"date >> {log_file}; /usr/local/bin/audius-cli pull >> {log_file}; /usr/local/bin/audius-cli launch {service} >> {log_file} 2>&1", "audius-cli auto-upgrade", ) From aa8ca60b7ffc4df787dda7478f39fb1fdf17a410 Mon Sep 17 00:00:00 2001 From: Cheran Senthilkumar Date: Wed, 18 May 2022 23:51:51 +0530 Subject: [PATCH 09/19] oops --- audius-cli | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/audius-cli b/audius-cli index 12ae3c355..f4b941abd 100755 --- a/audius-cli +++ b/audius-cli @@ -383,7 +383,7 @@ def upgrade(ctx, branch): @click.argument("cron-expression", default=f"{random.randint(0, 59)} * * * *") @click.option("--remove", is_flag=True) @click.pass_context -def auto_upgrade(ctx, remove, cron_expression): +def auto_upgrade(ctx, remove, service, cron_expression): """Setup auto upgrade with a cron job""" with crontab.CronTab(user=True) as cron: for job in cron.find_comment("audius-cli auto-upgrade"): From a1f6a5ad8442d1decfd147b317c3742817a37ff9 Mon Sep 17 00:00:00 2001 From: Cheran Senthilkumar Date: Wed, 18 May 2022 23:53:59 +0530 Subject: [PATCH 10/19] Move remove to end --- audius-cli | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/audius-cli b/audius-cli index f4b941abd..d7454ace4 100755 --- a/audius-cli +++ b/audius-cli @@ -383,7 +383,7 @@ def upgrade(ctx, branch): @click.argument("cron-expression", default=f"{random.randint(0, 59)} * * * *") @click.option("--remove", is_flag=True) @click.pass_context -def auto_upgrade(ctx, remove, service, cron_expression): +def auto_upgrade(ctx, service, cron_expression, remove): """Setup auto upgrade with a cron job""" with crontab.CronTab(user=True) as cron: for job in cron.find_comment("audius-cli auto-upgrade"): From b7fd6f9057065e6e74ed3d4c27e93907166610e2 Mon Sep 17 00:00:00 2001 From: Cheran Senthilkumar Date: Wed, 18 May 2022 23:56:14 +0530 Subject: [PATCH 11/19] Add force flag --- audius-cli | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/audius-cli b/audius-cli index d7454ace4..47753e77a 100755 --- a/audius-cli +++ b/audius-cli @@ -140,8 +140,9 @@ def health_check(ctx, service): @cli.command() @click.argument("service", type=service_type) @click.option("--seed", is_flag=True) +@click.option("--force", is_flag=True) @click.pass_context -def launch(ctx, service, seed): +def launch(ctx, service, seed, force): """Launch the service""" try: ctx.invoke(check_config, service=service) @@ -169,7 +170,8 @@ def launch(ctx, service, seed): else: click.secho("System meets requirements", fg="green") - click.confirm(click.style("Do you want to continue?", bold=True), abort=True) + if not force: + click.confirm(click.style("Do you want to continue?", bold=True), abort=True) if seed and service == "discovery-provider": # make sure backend is not running @@ -392,7 +394,7 @@ def auto_upgrade(ctx, service, cron_expression, remove): if not remove: log_file = ctx.obj["manifests_path"] / "auto-upgrade.log" job = cron.new( - f"date >> {log_file}; /usr/local/bin/audius-cli pull >> {log_file}; /usr/local/bin/audius-cli launch {service} >> {log_file} 2>&1", + f"date >> {log_file}; /usr/local/bin/audius-cli pull >> {log_file}; /usr/local/bin/audius-cli launch {service} --force >> {log_file} 2>&1", "audius-cli auto-upgrade", ) From 3423a4daa30a7bf955156a9e36d5f2a3e6893b3f Mon Sep 17 00:00:00 2001 From: Dheeraj Manjunath Date: Wed, 18 May 2022 20:13:53 -0400 Subject: [PATCH 12/19] Bugfix and default --- setup.sh | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/setup.sh b/setup.sh index 6b75ab036..dc7fad776 100755 --- a/setup.sh +++ b/setup.sh @@ -57,11 +57,7 @@ touch discovery-provider/.env # setup service if [[ "$1" != "" ]]; then - read -p "Setup auto upgrade? [Y/n] " -n 1 -r - echo - if [[ "$REPLY" =~ ^([Yy]|)$ ]]; then - audius-cli auto-upgrade - fi + audius-cli auto-upgrade "$1" || true audius-cli set-config --required "$1" From fdf71de6a3a3e352247e6f29588ba4e161b36429 Mon Sep 17 00:00:00 2001 From: Dheeraj Manjunath Date: Wed, 18 May 2022 21:44:29 -0400 Subject: [PATCH 13/19] Force launch on launch --- setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.sh b/setup.sh index dc7fad776..dc1d367bf 100755 --- a/setup.sh +++ b/setup.sh @@ -87,7 +87,7 @@ if [[ "$1" != "" ]]; then extra_args="--seed" fi fi - audius-cli launch $extra_args "$1" + audius-cli launch $extra_args "$1" --force fi fi From 448c010174218d6e0a2a4442a5e9bee7ed697397 Mon Sep 17 00:00:00 2001 From: Cheran Senthilkumar Date: Thu, 19 May 2022 17:10:19 +0530 Subject: [PATCH 14/19] Change flow for upgrade --- audius-cli | 84 +++++++++++++++++++++++++++++++++++------------------- 1 file changed, 54 insertions(+), 30 deletions(-) diff --git a/audius-cli b/audius-cli index 47753e77a..04529a2e6 100755 --- a/audius-cli +++ b/audius-cli @@ -138,11 +138,11 @@ def health_check(ctx, service): @cli.command() -@click.argument("service", type=service_type) +@click.option("-y", "--yes", is_flag=True) @click.option("--seed", is_flag=True) -@click.option("--force", is_flag=True) +@click.argument("service", type=service_type) @click.pass_context -def launch(ctx, service, seed, force): +def launch(ctx, service, seed, yes): """Launch the service""" try: ctx.invoke(check_config, service=service) @@ -170,7 +170,7 @@ def launch(ctx, service, seed, force): else: click.secho("System meets requirements", fg="green") - if not force: + if not yes: click.confirm(click.style("Do you want to continue?", bold=True), abort=True) if seed and service == "discovery-provider": @@ -239,7 +239,6 @@ def restart(ctx, service, containers): if service is None: services = ["discovery-provider", "creator-node"] - filtered = [] for service in services: proc = subprocess.run( [ @@ -255,30 +254,27 @@ def restart(ctx, service, containers): ) if proc.stdout: - filtered.append(service) - - for service in services: - subprocess.run( - [ - "docker", - "compose", - "--project-directory", - ctx.obj["manifests_path"] / service, - "up", - "--build", - "--force-recreate", - "-d", - *containers, - ], - ) + subprocess.run( + [ + "docker", + "compose", + "--project-directory", + ctx.obj["manifests_path"] / service, + "up", + "--build", + "--force-recreate", + "-d", + *containers, + ], + ) @cli.command() +@click.option("--unset", is_flag=True) +@click.option("--required", is_flag=True) @click.argument("service", type=service_type) @click.argument("key", required=False) @click.argument("value", required=False) -@click.option("--unset", is_flag=True) -@click.option("--required", is_flag=True) @click.pass_context def set_config(ctx, service, unset, required, key, value): """Set a config value""" @@ -318,8 +314,8 @@ def set_config(ctx, service, unset, required, key, value): @cli.command() -@click.argument("tag", required=False) @click.option("--unset", is_flag=True) +@click.argument("tag", required=False) @click.pass_context def set_tag(ctx, unset, tag): """Set the commit tag""" @@ -336,8 +332,8 @@ def set_tag(ctx, unset, tag): @cli.command() -@click.argument("network", type=network_type, required=False) @click.option("--unset", is_flag=True) +@click.argument("network", type=network_type, required=False) @click.pass_context def set_network(ctx, unset, network): """Set the deployment network""" @@ -376,16 +372,41 @@ def pull(ctx, branch): def upgrade(ctx, branch): """Pull and restarts all running services""" ctx.forward(pull) - ctx.invoke(restart) + + for service in ["creator-node", "discovery-provider"]: + proc = subprocess.run( + [ + "docker", + "compose", + "--project-directory", + ctx.obj["manifests_path"] / service, + "ps", + "-q", + "backend", + ], + capture_output=True, + ) + + # update if service is running + if proc.stdout: + subprocess.run( + [ + "docker", + "compose", + "--project-directory", + ctx.obj["manifests_path"] / service, + "up", + "-d", + ], + ) @cli.command() -@click.argument("service", type=service_type) # random so nodes in the network stagger upgrades -@click.argument("cron-expression", default=f"{random.randint(0, 59)} * * * *") @click.option("--remove", is_flag=True) +@click.argument("cron-expression", default=f"{random.randint(0, 59)} * * * *") @click.pass_context -def auto_upgrade(ctx, service, cron_expression, remove): +def auto_upgrade(ctx, remove, cron_expression): """Setup auto upgrade with a cron job""" with crontab.CronTab(user=True) as cron: for job in cron.find_comment("audius-cli auto-upgrade"): @@ -394,7 +415,10 @@ def auto_upgrade(ctx, service, cron_expression, remove): if not remove: log_file = ctx.obj["manifests_path"] / "auto-upgrade.log" job = cron.new( - f"date >> {log_file}; /usr/local/bin/audius-cli pull >> {log_file}; /usr/local/bin/audius-cli launch {service} --force >> {log_file} 2>&1", + ( + f"date >> {log_file};" + f"/usr/local/bin/audius-cli upgrade >> {log_file} 2>&1;" + ), "audius-cli auto-upgrade", ) From 8a65244172c13db228667d91a8f128bd0227e25b Mon Sep 17 00:00:00 2001 From: Cheran Senthilkumar Date: Thu, 19 May 2022 17:11:35 +0530 Subject: [PATCH 15/19] Add comment --- setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.sh b/setup.sh index dc1d367bf..005161396 100755 --- a/setup.sh +++ b/setup.sh @@ -87,7 +87,7 @@ if [[ "$1" != "" ]]; then extra_args="--seed" fi fi - audius-cli launch $extra_args "$1" --force + audius-cli launch $extra_args "$1" # do not pass --yes so that we check for machine requirements fi fi From 7172d956a0ba6084702bc10135f3e75430b510f5 Mon Sep 17 00:00:00 2001 From: Dheeraj Manjunath Date: Thu, 19 May 2022 11:07:30 -0400 Subject: [PATCH 16/19] Add infra setup env var --- creator-node/docker-compose.yml | 2 ++ discovery-provider/docker-compose.yml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/creator-node/docker-compose.yml b/creator-node/docker-compose.yml index e6995307b..6b52bbc79 100644 --- a/creator-node/docker-compose.yml +++ b/creator-node/docker-compose.yml @@ -36,6 +36,8 @@ services: env_file: - ${NETWORK:-prod}.env - override.env + environment: + - audiusInfraSetup="audius-docker-compose" networks: - creator-node-network diff --git a/discovery-provider/docker-compose.yml b/discovery-provider/docker-compose.yml index 4e6ee53e0..e6452fe42 100644 --- a/discovery-provider/docker-compose.yml +++ b/discovery-provider/docker-compose.yml @@ -68,6 +68,8 @@ services: env_file: - ${NETWORK:-prod}.env - override.env + environment: + - audius_infra_setup="audius-docker-compose" networks: - discovery-provider-network From f3836b57804f6d26aad572db20aa640fea4e0b32 Mon Sep 17 00:00:00 2001 From: Dheeraj Manjunath Date: Thu, 19 May 2022 11:13:42 -0400 Subject: [PATCH 17/19] Add auto upgrade prompt back in --- setup.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/setup.sh b/setup.sh index 005161396..e9e5fb583 100755 --- a/setup.sh +++ b/setup.sh @@ -57,7 +57,11 @@ touch discovery-provider/.env # setup service if [[ "$1" != "" ]]; then - audius-cli auto-upgrade "$1" || true + read -p "Setup auto upgrade? [Y/n] " -n 1 -r + echo + if [[ "$REPLY" =~ ^([Yy]|)$ ]]; then + audius-cli auto-upgrade "$1" + fi audius-cli set-config --required "$1" From 035293be63155008da425673e73ec901fcad8bb9 Mon Sep 17 00:00:00 2001 From: Dheeraj Manjunath Date: Thu, 19 May 2022 11:21:07 -0400 Subject: [PATCH 18/19] Remove service type --- setup.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.sh b/setup.sh index e9e5fb583..da5a2ce0d 100755 --- a/setup.sh +++ b/setup.sh @@ -57,10 +57,10 @@ touch discovery-provider/.env # setup service if [[ "$1" != "" ]]; then - read -p "Setup auto upgrade? [Y/n] " -n 1 -r + read -p "Enable auto upgrade? [Y/n] " -n 1 -r echo if [[ "$REPLY" =~ ^([Yy]|)$ ]]; then - audius-cli auto-upgrade "$1" + audius-cli auto-upgrade fi audius-cli set-config --required "$1" From f8ac9d122a78b161774d04e73f088dead20b6743 Mon Sep 17 00:00:00 2001 From: Dheeraj Manjunath Date: Thu, 19 May 2022 16:42:37 -0400 Subject: [PATCH 19/19] Updated env vars --- creator-node/docker-compose.yml | 2 +- discovery-provider/docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/creator-node/docker-compose.yml b/creator-node/docker-compose.yml index 6b52bbc79..5cacba118 100644 --- a/creator-node/docker-compose.yml +++ b/creator-node/docker-compose.yml @@ -37,7 +37,7 @@ services: - ${NETWORK:-prod}.env - override.env environment: - - audiusInfraSetup="audius-docker-compose" + - audiusContentInfraSetup=audius-docker-compose networks: - creator-node-network diff --git a/discovery-provider/docker-compose.yml b/discovery-provider/docker-compose.yml index e6452fe42..d1b6bc005 100644 --- a/discovery-provider/docker-compose.yml +++ b/discovery-provider/docker-compose.yml @@ -69,7 +69,7 @@ services: - ${NETWORK:-prod}.env - override.env environment: - - audius_infra_setup="audius-docker-compose" + - audius_discprov_infra_setup=audius-docker-compose networks: - discovery-provider-network