feat: health check for scheduler (ENG-2154) (#43)

PR #42 added the check for expected periodic jobs to the worker health
check, but that doesn't actually help because it restarts the worker
process rather than the scheduler process. This creates a health check
for the scheduler and moves the periodic jobs check to that.

Fixes: [ENG-2154](https://stacklet.atlassian.net/browse/ENG-2154)

Author: johnsca

bin/docker-entrypoint

@@ -4,7 +4,7 @@ set -e
 scheduler() {
   echo "Starting RQ scheduler..."
 
-  exec /app/manage.py rq scheduler
+  exec supervisord -c scheduler.conf
 }
 
 dev_scheduler() {

redash/cli/rq.py

@@ -32,6 +32,34 @@ def scheduler():
     rq_scheduler.run()
 
 
+class SchedulerHealthcheck(base.BaseCheck):
+    NAME = "RQ Scheduler Healthcheck"
+
+    def __call__(self, process_spec):
+        pjobs_ok, num_pjobs, num_missing_pjobs = check_periodic_jobs()
+
+        is_healthy = pjobs_ok
+
+        self._log(
+            "Scheduler healthcheck: "
+            "Periodic jobs ok? %s (%s/%s jobs scheduled). "
+            "==> Is healthy? %s",
+            pjobs_ok,
+            num_pjobs - num_missing_pjobs,
+            num_pjobs,
+            is_healthy,
+        )
+
+        return is_healthy
+
+
+@manager.command()
+def scheduler_healthcheck():
+    return check_runner.CheckRunner(
+        "scheduler_healthcheck", "scheduler", None, [(SchedulerHealthcheck, {})]
+    ).run()
+
+
 @manager.command()
 @argument("queues", nargs=-1)
 def worker(queues):
@@ -76,33 +104,27 @@ def __call__(self, process_spec):
         total_jobs_in_watched_queues = sum([len(q.jobs) for q in worker.queues])
         has_nothing_to_do = total_jobs_in_watched_queues == 0
 
-        pjobs_ok, num_pjobs, num_missing_pjobs = check_periodic_jobs()
-
-        is_healthy = (is_busy or seen_lately or has_nothing_to_do) and pjobs_ok
+        is_healthy = is_busy or seen_lately or has_nothing_to_do
 
         self._log(
             "Worker %s healthcheck: Is busy? %s. "
             "Seen lately? %s (%d seconds ago). "
             "Has nothing to do? %s (%d jobs in watched queues). "
-            "Periodic jobs ok? %s (%s missing of %s). "
             "==> Is healthy? %s",
             worker.key,
             is_busy,
             seen_lately,
             time_since_seen.seconds,
             has_nothing_to_do,
             total_jobs_in_watched_queues,
-            pjobs_ok,
-            num_missing_pjobs,
-            num_pjobs,
             is_healthy,
         )
 
         return is_healthy
 
 
 @manager.command()
-def healthcheck():
+def worker_healthcheck():
     return check_runner.CheckRunner(
         "worker_healthcheck", "worker", None, [(WorkerHealthcheck, {})]
     ).run()

scheduler.conf

@@ -0,0 +1,33 @@
+[supervisord]
+logfile=/dev/null
+pidfile=/tmp/supervisord.pid
+nodaemon=true
+
+[unix_http_server]
+file = /tmp/supervisor.sock
+
+[rpcinterface:supervisor]
+supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
+
+[program:scheduler]
+command=./manage.py rq scheduler
+process_name=%(program_name)s-%(process_num)s
+numprocs=1
+directory=/app
+stopsignal=TERM
+autostart=true
+autorestart=true
+startsecs=300
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+
+[eventlistener:scheduler_healthcheck]
+serverurl=AUTO
+command=./manage.py rq scheduler_healthcheck
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+events=TICK_60

worker.conf

@@ -25,7 +25,7 @@ stderr_logfile_maxbytes=0
 
 [eventlistener:worker_healthcheck]
 serverurl=AUTO
-command=./manage.py rq healthcheck
+command=./manage.py rq worker_healthcheck
 stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0
 stderr_logfile=/dev/stderr