Merge pull request #32 from figma/ck/log-heartbeat

Write to stdout every 5 minutes on master processes

Author: ckilpatrick-stay-hydrated-call-your-mom

javascript/src/queue/Worker.ts

@@ -12,6 +12,7 @@ export class Worker extends BaseRunner {
 
   async *pollIter() {
     let lastLogTime = 0;
+    let lastHeartbeatTime = Date.now() / 1000;
     await this.waitForMaster();
     while (
       !this.shutdownRequired &&
@@ -35,6 +36,12 @@ export class Worker extends BaseRunner {
           console.log('[ci-queue] No test to reserve, sleeping');
           lastLogTime = now;
         }
+
+        // Log heartbeat every 5 minutes if this is the master process
+        if (this.isMaster && now - lastHeartbeatTime > 300) {
+          console.log('[ci-queue] Still working');
+          lastHeartbeatTime = now;
+        }
         await sleep(500);
       }
     }

python/ciqueue/distributed.py

@@ -70,11 +70,16 @@ def __init__(self, tests, worker_id, redis, build_id,  # pylint: disable=too-man
 
     def __iter__(self):
         def poll():
+            last_heartbeat_time = time.time()
             while not self.shutdown_required and len(self):  # pylint: disable=len-as-condition
                 test = self._reserve()
                 if test:
                     yield test.decode()
                 else:
+                    # Log heartbeat every 5 minutes if this is the master process
+                    if self.is_master and time.time() - last_heartbeat_time > 300:
+                        print('[ci-queue] Still working')
+                        last_heartbeat_time = time.time()
                     time.sleep(0.05)
 
         try:

ruby/lib/ci/queue/redis/worker.rb

@@ -50,10 +50,16 @@ def master?
 
         def poll
           wait_for_master
+          last_heartbeat_time = Time.now
           until shutdown_required? || config.circuit_breakers.any?(&:open?) || exhausted? || max_test_failed?
             if test = reserve
               yield index.fetch(test)
             else
+              # Log heartbeat every 5 minutes if this is the master process
+              if master? && Time.now - last_heartbeat_time > 300
+                puts '[ci-queue] Still working'
+                last_heartbeat_time = Time.now
+              end
               sleep 0.05
             end
           end