parse expiration time and reload config at time out

Author: duke8253

plugins/s3_auth/s3_auth.cc

@@ -37,6 +37,11 @@
 #include <openssl/sha.h>
 #include <openssl/hmac.h>
 
+#include <chrono>
+#include <atomic>
+#include <mutex>
+#include <shared_mutex>
+
 #include <ts/ts.h>
 #include <ts/remap.h>
 #include "tscore/ink_config.h"
@@ -153,6 +158,7 @@ ConfigCache gConfCache;
 // One configuration setup
 //
 int event_handler(TSCont, TSEvent, void *); // Forward declaration
+int config_reloader(TSCont, TSEvent, void *);
 
 class S3Config
 {
@@ -162,6 +168,9 @@ class S3Config
     if (get_cont) {
       _cont = TSContCreate(event_handler, nullptr);
       TSContDataSet(_cont, static_cast<void *>(this));
+
+      _conf_rld = TSContCreate(config_reloader, TSMutexCreate());
+      TSContDataSet(_conf_rld, static_cast<void *>(this));
     }
   }
 
@@ -171,9 +180,13 @@ class S3Config
     TSfree(_secret);
     TSfree(_keyid);
     TSfree(_token);
+    TSfree(_conf_fname);
     if (_cont) {
       TSContDestroy(_cont);
     }
+    if (_conf_rld) {
+      TSContDestroy(_conf_rld);
+    }
   }
 
   // Is this configuration usable?
@@ -212,16 +225,19 @@ class S3Config
   copy_changes_from(const S3Config *src)
   {
     if (src->_secret) {
+      TSfree(_secret);
       _secret     = TSstrdup(src->_secret);
       _secret_len = src->_secret_len;
     }
 
     if (src->_keyid) {
+      TSfree(_keyid);
       _keyid     = TSstrdup(src->_keyid);
       _keyid_len = src->_keyid_len;
     }
 
     if (src->_token) {
+      TSfree(_token);
       _token     = TSstrdup(src->_token);
       _token_len = src->_token_len;
     }
@@ -250,6 +266,13 @@ class S3Config
       _region_map          = src->_region_map;
       _region_map_modified = true;
     }
+
+    _expiration = src->_expiration;
+
+    if (src->_conf_fname) {
+      TSfree(_conf_fname);
+      _conf_fname = TSstrdup(src->_conf_fname);
+    }
   }
 
   // Getters
@@ -319,6 +342,24 @@ class S3Config
     return _region_map;
   }
 
+  long
+  expiration() const
+  {
+    return _expiration;
+  }
+
+  const char *
+  conf_fname() const
+  {
+    return _conf_fname;
+  }
+
+  int
+  incr_conf_reload_count()
+  {
+    return _conf_reload_count++;
+  }
+
   // Setters
   void
   set_secret(const char *s)
@@ -380,6 +421,25 @@ class S3Config
     _region_map_modified = true;
   }
 
+  void
+  set_expiration(const char *s)
+  {
+    _expiration = strtol(s, nullptr, 10);
+  }
+
+  void
+  set_conf_fname(const char *s)
+  {
+    TSfree(_conf_fname);
+    _conf_fname = TSstrdup(s);
+  }
+
+  void
+  reset_conf_reload_count()
+  {
+    _conf_reload_count = 0;
+  }
+
   // Parse configs from an external file
   bool parse_config(const std::string &filename);
 
@@ -391,6 +451,15 @@ class S3Config
     TSHttpTxnHookAdd(txnp, TS_HTTP_SEND_REQUEST_HDR_HOOK, _cont);
   }
 
+  void
+  schedule_conf_reload(long delay) const
+  {
+    TSContScheduleOnPool(_conf_rld, delay * 1000, TS_THREAD_POOL_NET);
+  }
+
+  std::shared_mutex reload_mutex;
+  std::atomic_bool reload_waiting = false;
+
 private:
   char *_secret            = nullptr;
   size_t _secret_len       = 0;
@@ -403,12 +472,16 @@ class S3Config
   bool _version_modified   = false;
   bool _virt_host_modified = false;
   TSCont _cont             = nullptr;
+  TSCont _conf_rld         = nullptr;
   StringSet _v4includeHeaders;
   bool _v4includeHeaders_modified = false;
   StringSet _v4excludeHeaders;
   bool _v4excludeHeaders_modified = false;
   StringMap _region_map;
   bool _region_map_modified = false;
+  long _expiration          = 0;
+  char *_conf_fname         = nullptr;
+  int _conf_reload_count    = 0;
 };
 
 bool
@@ -465,6 +538,8 @@ S3Config::parse_config(const std::string &config_fname)
         set_exclude_headers(pos2 + 19);
       } else if (0 == strncasecmp(pos2, "v4-region-map=", 14)) {
         set_region_map(pos2 + 14);
+      } else if (0 == strncasecmp(pos2, "expiration=", 11)) {
+        set_expiration(pos2 + 11);
       } else {
         // ToDo: warnings?
       }
@@ -499,6 +574,7 @@ ConfigCache::get(const char *fname)
     if (tv.tv_sec > (it->second.second + _ttl)) {
       // Update the cached configuration file.
       S3Config *s3 = new S3Config(false); // false == this config does not get the continuation
+      s3->set_conf_fname(fname);
 
       TSDebug(PLUGIN_NAME, "Configuration from %s is stale, reloading", config_fname.c_str());
       it->second.second = tv.tv_sec;
@@ -516,6 +592,7 @@ ConfigCache::get(const char *fname)
   } else {
     // Create a new cached file.
     S3Config *s3 = new S3Config(false); // false == this config does not get the continuation
+    s3->set_conf_fname(fname);
 
     if (s3->parse_config(config_fname)) {
       _cache[config_fname] = std::make_pair(s3, tv.tv_sec);
@@ -876,7 +953,13 @@ event_handler(TSCont cont, TSEvent event, void *edata)
   switch (event) {
   case TS_EVENT_HTTP_SEND_REQUEST_HDR:
     if (request.initialize()) {
-      status = request.authorize(s3);
+      while (true) {
+        if (!s3->reload_waiting) {
+          std::shared_lock lock(s3->reload_mutex);
+          status = request.authorize(s3);
+          break;
+        }
+      }
     }
 
     if (TS_HTTP_STATUS_OK == status) {
@@ -897,6 +980,63 @@ event_handler(TSCont cont, TSEvent event, void *edata)
   return 0;
 }
 
+// If the token has more than one hour to expire, reload is scheduled one hour before expiration.
+// If the token has less than one hour to expire, reload is scheduled 15 minutes before expiration.
+// If the token has less than 15 minutes to expire, reload is scheduled at the expiration time.
+static long
+cal_reload_delay(long time_diff)
+{
+  if (time_diff > 3600) {
+    return time_diff - 3600;
+  } else if (time_diff > 900) {
+    return time_diff - 900;
+  } else {
+    return time_diff;
+  }
+}
+
+int
+config_reloader(TSCont cont, TSEvent event, void *edata)
+{
+  TSDebug(PLUGIN_NAME, "reloading configs");
+  S3Config *s3          = static_cast<S3Config *>(TSContDataGet(cont));
+  S3Config *file_config = gConfCache.get(s3->conf_fname());
+
+  if (!file_config->valid()) {
+    TSError("[%s] requires both shared and AWS secret configuration", PLUGIN_NAME);
+    return TS_ERROR;
+  }
+
+  s3->reload_waiting = true;
+  {
+    std::unique_lock lock(s3->reload_mutex);
+    s3->copy_changes_from(file_config);
+  }
+  s3->reload_waiting = false;
+
+  if (s3->expiration() == 0) {
+    TSDebug(PLUGIN_NAME, "disabling auto config reload");
+  } else {
+    // auto reload is scheduled to be 5 minutes before the expiration time to get some headroom
+    long time_diff = s3->expiration() -
+                     std::chrono::duration_cast<std::chrono::seconds>(std::chrono::system_clock::now().time_since_epoch()).count();
+    if (time_diff > 0) {
+      long delay = cal_reload_delay(time_diff);
+      TSDebug(PLUGIN_NAME, "scheduling config reload with %ld seconds delay", delay);
+      s3->reset_conf_reload_count();
+      s3->schedule_conf_reload(delay);
+    } else {
+      TSDebug(PLUGIN_NAME, "config expiration time is in the past, re-checking in 1 minute");
+      if (s3->incr_conf_reload_count() == 10) {
+        TSError("[%s] tried to reload config automatically but failed, please try manual reloading the config", PLUGIN_NAME);
+      }
+      s3->schedule_conf_reload(60);
+    }
+  }
+
+  return TS_SUCCESS;
+}
+
 ///////////////////////////////////////////////////////////////////////////////
 // Initialize the plugin.
 //
@@ -1000,6 +1140,23 @@ TSRemapNewInstance(int argc, char *argv[], void **ih, char * /* errbuf ATS_UNUSE
     return TS_ERROR;
   }
 
+  if (s3->expiration() == 0) {
+    TSDebug(PLUGIN_NAME, "disabling auto config reload");
+  } else {
+    // auto reload is scheduled to be 5 minutes before the expiration time to get some headroom
+    long time_diff = s3->expiration() -
+                     std::chrono::duration_cast<std::chrono::seconds>(std::chrono::system_clock::now().time_since_epoch()).count();
+    if (time_diff > 0) {
+      long delay = cal_reload_delay(time_diff);
+      TSDebug(PLUGIN_NAME, "scheduling config reload with %ld seconds delay", delay);
+      s3->reset_conf_reload_count();
+      s3->schedule_conf_reload(delay);
+    } else {
+      TSDebug(PLUGIN_NAME, "config expiration time is in the past, re-checking in 1 minute");
+      s3->schedule_conf_reload(60);
+    }
+  }
+
   *ih = static_cast<void *>(s3);
   TSDebug(PLUGIN_NAME, "New rule: access_key=%s, virtual_host=%s, version=%d", s3->keyid(), s3->virt_host() ? "yes" : "no",
           s3->version());