Feature/#4 source configs from s3

conf/config.json

@@ -1,6 +1,6 @@
 {
-    "accessConfig": "conf/access.json",
-    "topicsConfig": "conf/topics.json",
+    "accessConfig": "s3://<redacted>/access.json",
+    "topicsConfig": "s3://<redacted>/topics.json",
     "tokenProviderUrl": "https://<redacted>",
     "tokenPublicKeyUrl": "https://<redacted>",
     "kafkaBootstrapServer": "localhost:9092"

src/event_gate_lambda.py

@@ -24,19 +24,35 @@
 import jwt
 import requests
 
+import boto3
 from confluent_kafka import Producer
 
 with open("conf/config.json", "r") as file:
     CONFIG = json.load(file)
 
-with open(CONFIG["topicsConfig"], "r") as file:
-    TOPICS = json.load(file)
+aws_session = boto3.Session()
+aws_s3 = aws_session.resource('s3', verify=False)
 
-with open(CONFIG["accessConfig"], "r") as file:
-    ACCESS = json.load(file)
+if CONFIG["topicsConfig"].startswith("s3://"):
+    name_parts = CONFIG["topicsConfig"].split('/')
+    bucket_name = name_parts[2]
+    bucket_object = "/".join(name_parts[3:])
+    TOPICS = json.loads(aws_s3.Bucket(bucket_name).Object(bucket_object).get()["Body"].read().decode("utf-8"))
+else:
+    with open(CONFIG["topicsConfig"], "r") as file:
+        TOPICS = json.load(file)
+
+if CONFIG["accessConfig"].startswith("s3://"):
+    name_parts = CONFIG["accessConfig"].split('/')
+    bucket_name = name_parts[2]
+    bucket_object = "/".join(name_parts[3:])
+    ACCESS = json.loads(aws_s3.Bucket(bucket_name).Object(bucket_object).get()["Body"].read().decode("utf-8"))
+else:
+    with open(CONFIG["accessConfig"], "r") as file:
+        ACCESS = json.load(file)
 
 TOKEN_PROVIDER_URL = CONFIG["tokenProviderUrl"]
-print("Loaded config")
+print("Loaded configs")
 
 token_public_key_encoded = requests.get(CONFIG["tokenPublicKeyUrl"], verify=False).json()["key"]
 TOKEN_PUBLIC_KEY = serialization.load_der_public_key(base64.b64decode(token_public_key_encoded))

src/requirements.txt

@@ -3,4 +3,5 @@ cryptography
 jsonschema
 PyJWT
 requests
+boto3
 confluent_kafka