feat(helm): Cogstack CE - Opensearch dashboard provisioning

deployment/kubernetes/charts/cogstack-helm-ce/provisioning/base_index_settings.json

@@ -0,0 +1,70 @@
+{
+    "index_patterns": [
+        "admissions",
+        "drgcodes",
+        "emar",
+        "icustays",
+        "patients",
+        "poe"
+    ],
+    "template": {
+        "mappings": {
+            "date_detection": true,
+            "dynamic_date_formats": [
+                "yyyy-MM-dd HH:mm:ss",
+                "yyyy-MM-dd",
+                "basic_date",
+                "date_hour",
+                "date_hour_minute",
+                "date_hour_minute_second",
+                "time",
+                "hour_minute",
+                "yyyy/MM/dd",
+                "dd/MM/yyyy",
+                "dd/MM/yyyy HH:mm",
+                "date_time",
+                "t_time",
+                "date_hour_minute_second_millis",
+                "basic_time",
+                "basic_time_no_millis",
+                "basic_t_time",
+                "hour_minute_second",
+                "HH:mm.ss",
+                "HH:mm.ssZ"
+            ],
+            "dynamic_templates": [
+                {
+                    "dates": {
+                        "match_mapping_type": "string",
+                        "match_pattern": "regex",
+                        "match": "(?i).*(date|_datetime|_dt|_dttm|_dat|_datime|_time|_ts|_timestamp|time|when|dt|dttm|timestamp|created|updated|modified|inserted|recorded|logged|entered|performed|signed|cosigned|completed|admit|discharge|visit|appointment|service|start|end|effective|expiry|validfrom|validto|close)$",
+                        "mapping": {
+                            "type": "date",
+                            "format": "yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis||basic_date||date_hour_minute_second"
+                        }
+                    }
+                },
+                {
+                    "strings_as_text": {
+                        "match_mapping_type": "string",
+                        "mapping": {
+                            "type": "text",
+                            "analyzer": "standard",
+                            "fields": {
+                                "keyword": {
+                                    "type": "keyword",
+                                    "ignore_above": 256
+                                }
+                            }
+                        }
+                    }
+                }
+            ],
+            "properties": {
+                "id": {
+                    "type": "keyword"
+                }
+            }
+        }
+    }
+}

deployment/kubernetes/charts/cogstack-helm-ce/provisioning/opensearch-provisioning.sh

@@ -0,0 +1,134 @@
+#!/usr/bin/sh
+set -e
+log() {
+    local ts
+    ts=$(date +"%Y-%m-%d %H:%M:%S")
+    echo "[$ts] $*"
+}
+
+: "${OPENSEARCH_URL:?OPENSEARCH_URL is required. Include scheme, port and /api eg http://localhost:9200/ }"
+: "${OPENSEARCH_DASHBOARD_URL:?OPENSEARCH_DASHBOARD_URL is required. Include scheme, port and /api eg http://localhost:5601/api }"
+: "${OPENSEARCH_USERNAME:?OPENSEARCH_USERNAME is required. }"
+: "${OPENSEARCH_PASSWORD:?OPENSEARCH_PASSWORD is required. }"
+
+: "${CONFIG_DIR:?CONFIG_DIR is required. }"
+: "${CURL_BODY_FILE:=/tmp/curl_body.$$}"
+# OPENSEARCH_URL=https://localhost:9200
+# OPENSEARCH_DASHBOARD_URL=http://localhost:5601
+# OPENSEARCH_USERNAME=admin
+# OPENSEARCH_PASSWORD=opensearch-312$A
+
+wait_for_service() {
+    local service_name="$1"
+    local url="$2"
+    local curl_extra_args="${3:-}"
+    local max_wait=120
+    local interval=2
+    local elapsed=0
+
+    while true; do
+        if curl -sf --insecure $curl_extra_args "$url" >/dev/null; then
+            log "$service_name is running"
+            return 0
+        else
+            log "$service_name is not running at $url"
+            if [ "$elapsed" -ge "$max_wait" ]; then
+                log "Timed out waiting for $service_name to become healthy"
+                return 1
+            fi
+            log "Retrying in $interval seconds..."
+            sleep "$interval"
+            elapsed=$((elapsed + interval))
+            interval=$(( interval * 2 ))
+            if [ "$interval" -gt 10 ]; then
+                interval=10
+            fi
+        fi
+    done
+}
+
+OPENSEARCH_AUTH="$OPENSEARCH_USERNAME:$OPENSEARCH_PASSWORD"
+
+if [ "$PROVISION_OPENSEARCH_INDEX_TEMPLATE_ENABLED" = "true" ]; then
+    wait_for_service "OpenSearch" "$OPENSEARCH_URL" "-u $OPENSEARCH_AUTH" || exit 1
+    # See: https://docs.opensearch.org/latest/im-plugin/index-templates/
+    log "Creating index template - PUT $OPENSEARCH_URL/_index_template/base_index_template"
+    os_status="$(curl -sS \
+        -o "$CURL_BODY_FILE" \
+        -w "%{http_code}" \
+        -X PUT "$OPENSEARCH_URL/_index_template/base_index_template" \
+        -H "Content-Type: application/json" \
+        -u "$OPENSEARCH_AUTH" \
+        -k \
+        -d @"${CONFIG_DIR}/base_index_settings.json")"
+    if [ "$os_status" != "200" ] && [ "$os_status" != "201" ]; then
+        log "Failed to create index template (http_status=$os_status)"
+        if [ -s "$CURL_BODY_FILE" ]; then
+            log "Response body:"
+            sed 's/^/  /' "$CURL_BODY_FILE"
+        fi
+        exit 1
+    fi
+fi
+
+if [ "$PROVISION_OPENSEARCH_EXAMPLE_DOCUMENTS_ENABLED" = "true" ]; then
+    wait_for_service "OpenSearch" "$OPENSEARCH_URL" "-u $OPENSEARCH_AUTH" || exit 1
+    log "Creating example admissions document - POST $OPENSEARCH_URL/admissions/_doc"
+    os_status="$(curl -sS \
+        -o "$CURL_BODY_FILE" \
+        -w "%{http_code}" \
+        -X POST "$OPENSEARCH_URL/admissions/_doc" \
+        -H "Content-Type: application/json" \
+        -u "$OPENSEARCH_AUTH" \
+        -k \
+        -d '{
+  "subject_id": 10000032,
+  "hadm_id": 22595853,
+  "admittime": "2180-05-06 22:23:00",
+  "dischtime": "2180-05-07 17:15:00",
+  "admission_type": "URGENT",
+  "admit_provider_id": "P49AFC",
+  "admission_location": "TRANSFER FROM HOSPITAL",
+  "discharge_location": "HOME",
+  "insurance": "Medicaid",
+  "language": "English",
+  "marital_status": "WIDOWED",
+  "race": "WHITE",
+  "edregtime": "2180-05-06 19:17:00",
+  "edouttime": "2180-05-06 23:30:00",
+  "hospital_expire_flag": 0
+}')"
+    if [ "$os_status" != "200" ] && [ "$os_status" != "201" ]; then
+        log "Failed to create example admissions document (http_status=$os_status)"
+        if [ -s "$CURL_BODY_FILE" ]; then
+            log "Response body:"
+            sed 's/^/  /' "$CURL_BODY_FILE"
+        fi
+        exit 1
+    fi
+fi
+
+if [ "$PROVISION_OPENSEARCH_DASHBOARDS_ENABLED" = "true" ]; then
+    log "Provisioning OpenSearch Dashboards"
+    wait_for_service "OpenSearch Dashboard" "$OPENSEARCH_DASHBOARD_URL" || exit 1
+
+    log "Importing dashboards - POST $OPENSEARCH_DASHBOARD_URL/api/saved_objects/_import?overwrite=true"
+    osd_status="$(curl -sS \
+        -o "$CURL_BODY_FILE" \
+        -w "%{http_code}" \
+        -X POST "$OPENSEARCH_DASHBOARD_URL/api/saved_objects/_import?overwrite=true" \
+        -H "osd-xsrf: true" \
+        --form "file=@${CONFIG_DIR}/dashboards.ndjson" \
+        -u "$OPENSEARCH_AUTH" \
+        --insecure)"
+    if [ "$osd_status" != "200" ] && [ "$osd_status" != "201" ]; then
+        log "Failed to import dashboards (http_status=$osd_status)"
+        if [ -s "$CURL_BODY_FILE" ]; then
+            log "Response body:"
+            sed 's/^/  /' "$CURL_BODY_FILE"
+        fi
+        exit 1
+    fi
+fi
+
+log "Provisioning completed"

deployment/kubernetes/charts/cogstack-helm-ce/templates/_helpers.tpl

@@ -60,3 +60,32 @@ Create the name of the service account to use
 {{- default "default" .Values.serviceAccount.name }}
 {{- end }}
 {{- end }}
+
+{{/*
+Dependency URLs
+*/}}
+
+{{- define "opensearch.url" -}}
+{{- $serviceName := include "opensearch.serviceName" (index .Subcharts "opensearch") -}}
+{{- $scheme := default "https" .Values.opensearch.protocol -}}
+{{- $port := default 9200 .Values.opensearch.httpPort -}}
+{{- printf "%s://%s:%v" $scheme $serviceName $port -}}
+{{- end }}
+
+{{- define "opensearch-dashboards.url" -}}
+{{- $serviceName := include "opensearch-dashboards.fullname" (index .Subcharts "opensearch-dashboards") -}}
+{{- $scheme := "http" -}}
+{{- $port := default 5601 (index .Values "opensearch-dashboards" "service" "port") -}}
+{{- printf "%s://%s:%v" $scheme $serviceName $port -}}
+{{- end }}
+
+{{- define "opensearch.initialAdminPassword" -}}
+{{- $val := "" -}}
+{{- range .Values.opensearch.extraEnvs }}
+  {{- if eq .name "OPENSEARCH_INITIAL_ADMIN_PASSWORD" }}
+    {{- $val = .value -}}
+  {{- end }}
+{{- end }}
+{{- $val -}}
+{{- end }}
+

deployment/kubernetes/charts/cogstack-helm-ce/templates/opensearch-provisioning-config-map.yaml

@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: opensearch-provisioning-config
+data:
+{{ (.Files.Glob "provisioning/*").AsConfig | indent 2 }}

deployment/kubernetes/charts/cogstack-helm-ce/templates/opensearch-provisioning-post-install.yaml

@@ -0,0 +1,43 @@
+{{- if .Values.provisioning.enabled }}
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    {{- include "cogstack-helm-ce.labels" . | nindent 4 }}
+  annotations:
+    "helm.sh/hook": post-install,post-upgrade
+    "helm.sh/hook-weight": "1"
+    "helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation
+    #"helm.sh/hook-delete-policy": before-hook-creation
+  name: {{ include "cogstack-helm-ce.fullname" . }}-opensearch-provisioning
+spec:
+  restartPolicy: OnFailure
+  containers:
+  - image: curlimages/curl
+    name: opensearch-provisioning
+    command: ["/bin/sh", "/etc/config/opensearch-provisioning-config/opensearch-provisioning.sh"]
+    env:
+      - name: OPENSEARCH_URL
+        value: {{ include "opensearch.url" . }}
+      - name: OPENSEARCH_DASHBOARD_URL
+        value: {{ include "opensearch-dashboards.url" . }}
+      - name: OPENSEARCH_USERNAME
+        value: admin
+      - name: OPENSEARCH_PASSWORD
+        value: {{ include "opensearch.initialAdminPassword" . }}
+      - name: CONFIG_DIR
+        value: /etc/config/opensearch-provisioning-config
+      - name: PROVISION_OPENSEARCH_DASHBOARDS_ENABLED
+        value: "{{ .Values.provisioning.opensearchDashboards.createDashboards }}"
+      - name: PROVISION_OPENSEARCH_INDEX_TEMPLATE_ENABLED
+        value: "{{ .Values.provisioning.opensearch.createIndexTemplate }}"
+      - name: PROVISION_OPENSEARCH_EXAMPLE_DOCUMENTS_ENABLED
+        value: "{{ .Values.provisioning.opensearch.createExampleDocuments }}"
+    volumeMounts:
+    - name: opensearch-provisioning-config
+      mountPath: /etc/config/opensearch-provisioning-config
+  volumes:
+    - name: opensearch-provisioning-config
+      configMap:
+        name: opensearch-provisioning-config
+{{- end }}

deployment/kubernetes/charts/cogstack-helm-ce/values.yaml

@@ -34,6 +34,14 @@ medcat-trainer:
   env:
     CSRF_TRUSTED_ORIGINS: "http://localhost:8080"
 
+provisioning:
+  enabled: true
+  opensearch:
+    createIndexTemplate: true
+    createExampleDocuments: true
+  opensearchDashboards:
+    createDashboards: true
+
 opensearch:
   enabled: true
   extraEnvs: