From 26dd91b00e6b2f3a26f06061cad8b18df90c8692 Mon Sep 17 00:00:00 2001 From: "Triomphe Jules, INI-DNA-INF" Date: Wed, 1 Feb 2023 16:18:21 +0100 Subject: [PATCH 1/3] Add global volumes and volumeMounts in the pod template file --- .../files/pod-template-file.kubernetes-helm-yaml | 6 ++++++ tests/charts/test_pod_template_file.py | 15 +++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/chart/files/pod-template-file.kubernetes-helm-yaml b/chart/files/pod-template-file.kubernetes-helm-yaml index 80a907e7521b6..8d305c5c904d3 100644 --- a/chart/files/pod-template-file.kubernetes-helm-yaml +++ b/chart/files/pod-template-file.kubernetes-helm-yaml @@ -71,6 +71,9 @@ spec: {{- if or .Values.dags.gitSync.enabled .Values.dags.persistence.enabled }} {{- include "airflow_dags_mount" . | nindent 8 }} {{- end }} +{{- if .Values.volumeMounts }} +{{- toYaml .Values.volumeMounts | nindent 8 }} +{{- end }} {{- if .Values.workers.extraVolumeMounts }} {{ toYaml .Values.workers.extraVolumeMounts | indent 8 }} {{- end }} @@ -115,6 +118,9 @@ spec: - configMap: name: {{ include "airflow_config" . }} name: config + {{- if .Values.volumes }} + {{- toYaml .Values.volumes | nindent 2 }} + {{- end }} {{- if .Values.workers.extraVolumes }} {{ toYaml .Values.workers.extraVolumes | nindent 2 }} {{- end }} diff --git a/tests/charts/test_pod_template_file.py b/tests/charts/test_pod_template_file.py index fa5a9128c068f..1fe2635849bd4 100644 --- a/tests/charts/test_pod_template_file.py +++ b/tests/charts/test_pod_template_file.py @@ -147,6 +147,21 @@ def test_dags_mount(self, dag_values, expected_read_only): "readOnly": expected_read_only, } in jmespath.search("spec.containers[0].volumeMounts", docs[0]) + def test_should_add_global_volume_and_global_volume_mount(self): + expected_volume = {"name": "test-volume", "emptyDir": {}} + expected_volume_mount = {"name": "test-volume", "mountPath": "/opt/test"} + docs = render_chart( + values={ + "volumes": [expected_volume], + "volumeMounts": [expected_volume_mount], + }, + show_only=["templates/pod-template-file.yaml"], + chart_dir=self.temp_chart_dir, + ) + + assert expected_volume in jmespath.search("spec.volumes", docs[0]) + assert expected_volume_mount in jmespath.search("spec.containers[0].volumeMounts", docs[0]) + def test_validate_if_ssh_params_are_added(self): docs = render_chart( values={ From 151d910dac6e6447ad8c371dda8a0745e6f2022e Mon Sep 17 00:00:00 2001 From: "Triomphe Jules, INI-DNA-INF" Date: Wed, 1 Feb 2023 16:18:53 +0100 Subject: [PATCH 2/3] Fix formatting for all volumeMounts inclusions --- .../templates/dag-processor/dag-processor-deployment.yaml | 2 +- chart/templates/flower/flower-deployment.yaml | 2 +- chart/templates/jobs/create-user-job.yaml | 2 +- chart/templates/jobs/migrate-database-job.yaml | 2 +- chart/templates/pgbouncer/pgbouncer-deployment.yaml | 2 +- chart/templates/scheduler/scheduler-deployment.yaml | 6 +++--- chart/templates/triggerer/triggerer-deployment.yaml | 4 ++-- chart/templates/webserver/webserver-deployment.yaml | 4 ++-- chart/templates/workers/worker-deployment.yaml | 8 ++++---- 9 files changed, 16 insertions(+), 16 deletions(-) diff --git a/chart/templates/dag-processor/dag-processor-deployment.yaml b/chart/templates/dag-processor/dag-processor-deployment.yaml index 758d5ab5be61a..eaab6f5824536 100644 --- a/chart/templates/dag-processor/dag-processor-deployment.yaml +++ b/chart/templates/dag-processor/dag-processor-deployment.yaml @@ -152,7 +152,7 @@ spec: {{ toYaml .Values.dagProcessor.resources | nindent 12 }} volumeMounts: {{- if .Values.volumeMounts }} - {{ toYaml .Values.volumeMounts | nindent 12 }} + {{- toYaml .Values.volumeMounts | nindent 12 }} {{- end }} {{- if .Values.dagProcessor.extraVolumeMounts }} {{ toYaml .Values.dagProcessor.extraVolumeMounts | nindent 12 }} diff --git a/chart/templates/flower/flower-deployment.yaml b/chart/templates/flower/flower-deployment.yaml index 0e52190644a9f..85f0e43f431dd 100644 --- a/chart/templates/flower/flower-deployment.yaml +++ b/chart/templates/flower/flower-deployment.yaml @@ -102,7 +102,7 @@ spec: volumeMounts: {{- include "airflow_config_mount" . | nindent 12 }} {{- if .Values.volumeMounts }} - {{ toYaml .Values.volumeMounts | nindent 12 }} + {{- toYaml .Values.volumeMounts | nindent 12 }} {{- end }} {{- if .Values.flower.extraVolumeMounts }} {{ toYaml .Values.flower.extraVolumeMounts | nindent 12 }} diff --git a/chart/templates/jobs/create-user-job.yaml b/chart/templates/jobs/create-user-job.yaml index a4bb2bde9f899..aae0c40318a5a 100644 --- a/chart/templates/jobs/create-user-job.yaml +++ b/chart/templates/jobs/create-user-job.yaml @@ -107,7 +107,7 @@ spec: volumeMounts: {{- include "airflow_config_mount" . | nindent 12 }} {{- if .Values.volumeMounts }} -{{ toYaml .Values.volumeMounts | nindent 12 }} +{{- toYaml .Values.volumeMounts | nindent 12 }} {{- end }} {{- if .Values.createUserJob.extraVolumeMounts }} {{ toYaml .Values.createUserJob.extraVolumeMounts | nindent 12 }} diff --git a/chart/templates/jobs/migrate-database-job.yaml b/chart/templates/jobs/migrate-database-job.yaml index 67e61abf48dc0..a37e0f5345fde 100644 --- a/chart/templates/jobs/migrate-database-job.yaml +++ b/chart/templates/jobs/migrate-database-job.yaml @@ -108,7 +108,7 @@ spec: volumeMounts: {{- include "airflow_config_mount" . | nindent 12 }} {{- if .Values.volumeMounts }} -{{ toYaml .Values.volumeMounts | nindent 12 }} +{{- toYaml .Values.volumeMounts | nindent 12 }} {{- end }} {{- if .Values.migrateDatabaseJob.extraVolumeMounts }} {{ toYaml .Values.migrateDatabaseJob.extraVolumeMounts | nindent 12 }} diff --git a/chart/templates/pgbouncer/pgbouncer-deployment.yaml b/chart/templates/pgbouncer/pgbouncer-deployment.yaml index b8e2e0d421ef2..4ddf5c906c2d8 100644 --- a/chart/templates/pgbouncer/pgbouncer-deployment.yaml +++ b/chart/templates/pgbouncer/pgbouncer-deployment.yaml @@ -136,7 +136,7 @@ spec: readOnly: true {{- end }} {{- if .Values.volumeMounts }} -{{ toYaml .Values.volumeMounts | nindent 12 }} +{{- toYaml .Values.volumeMounts | nindent 12 }} {{- end }} {{- if .Values.pgbouncer.extraVolumeMounts }} {{ toYaml .Values.pgbouncer.extraVolumeMounts | indent 12 }} diff --git a/chart/templates/scheduler/scheduler-deployment.yaml b/chart/templates/scheduler/scheduler-deployment.yaml index 6814ada50a70e..1f23cd8bb517c 100644 --- a/chart/templates/scheduler/scheduler-deployment.yaml +++ b/chart/templates/scheduler/scheduler-deployment.yaml @@ -142,7 +142,7 @@ spec: volumeMounts: {{- include "airflow_config_mount" . | nindent 12 }} {{- if .Values.volumeMounts }} -{{ toYaml .Values.volumeMounts | nindent 12 }} +{{- toYaml .Values.volumeMounts | nindent 12 }} {{- end }} {{- if .Values.scheduler.extraVolumeMounts }} {{ toYaml .Values.scheduler.extraVolumeMounts | indent 12 }} @@ -221,7 +221,7 @@ spec: {{- include "airflow_dags_mount" . | nindent 12 }} {{- end }} {{- if .Values.volumeMounts }} -{{ toYaml .Values.volumeMounts | nindent 12 }} +{{- toYaml .Values.volumeMounts | nindent 12 }} {{- end }} {{- if .Values.scheduler.extraVolumeMounts }} {{ toYaml .Values.scheduler.extraVolumeMounts | indent 12 }} @@ -250,7 +250,7 @@ spec: - name: logs mountPath: {{ template "airflow_logs" . }} {{- if .Values.volumeMounts }} -{{ toYaml .Values.volumeMounts | nindent 12 }} +{{- toYaml .Values.volumeMounts | nindent 12 }} {{- end }} {{- if .Values.scheduler.extraVolumeMounts }} {{ toYaml .Values.scheduler.extraVolumeMounts | indent 12 }} diff --git a/chart/templates/triggerer/triggerer-deployment.yaml b/chart/templates/triggerer/triggerer-deployment.yaml index 78310394e91c9..9de82363abe47 100644 --- a/chart/templates/triggerer/triggerer-deployment.yaml +++ b/chart/templates/triggerer/triggerer-deployment.yaml @@ -122,7 +122,7 @@ spec: volumeMounts: {{- include "airflow_config_mount" . | nindent 12 }} {{- if .Values.volumeMounts }} - {{ toYaml .Values.volumeMounts | nindent 12 }} + {{- toYaml .Values.volumeMounts | nindent 12 }} {{- end }} {{- if .Values.triggerer.extraVolumeMounts }} {{ toYaml .Values.triggerer.extraVolumeMounts | nindent 12 }} @@ -161,7 +161,7 @@ spec: {{ toYaml .Values.triggerer.resources | nindent 12 }} volumeMounts: {{- if .Values.volumeMounts }} - {{ toYaml .Values.volumeMounts | nindent 12 }} + {{- toYaml .Values.volumeMounts | nindent 12 }} {{- end }} {{- if .Values.triggerer.extraVolumeMounts }} {{ toYaml .Values.triggerer.extraVolumeMounts | nindent 12 }} diff --git a/chart/templates/webserver/webserver-deployment.yaml b/chart/templates/webserver/webserver-deployment.yaml index 55b065a68eda9..f43d6d5400acb 100644 --- a/chart/templates/webserver/webserver-deployment.yaml +++ b/chart/templates/webserver/webserver-deployment.yaml @@ -134,7 +134,7 @@ spec: volumeMounts: {{- include "airflow_config_mount" . | nindent 12 }} {{- if .Values.volumeMounts }} -{{ toYaml .Values.volumeMounts | nindent 12 }} +{{- toYaml .Values.volumeMounts | nindent 12 }} {{- end }} {{- if .Values.webserver.extraVolumeMounts }} {{ toYaml .Values.webserver.extraVolumeMounts | indent 12 }} @@ -190,7 +190,7 @@ spec: mountPath: {{ template "airflow_logs" . }} {{- end }} {{- if .Values.volumeMounts }} -{{ toYaml .Values.volumeMounts | nindent 12 }} +{{- toYaml .Values.volumeMounts | nindent 12 }} {{- end }} {{- if .Values.webserver.extraVolumeMounts }} {{ toYaml .Values.webserver.extraVolumeMounts | indent 12 }} diff --git a/chart/templates/workers/worker-deployment.yaml b/chart/templates/workers/worker-deployment.yaml index 2b0742bb5f20a..95cc7f137b595 100644 --- a/chart/templates/workers/worker-deployment.yaml +++ b/chart/templates/workers/worker-deployment.yaml @@ -152,7 +152,7 @@ spec: volumeMounts: {{- include "airflow_config_mount" . | nindent 12 }} {{- if .Values.volumeMounts }} -{{ toYaml .Values.volumeMounts | nindent 12 }} +{{- toYaml .Values.volumeMounts | nindent 12 }} {{- end }} {{- if .Values.workers.extraVolumeMounts }} {{ toYaml .Values.workers.extraVolumeMounts | indent 12 }} @@ -209,7 +209,7 @@ spec: containerPort: {{ .Values.ports.workerLogs }} volumeMounts: {{- if .Values.volumeMounts }} -{{ toYaml .Values.volumeMounts | nindent 12 }} +{{- toYaml .Values.volumeMounts | nindent 12 }} {{- end }} {{- if .Values.workers.extraVolumeMounts }} {{ toYaml .Values.workers.extraVolumeMounts | indent 12 }} @@ -271,7 +271,7 @@ spec: - name: logs mountPath: {{ template "airflow_logs" . }} {{- if .Values.volumeMounts }} -{{ toYaml .Values.volumeMounts | nindent 12 }} +{{- toYaml .Values.volumeMounts | nindent 12 }} {{- end }} {{- if .Values.workers.extraVolumeMounts }} {{ toYaml .Values.workers.extraVolumeMounts | indent 12 }} @@ -303,7 +303,7 @@ spec: mountPath: {{ .Values.kerberos.ccacheMountPath | quote }} readOnly: false {{- if .Values.volumeMounts }} -{{ toYaml .Values.volumeMounts | nindent 12 }} +{{- toYaml .Values.volumeMounts | nindent 12 }} {{- end }} {{- if .Values.workers.extraVolumeMounts }} {{ toYaml .Values.workers.extraVolumeMounts | indent 12 }} From 35c59c92fa3b1334ef20805661523e8cb50c9fbc Mon Sep 17 00:00:00 2001 From: "Triomphe Jules, INI-DNA-INF" Date: Wed, 1 Feb 2023 16:22:54 +0100 Subject: [PATCH 3/3] Fix some line breaks in the helm chart renders --- chart/templates/_helpers.yaml | 2 +- chart/templates/webserver/webserver-service.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/chart/templates/_helpers.yaml b/chart/templates/_helpers.yaml index 074ecc7d4914c..80962cbb3b6c5 100644 --- a/chart/templates/_helpers.yaml +++ b/chart/templates/_helpers.yaml @@ -428,7 +428,7 @@ server_tls_key_file = /etc/pgbouncer/server.key {{ define "airflow_dags_mount" -}} - name: dags - mountPath: {{ (printf "%s/dags" .Values.airflowHome) }} + mountPath: {{ (printf "%s/dags" .Values.airflowHome) -}} {{ if .Values.dags.persistence.subPath -}} subPath: {{ .Values.dags.persistence.subPath }} {{- end }} diff --git a/chart/templates/webserver/webserver-service.yaml b/chart/templates/webserver/webserver-service.yaml index 7c5f9958c5f9b..45394c46edc73 100644 --- a/chart/templates/webserver/webserver-service.yaml +++ b/chart/templates/webserver/webserver-service.yaml @@ -42,7 +42,7 @@ spec: component: webserver release: {{ .Release.Name }} ports: - {{ range .Values.webserver.service.ports }} + {{- range .Values.webserver.service.ports }} - {{- range $key, $val := . }} {{ $key }}: {{ tpl (toString $val) $ }}