From a99b73a770b2c90209d19a09dc34792a44b7b438 Mon Sep 17 00:00:00 2001 From: "Tobias.Mikula" Date: Wed, 15 Oct 2025 13:54:31 +0200 Subject: [PATCH 01/10] Terraform Trivy and TFLint CI check implementation. --- .github/workflows/check_terraform.yml | 80 +++++++++++++++++++++++++++ .gitignore | 7 ++- 2 files changed, 86 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/check_terraform.yml diff --git a/.github/workflows/check_terraform.yml b/.github/workflows/check_terraform.yml new file mode 100644 index 0000000..20e7890 --- /dev/null +++ b/.github/workflows/check_terraform.yml @@ -0,0 +1,80 @@ +name: Static Terraform Check + +on: + pull_request: + types: [ opened, synchronize, reopened, edited, labeled, unlabeled ] + paths: + - terraform/** + push: + branches: [ master ] + paths: + - terraform/** + workflow_dispatch: + +concurrency: + group: terraform-static-check-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + security-events: write + +jobs: + trivy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup Trivy + uses: aquasecurity/setup-trivy@v0 + + - name: Trivy config scan + id: scan + working-directory: terraform + run: | + set +e + trivy config . \ + --format sarif \ + --output $GITHUB_WORKSPACE/trivy.sarif \ + --severity HIGH,CRITICAL \ + --exit-code 1 + code=$? + echo "exit_code=$code" >> "$GITHUB_OUTPUT" + exit 0 + + - name: Upload Trivy SARIF file + uses: github/codeql-action/upload-sarif@v3 + with: + sarif_file: ${{ github.workspace }}/trivy.sarif + + - name: Enforce failure on HIGH/CRITICAL + if: steps.scan.outputs.exit_code != '0' + run: | + echo "Trivy found HIGH/CRITICAL issues in ${{ github.workspace }}" + exit 1 + tflint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup TFLint + uses: terraform-linters/setup-tflint@v6 + with: + tflint_version: latest + + - name: TFLint init + working-directory: terraform + run: tflint --init + + - name: Run TFLint + working-directory: terraform + run: tflint -f sarif > "$GITHUB_WORKSPACE/terraform_tflint.sarif" || true + + - name: Upload TFLint SARIF file + uses: github/codeql-action/upload-sarif@v3 + with: + sarif_file: ${{ github.workspace }}/terraform_tflint.sarif diff --git a/.gitignore b/.gitignore index 862270d..4c9edc4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,14 @@ .ipynb_checkpoints .venv __pycache__ +/.idea/ /dependencies /lambda_function.zip + +# Terraform files /terraform/*.tfvars /terraform/*.tfstate* /terraform/.terraform* -/.idea/ + +# Terraform Plan output files +*.sarif From 65d8ed7bfcbcc5ceffb0dd7e61fc97805ff4e455 Mon Sep 17 00:00:00 2001 From: "Tobias.Mikula" Date: Wed, 15 Oct 2025 14:31:04 +0200 Subject: [PATCH 02/10] Comment implementation. --- .github/workflows/check_terraform.yml | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/.github/workflows/check_terraform.yml b/.github/workflows/check_terraform.yml index 20e7890..cc13269 100644 --- a/.github/workflows/check_terraform.yml +++ b/.github/workflows/check_terraform.yml @@ -37,7 +37,7 @@ jobs: set +e trivy config . \ --format sarif \ - --output $GITHUB_WORKSPACE/trivy.sarif \ + --output $GITHUB_WORKSPACE/terraform_trivy.sarif \ --severity HIGH,CRITICAL \ --exit-code 1 code=$? @@ -47,7 +47,7 @@ jobs: - name: Upload Trivy SARIF file uses: github/codeql-action/upload-sarif@v3 with: - sarif_file: ${{ github.workspace }}/trivy.sarif + sarif_file: ${{ github.workspace }}/terraform_trivy.sarif - name: Enforce failure on HIGH/CRITICAL if: steps.scan.outputs.exit_code != '0' @@ -71,10 +71,22 @@ jobs: run: tflint --init - name: Run TFLint + id: lint working-directory: terraform - run: tflint -f sarif > "$GITHUB_WORKSPACE/terraform_tflint.sarif" || true + run: | + set +e + tflint -f sarif > "$GITHUB_WORKSPACE/terraform_tflint.sarif" + code=$? + echo "exit_code=$code" >> "$GITHUB_OUTPUT" + exit 0 - name: Upload TFLint SARIF file uses: github/codeql-action/upload-sarif@v3 with: sarif_file: ${{ github.workspace }}/terraform_tflint.sarif + + - name: Enforce failure on TFLint findings + if: steps.lint.outputs.exit_code != '0' + run: | + echo "TFLint reported issues" + exit 1 From 994681209df90fb4a60991d7ab6efd8e99b4e1e0 Mon Sep 17 00:00:00 2001 From: "Tobias.Mikula" Date: Wed, 15 Oct 2025 14:42:09 +0200 Subject: [PATCH 03/10] Pseudo change to see the GH workflow behaviour. --- terraform/lambda.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/lambda.tf b/terraform/lambda.tf index 0eeedfa..5344439 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -22,7 +22,7 @@ resource "aws_lambda_function" "event_gate_lambda" { role = var.lambda_role_arn architectures = ["x86_64"] timeout = 60 - runtime = "python3.11" + runtime = "python3.13" package_type = var.lambda_package_type s3_bucket = var.lambda_package_type == "Zip" ? var.lambda_src_s3_bucket : null From 0b91fe7fc577fdd27aef8dce3754522e09713936 Mon Sep 17 00:00:00 2001 From: "Tobias.Mikula" Date: Wed, 15 Oct 2025 14:56:01 +0200 Subject: [PATCH 04/10] Workflow bug fix. --- .github/workflows/check_terraform.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/check_terraform.yml b/.github/workflows/check_terraform.yml index cc13269..c503c5a 100644 --- a/.github/workflows/check_terraform.yml +++ b/.github/workflows/check_terraform.yml @@ -2,7 +2,7 @@ name: Static Terraform Check on: pull_request: - types: [ opened, synchronize, reopened, edited, labeled, unlabeled ] + types: [ opened, synchronize, reopened ] paths: - terraform/** push: @@ -28,7 +28,7 @@ jobs: fetch-depth: 0 - name: Setup Trivy - uses: aquasecurity/setup-trivy@v0 + uses: aquasecurity/setup-trivy@v0.2.4 - name: Trivy config scan id: scan From 06d779634e3f191db766a2bfbec53affbcc29313 Mon Sep 17 00:00:00 2001 From: "Tobias.Mikula" Date: Wed, 15 Oct 2025 15:06:05 +0200 Subject: [PATCH 05/10] Pseudo change. --- terraform/lambda.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/lambda.tf b/terraform/lambda.tf index 5344439..45587b1 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -22,7 +22,7 @@ resource "aws_lambda_function" "event_gate_lambda" { role = var.lambda_role_arn architectures = ["x86_64"] timeout = 60 - runtime = "python3.13" + runtime = "python3.12" package_type = var.lambda_package_type s3_bucket = var.lambda_package_type == "Zip" ? var.lambda_src_s3_bucket : null From 9c1d12389d86efaae3ae01acb1cc7f0fd9e65dc5 Mon Sep 17 00:00:00 2001 From: "Tobias.Mikula" Date: Wed, 15 Oct 2025 15:07:31 +0200 Subject: [PATCH 06/10] Pseudo change. --- terraform/lambda.tf | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/terraform/lambda.tf b/terraform/lambda.tf index 45587b1..a747da5 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -8,9 +8,10 @@ resource "aws_security_group" "event_gate_sg" { resource "aws_vpc_security_group_egress_rule" "allow_all_traffic_ipv4" { security_group_id = aws_security_group.event_gate_sg.id cidr_ipv4 = "0.0.0.0/0" - ip_protocol = "-1" + ip_protocol = "-2" } + data "aws_s3_object" "event_gate_lambda_zip" { count = var.lambda_package_type == "Zip" ? 1 : 0 bucket = var.lambda_src_s3_bucket @@ -22,7 +23,7 @@ resource "aws_lambda_function" "event_gate_lambda" { role = var.lambda_role_arn architectures = ["x86_64"] timeout = 60 - runtime = "python3.12" + runtime = "python3.13" package_type = var.lambda_package_type s3_bucket = var.lambda_package_type == "Zip" ? var.lambda_src_s3_bucket : null From 42873784a078347e4aa5933ba43d73b9ea0b8dd7 Mon Sep 17 00:00:00 2001 From: "Tobias.Mikula" Date: Thu, 16 Oct 2025 11:29:52 +0200 Subject: [PATCH 07/10] Documentation and setting the workflow. --- .github/CODEOWNERS | 2 +- .github/workflows/check_terraform.yml | 6 ++-- DEVELOPER.md | 43 +++++++++++++++++++++++++-- README.md | 20 +++++++------ terraform/lambda.tf | 3 +- 5 files changed, 57 insertions(+), 17 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 8423567..a275fc1 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1 +1 @@ -* @ABMC831 @Zejnilovic @oto-macenauer-absa @petr-pokorny-absa +* @ABMC831 @Zejnilovic @oto-macenauer-absa @petr-pokorny-absa *tmikula-dev diff --git a/.github/workflows/check_terraform.yml b/.github/workflows/check_terraform.yml index c503c5a..0d641d7 100644 --- a/.github/workflows/check_terraform.yml +++ b/.github/workflows/check_terraform.yml @@ -52,7 +52,7 @@ jobs: - name: Enforce failure on HIGH/CRITICAL if: steps.scan.outputs.exit_code != '0' run: | - echo "Trivy found HIGH/CRITICAL issues in ${{ github.workspace }}" + echo "Trivy detected HIGH/CRITICAL findings. View details in PR code scanning annotations or in Security tab > Code scanning alerts (filter for pr:pr_number)." exit 1 tflint: runs-on: ubuntu-latest @@ -75,7 +75,7 @@ jobs: working-directory: terraform run: | set +e - tflint -f sarif > "$GITHUB_WORKSPACE/terraform_tflint.sarif" + tflint --minimum-failure-severity=error -f sarif > "$GITHUB_WORKSPACE/terraform_tflint.sarif" code=$? echo "exit_code=$code" >> "$GITHUB_OUTPUT" exit 0 @@ -88,5 +88,5 @@ jobs: - name: Enforce failure on TFLint findings if: steps.lint.outputs.exit_code != '0' run: | - echo "TFLint reported issues" + echo "TFLint reported error issues. View details in PR code scanning annotations or in Security tab > Code scanning alerts (filter for pr:pr_number)." exit 1 diff --git a/DEVELOPER.md b/DEVELOPER.md index df23aca..67093be 100644 --- a/DEVELOPER.md +++ b/DEVELOPER.md @@ -2,10 +2,13 @@ - [Get Started](#get-started) - [Set Up Python Environment](#set-up-python-environment) -- [Run Static Code Analysis](#running-static-code-analysis) +- [Run Pylint Tool Locally](#run-pylint-tool-locally) - [Run Black Tool Locally](#run-black-tool-locally) - [Run mypy Tool Locally](#run-mypy-tool-locally) +- [Run TFLint Tool Locally](#run-tflint-tool-locally) +- [Run Trivy Tool Locally](#run-trivy-tool-locally) - [Run Unit Test](#running-unit-test) +- [Code Coverage](#code-coverage) ## Get Started @@ -23,7 +26,7 @@ source .venv/bin/activate pip install -r requirements.txt ``` -## Running Static Code Analysis +## Run Pylint Tool Locally This project uses the Pylint tool for static code analysis. Pylint analyzes your code without actually running it. It checks for errors, enforces coding standards, looks for code smells, etc. @@ -97,6 +100,42 @@ Example: mypy src/writer_kafka.py ``` +## Run TFLint Tool Locally + +This project uses the [TFLint](https://github.com/terraform-linters/tflint) tool for static analysis of Terraform code. +We are forcing to eliminate **all** errors reported by TFLint. Any detected warnings and notices should be corrected as well as a best practice. + +- Find possible errors (like invalid instance types) for Major Cloud providers (AWS/Azure/GCP). +- Warn about deprecated syntax, unused declarations. +- Enforce best practices, naming conventions. + +> For installation instructions, please refer to the [following link.](https://github.com/terraform-linters/tflint) + +### Run TFLint + +For running TFLint you need to be in the `terraform/` directory. From the root file run the following commands: +```shell +cd terraform +tflint --init +tflint +cd .. +``` + +## Run Trivy Tool Locally + +This project uses the [Trivy](https://trivy.dev/latest/) tool to scan Infrastructure as Code (terraform files) for security issues and misconfigurations. +It is an open‑source security scanner maintained by Aqua Security (AquaSec). + +> For installation instructions, please refer to the [following link.](https://trivy.dev/latest/getting-started/installation/) + +### Run Trivy + +For running Trivy tool locally run the following command from the root file: +```shell +trivy config terraform/ # Default table output (all severities) +trivy config --severity HIGH,CRITICAL terraform/ # Show only HIGH and CRITICAL severities +``` + ## Running Unit Test Unit tests are written using pytest. To run the tests, use the following command: diff --git a/README.md b/README.md index b404b26..42286c1 100644 --- a/README.md +++ b/README.md @@ -124,15 +124,17 @@ Use when Kafka access needs Kerberos / SASL_SSL or custom `librdkafka` build. ## Local Development & Testing -| Purpose | Relative link | -|---------|---------------| -| Get started | [Get Started](./DEVELOPER.md#get-started) | -| Python environment setup | [Set Up Python Environment](./DEVELOPER.md#set-up-python-environment) | -| Static code analysis (Pylint) | [Running Static Code Analysis](./DEVELOPER.md#running-static-code-analysis) | -| Formatting (Black) | [Run Black Tool Locally](./DEVELOPER.md#run-black-tool-locally) | -| Type checking (mypy) | [Run mypy Tool Locally](./DEVELOPER.md#run-mypy-tool-locally) | -| Unit tests | [Running Unit Test](./DEVELOPER.md#running-unit-test) | -| Code coverage | [Code Coverage](./DEVELOPER.md#code-coverage) | +| Purpose | Relative link | +|------------------------------------|-----------------------------------------------------------------------| +| Get started | [Get Started](./DEVELOPER.md#get-started) | +| Python environment setup | [Set Up Python Environment](./DEVELOPER.md#set-up-python-environment) | +| Static code analysis (Pylint) | [Run Pylint Tool Locally](./DEVELOPER.md#run-pylint-tool-locally) | +| Formatting (Black) | [Run Black Tool Locally](./DEVELOPER.md#run-black-tool-locally) | +| Type checking (mypy) | [Run mypy Tool Locally](./DEVELOPER.md#run-mypy-tool-locally) | +| Terraform Linter (TFLint) | [Run TFLint Tool Locally](./DEVELOPER.md#run-tflint-tool-locally) | +| Terraform Security Scanner (Trivy) | [Run Trivy Tool Locally](./DEVELOPER.md#run-trivy-tool-locally) | +| Unit tests | [Running Unit Test](./DEVELOPER.md#running-unit-test) | +| Code coverage | [Code Coverage](./DEVELOPER.md#code-coverage) | ## Security & Authorization - JWT tokens must be RS256 signed; the public key is fetched at cold start from `token_public_key_url` (DER base64 inside JSON `{ "key": "..." }`). diff --git a/terraform/lambda.tf b/terraform/lambda.tf index a747da5..142089f 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -11,7 +11,6 @@ resource "aws_vpc_security_group_egress_rule" "allow_all_traffic_ipv4" { ip_protocol = "-2" } - data "aws_s3_object" "event_gate_lambda_zip" { count = var.lambda_package_type == "Zip" ? 1 : 0 bucket = var.lambda_src_s3_bucket @@ -23,7 +22,7 @@ resource "aws_lambda_function" "event_gate_lambda" { role = var.lambda_role_arn architectures = ["x86_64"] timeout = 60 - runtime = "python3.13" + runtime = "python3.11" package_type = var.lambda_package_type s3_bucket = var.lambda_package_type == "Zip" ? var.lambda_src_s3_bucket : null From e7763b5624c850ec0c5e186c1941f95ede0141af Mon Sep 17 00:00:00 2001 From: "Tobias.Mikula" Date: Thu, 16 Oct 2025 11:45:24 +0200 Subject: [PATCH 08/10] Bug fix. --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index a275fc1..939832a 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1 +1 @@ -* @ABMC831 @Zejnilovic @oto-macenauer-absa @petr-pokorny-absa *tmikula-dev +* @ABMC831 @Zejnilovic @oto-macenauer-absa @petr-pokorny-absa @tmikula-dev From c1a529c056d0eec57913ce9bab2943877773307b Mon Sep 17 00:00:00 2001 From: "Tobias.Mikula" Date: Thu, 16 Oct 2025 11:47:06 +0200 Subject: [PATCH 09/10] Bug fix. --- terraform/lambda.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/lambda.tf b/terraform/lambda.tf index 0d95675..5344439 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -8,7 +8,7 @@ resource "aws_security_group" "event_gate_sg" { resource "aws_vpc_security_group_egress_rule" "allow_all_traffic_ipv4" { security_group_id = aws_security_group.event_gate_sg.id cidr_ipv4 = "0.0.0.0/0" - ip_protocol = "-2" + ip_protocol = "-1" } data "aws_s3_object" "event_gate_lambda_zip" { From eac69663dc89e13b065c12409ec4eb65fa974165 Mon Sep 17 00:00:00 2001 From: "Tobias.Mikula" Date: Thu, 16 Oct 2025 13:51:17 +0200 Subject: [PATCH 10/10] Typo fix. --- .github/workflows/check_terraform.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check_terraform.yml b/.github/workflows/check_terraform.yml index 0d641d7..6d05444 100644 --- a/.github/workflows/check_terraform.yml +++ b/.github/workflows/check_terraform.yml @@ -1,4 +1,4 @@ -name: Static Terraform Check +name: Static Terraform Check on: pull_request: