Weekly environment - Vanilla POC

.github/workflows/unit-test.yml

@@ -119,6 +119,7 @@ jobs:
       - uses: actions/setup-go@v3
         with:
           go-version-file: .go-version
+
       - name: Check out the repo
         uses: actions/checkout@v3
         with:
@@ -156,3 +157,16 @@ jobs:
         run: |
           cfn-lint --version
           cfn-lint -I -t ./deploy/cloudformation/elastic-agent-ec2.yml
+
+  terraform-linter:
+    name: terraform-lint
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v3
+
+      - name: Init Hermit
+        run: ./bin/hermit env -r >> $GITHUB_ENV
+
+      - name: Terraform fmt
+        run: terraform fmt -check -recursive

.github/workflows/weekly-enviroment.yml

@@ -0,0 +1,126 @@
+name: Weekly environment deployment
+
+on:
+  workflow_dispatch:
+    inputs:
+      environment:
+        description: 'Environment to deploy'
+        type: choice
+        options:
+          - weekly environment
+      logLevel:
+        description: 'Log level'
+        required: true
+        default: 'INFO'
+        type: choice
+        options:
+          - TRACE
+          - DEBUG
+          - INFO
+          - WARN
+          - ERROR
+
+env:
+  WORKING_DIR: deploy/weekly-environment
+  SCRIPTS_DIR: deploy/weekly-environment/scripts/benchmarks/kspm_vanilla
+  TF_VAR_ec_api_key: ${{ secrets.WEEKLY_ENVIRONMENT_EC_API_KEY }}
+  TF_VAR_environment: ${{ github.event.inputs.logLevel }}
+  TF_LOG: ${{ github.event.inputs.logLevel }}
+  TF_VAR_stack_version: 8.6.1
+
+jobs:
+  terraform:
+    name: Deploy KSPM cloud environment
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ${{ env.WORKING_DIR }}
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v2
+
+      - name: Init Hermit
+        run: ./bin/hermit env -r >> $GITHUB_ENV
+        working-directory: ./
+
+      - name: Terraform Init
+        run: terraform init -no-color
+
+      - name: Terraform Validate
+        run: terraform validate -no-color
+
+      - name: Deploy Elastic Cloud
+        run: terraform apply --auto-approve
+
+      - name: Set terraform output as env variable
+        run: |
+          echo "KIBANA_URL=$(terraform output kibana_url)" >> $GITHUB_ENV
+
+      - name: Set sensitive terraform output as env variable
+        run: |
+          export ELASTICSEARCH_USERNAME=$(terraform output elasticsearch_username)
+          echo "::add-mask::$ELASTICSEARCH_USERNAME"
+          echo "ELASTICSEARCH_USERNAME=$ELASTICSEARCH_USERNAME" >> $GITHUB_ENV
+
+          export ELASTICSEARCH_PASSWORD=$(terraform output elasticsearch_password)
+          echo "::add-mask::$ELASTICSEARCH_PASSWORD"
+          echo "ELASTICSEARCH_PASSWORD=$ELASTICSEARCH_PASSWORD" >> $GITHUB_ENV
+
+      - name: Install KSPM vanilla integration
+        working-directory: ${{ env.SCRIPTS_DIR }}
+        run: |
+          ./install-kspm-vanilla-integration.sh ${{ env.KIBANA_URL }} ${{ env.ELASTICSEARCH_PASSWORD }}
+
+      - name: Deploy agent on EC2
+        working-directory: ${{ env.SCRIPTS_DIR }}
+        run: |
+          echo -e "${{ secrets.WEEKLY_ENVIRONMENT_EC2_PRIVATE_KEY }}" > weekly-key.pem
+          chmod 600 weekly-key.pem
+          # Copy the manifest file to the EC2 instance
+          scp -o StrictHostKeyChecking=no -v -i weekly-key.pem manifest.yaml "ubuntu@${{ secrets.WEEKLY_ENVIRONMENT_EC2_PUBLIC_IP }}:~/."
+          # Apply the manifest file
+          ssh -o StrictHostKeyChecking=no -v -i weekly-key.pem "ubuntu@${{ secrets.WEEKLY_ENVIRONMENT_EC2_PUBLIC_IP }}" "kubectl apply -f manifest.yaml"
+
+      # Once https://github.com/slackapi/slack-github-action/issues/84 will be resolved we can push the payload to a different file
+      - name: Send custom JSON data to Slack workflow
+        uses: slackapi/slack-github-action@v1.23.0
+        with:
+          payload: |
+            {
+              "text": "A new deployment job has been triggered",
+              "attachments": [
+                {
+                  "color": "#36a64f",
+                  "fields": [
+                    {
+                      "title": "Environment",
+                      "value": "${{ github.event.inputs.environment }}",
+                      "short": true
+                    },
+                    {
+                      "title": "Log level",
+                      "value": "${{ github.event.inputs.logLevel }}",
+                      "short": true
+                    },
+                    {
+                      "title": "Kibana URL",
+                      "value": ${{ env.KIBANA_URL }},
+                      "short": false
+                    },
+                    {
+                      "title": "ElasticSearch username",
+                      "value": ${{ env.ELASTICSEARCH_USERNAME }},
+                      "short": false
+                    },
+                    {
+                      "title": "ElasticSearch password",
+                      "value": ${{ env.ELASTICSEARCH_PASSWORD }},
+                      "short": false
+                    }
+                  ]
+                }
+              ]
+            }
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+          SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK

.gitignore

@@ -41,7 +41,31 @@ bundle.tar.gz
 /deploy/k8s/cloudbeat-ds.yaml
 
 # terraform
-deploy/cloud/.terraform
-deploy/cloud/.terraform.lock.hcl
-deploy/cloud/terraform.tfstate
-deploy/cloud/terraform.tfstate.backup
+.terraform
+.terraform.lock.hcl
+.tfstate.backup
+
+# .tfstate files
+*.tfstate
+*.tfstate.*
+*.tfplan
+
+# Crash log files
+crash.log
+
+# Exclude all .tfvars files, which are likely to contain sentitive data, such as
+# password, private keys, and other secrets. These should not be part of version
+# control as they are data points which are potentially sensitive and subject
+# to change depending on the environment.
+*.tfvars
+
+# Ignore override files as they are usually used to override resources locally and so
+# are not checked in
+override.tf
+override.tf.json
+*_override.tf
+*_override.tf.json
+
+# Ignore CLI configuration files
+.terraformrc
+terraform.rc

deploy/cloud/modules/api/terraform.tf

@@ -1,11 +1,11 @@
 terraform {
   required_providers {
     restapi = {
-      source = "mastercard/restapi"
+      source  = "mastercard/restapi"
       version = "~> 1.18.0"
     }
     http = {
-      source = "hashicorp/http"
+      source  = "hashicorp/http"
       version = "~> 3.2.1"
     }
   }

deploy/cloud/modules/provision-apps/aws-ebs-csi-driver.tf

@@ -1,12 +1,12 @@
 resource "helm_release" "aws_ebs_csi_driver" {
-  chart = "aws-ebs-csi-driver"
-  name = "aws-ebs-csi-driver"
-  namespace = var.namespace
+  chart      = "aws-ebs-csi-driver"
+  name       = "aws-ebs-csi-driver"
+  namespace  = var.namespace
   repository = "https://kubernetes-sigs.github.io/aws-ebs-csi-driver"
 
 
   set {
-    name = "controller.serviceAccount.name"
+    name  = "controller.serviceAccount.name"
     value = "ebs-csi-controller-sa"
   }
 

deploy/cloud/modules/provision-apps/nginx-ingress.tf

@@ -1,18 +1,18 @@
 resource "helm_release" "nginx_ingress" {
-  chart      = "nginx-ingress-controller"
-  name       = "nginx-ingress-controller"
+  chart = "nginx-ingress-controller"
+  name  = "nginx-ingress-controller"
 
   repository = "https://charts.bitnami.com/bitnami"
-  timeout = 600
-  namespace = var.namespace
+  timeout    = 600
+  namespace  = var.namespace
 
   set {
     name  = "service.type"
     value = "ClusterIP"
   }
 
   set {
-    name = "replicaCount"
+    name  = "replicaCount"
     value = var.replica_count
   }
 }

deploy/cloud/modules/provision-apps/terraform.tf

@@ -1,7 +1,7 @@
 terraform {
   required_providers {
     helm = {
-      source = "hashicorp/helm"
+      source  = "hashicorp/helm"
       version = ">=2.8.0"
     }
 

deploy/cloud/modules/provision-apps/variables.tf

@@ -1,9 +1,9 @@
 variable "namespace" {
-  type = string
+  type    = string
   default = "default"
 }
 
 variable "replica_count" {
-  type = string
+  type    = string
   default = "2"
 }

deploy/weekly-environment/main.tf

@@ -0,0 +1,25 @@
+provider "ec" {
+  apikey = var.ec_api_key
+}
+
+module "ec_deployment" {
+  source = "github.com/elastic/apm-server/testing/infra/terraform/modules/ec_deployment"
+
+  region        = var.ess_region
+  stack_version = var.stack_version
+
+  deployment_template    = var.deployment_template
+  deployment_name_prefix = "${var.deployment_name_prefix}-${formatdate("MMM DD, YYYY", timestamp())}"
+
+  integrations_server = true
+
+  elasticsearch_size       = var.elasticsearch_size
+  elasticsearch_zone_count = var.elasticsearch_zone_count
+
+  docker_image = var.docker_image_override
+  docker_image_tag_override = {
+    "elasticsearch" : "",
+    "kibana" : "",
+    "apm" : ""
+  }
+}

deploy/weekly-environment/outputs.tf

@@ -0,0 +1,26 @@
+output "elasticsearch_url" {
+  value       = module.ec_deployment.elasticsearch_url
+  description = "The secure Elasticsearch URL"
+}
+
+output "elasticsearch_username" {
+  value       = module.ec_deployment.elasticsearch_username
+  description = "The Elasticsearch username"
+  sensitive   = true
+}
+
+output "elasticsearch_password" {
+  value       = module.ec_deployment.elasticsearch_password
+  description = "The Elasticsearch password"
+  sensitive   = true
+}
+
+output "kibana_url" {
+  value       = module.ec_deployment.kibana_url
+  description = "The secure Kibana URL"
+}
+
+output "admin_console_url" {
+  value       = module.ec_deployment.admin_console_url
+  description = "The admin console URL"
+}

deploy/weekly-environment/scripts/benchmarks/kspm_vanilla/data/agent_policy_vanilla.json

@@ -0,0 +1,8 @@
+{
+    "name": "weekly-environment-vanilla-policy",
+    "namespace": "default",
+    "monitoring_enabled": [
+        "logs",
+        "metrics"
+    ]
+}