Add new containers for execution of scripts and directly consume them in generate diffs

.ci/containers/bash-plus/Dockerfile

@@ -0,0 +1,5 @@
+FROM alpine/git
+
+RUN apk update && apk upgrade && \
+    apk add --no-cache bash jq curl && \
+    rm -rf /var/cache/apk/*

.ci/containers/build-environment/Dockerfile

@@ -0,0 +1,53 @@
+# Stage 1: Building Go dependencies
+FROM golang:1.19-bullseye AS builder
+
+# Set working directory
+WORKDIR /app
+
+# Download go.mod and go.sum from your repository
+ADD "https://github.com/GoogleCloudPlatform/magic-modules/main/tpgtools/go.mod" go.mod
+ADD "https://github.com/GoogleCloudPlatform/magic-modules/main/tpgtools/go.sum" go.sum
+
+# Install the go dependencies
+RUN go mod download
+
+# Stage 2: Creating the final imag
+FROM ruby:3.1-bullseye
+
+# golang
+COPY --from=golang:1.19-bullseye /usr/local/go /usr/local/go
+ENV GOPATH /go
+ENV PATH /usr/local/go/bin:$PATH
+ENV PATH $GOPATH/bin:$PATH
+RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 1777 "$GOPATH"
+WORKDIR $GOPATH
+
+# terraform binary used by tfv/tgc
+COPY --from=hashicorp/terraform:1.4.2 /bin/terraform /bin/terraform
+
+SHELL ["/bin/bash", "-c"]
+
+ENV GO111MODULE "on"
+ENV LC_ALL=C.UTF-8
+ENV LANG=C.UTF-8
+
+RUN apt-get update && apt-get install -y --no-install-recommends git openssh-client apt-transport-https ca-certificates curl netbase wget gcc make jq libjq1
+
+RUN git config --global user.name "Modular Magician"
+RUN git config --global user.email "magic-modules@google.com"
+
+# Set up Github SSH cloning.
+RUN ssh-keyscan github.com >> /known_hosts
+RUN echo "UserKnownHostsFile /known_hosts" >> /etc/ssh/ssh_config
+
+RUN go install golang.org/x/tools/cmd/goimports@d088b475e3360caabc032aaee1dc66351d4e729a
+RUN go install github.com/github/hub@v2.11.2+incompatible
+
+ADD "https://github.com/GoogleCloudPlatform/magic-modules/main/mmv1/Gemfile" Gemfile
+ADD "https://github.com/GoogleCloudPlatform/magic-modules/main/mmv1/Gemfile.lock" Gemfile.lock
+RUN bundle install
+RUN rm Gemfile Gemfile.lock
+
+# Copy Go dependencies from builder stage
+COPY --from=builder /go/pkg /go/pkg
+

.ci/containers/go-plus/Dockerfile

@@ -0,0 +1,32 @@
+from golang:1.19-bullseye as resource
+SHELL ["/bin/bash", "-c"]
+# Set up Github SSH cloning.
+RUN ssh-keyscan github.com >> /known_hosts
+RUN echo "UserKnownHostsFile /known_hosts" >> /etc/ssh/ssh_config
+
+RUN apt-get update && \
+    apt-get install -y git jq unzip parallel curl && \
+    echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && \
+    curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key --keyring /usr/share/keyrings/cloud.google.gpg  add - && \
+    apt-get update -y && \
+    apt-get install google-cloud-sdk -y && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN wget https://releases.hashicorp.com/terraform/1.2.5/terraform_1.2.5_linux_amd64.zip \
+    && unzip terraform_1.2.5_linux_amd64.zip \
+    && rm terraform_1.2.5_linux_amd64.zip \
+    && mv ./terraform /bin/terraform
+
+### the following version of terraform are required for
+### terraform-validator-tester-integration
+RUN wget https://releases.hashicorp.com/terraform/0.13.7/terraform_0.13.7_linux_amd64.zip \
+    && unzip terraform_0.13.7_linux_amd64.zip \
+    && rm terraform_0.13.7_linux_amd64.zip \
+    && mkdir /terraform \
+    && mv terraform /terraform/0.13.7
+
+RUN wget https://releases.hashicorp.com/terraform/0.12.31/terraform_0.12.31_linux_amd64.zip \
+    && unzip terraform_0.12.31_linux_amd64.zip \
+    && rm terraform_0.12.31_linux_amd64.zip \
+    && mv terraform /terraform/0.12.31

.ci/gcb-generate-diffs-new.yml

@@ -64,17 +64,19 @@ steps:
           - --no-ff
           - head/$_HEAD_BRANCH
 
-    - name: 'gcr.io/graphite-docker-images/pre-build-validator'
+    - name: 'gcr.io/graphite-docker-images/bash-plus'
       id: pre-build-validate
+      entrypoint: '/workspace/.ci/scripts/bash-plus/pre-build-validator/validate.sh'
       secretEnv: ["GITHUB_TOKEN"]
       waitFor: ["merged"]
       env:
         - COMMIT_SHA=$COMMIT_SHA
         - PR_NUMBER=$_PR_NUMBER
 
-    - name: 'gcr.io/graphite-docker-images/downstream-builder'
-      secretEnv: ["GITHUB_TOKEN"]
+    - name: 'gcr.io/graphite-docker-images/build-environment'
+      entrypoint: '/workspace/.ci/scripts/build-environment/downstream-builder/generate_downstream.sh'
       id: tpg-head
+      secretEnv: ["GITHUB_TOKEN"]
       waitFor: ["merged"]
       env:
         - BASE_BRANCH=$_BASE_BRANCH
@@ -84,7 +86,8 @@ steps:
           - 'ga'
           - $_PR_NUMBER
 
-    - name: 'gcr.io/graphite-docker-images/downstream-builder'
+    - name: 'gcr.io/graphite-docker-images/build-environment'
+      entrypoint: '/workspace/.ci/scripts/build-environment/downstream-builder/generate_downstream.sh'
       id: tpg-base
       secretEnv: ["GITHUB_TOKEN"]
       waitFor: ["merged"]
@@ -96,7 +99,8 @@ steps:
           - 'ga'
           - $_PR_NUMBER
 
-    - name: 'gcr.io/graphite-docker-images/downstream-builder'
+    - name: 'gcr.io/graphite-docker-images/build-environment'
+      entrypoint: '/workspace/.ci/scripts/build-environment/downstream-builder/generate_downstream.sh'
       secretEnv: ["GITHUB_TOKEN"]
       id: tpgb-head
       waitFor: ["merged"]
@@ -108,7 +112,8 @@ steps:
           - 'beta'
           - $_PR_NUMBER
 
-    - name: 'gcr.io/graphite-docker-images/downstream-builder'
+    - name: 'gcr.io/graphite-docker-images/build-environment'
+      entrypoint: '/workspace/.ci/scripts/build-environment/downstream-builder/generate_downstream.sh'
       id: tpgb-base
       secretEnv: ["GITHUB_TOKEN"]
       waitFor: ["merged"]
@@ -120,7 +125,8 @@ steps:
           - 'beta'
           - $_PR_NUMBER
 
-    - name: 'gcr.io/graphite-docker-images/downstream-builder'
+    - name: 'gcr.io/graphite-docker-images/build-environment'
+      entrypoint: '/workspace/.ci/scripts/build-environment/downstream-builder/generate_downstream.sh'
       id: tgc-head
       secretEnv: ["GITHUB_TOKEN"]
       waitFor: ["merged", "tpg-head"]
@@ -132,7 +138,8 @@ steps:
           - 'beta'
           - $_PR_NUMBER
 
-    - name: 'gcr.io/graphite-docker-images/downstream-builder'
+    - name: 'gcr.io/graphite-docker-images/build-environment'
+      entrypoint: '/workspace/.ci/scripts/build-environment/downstream-builder/generate_downstream.sh'
       id: tgc-base
       secretEnv: ["GITHUB_TOKEN"]
       waitFor: ["merged", "tpg-base"]
@@ -144,7 +151,9 @@ steps:
           - 'beta'
           - $_PR_NUMBER
 
-    - name: 'gcr.io/graphite-docker-images/downstream-builder'
+    - name: 'gcr.io/graphite-docker-images/build-environment'
+      entrypoint: '/workspace/.ci/scripts/build-environment/downstream-builder/generate_downstream.sh'
+      id: tf-oics-head
       secretEnv: ["GITHUB_TOKEN"]
       waitFor: ["merged"]
       env:
@@ -155,7 +164,9 @@ steps:
           - 'beta'
           - $_PR_NUMBER
 
-    - name: 'gcr.io/graphite-docker-images/downstream-builder'
+    - name: 'gcr.io/graphite-docker-images/build-environment'
+      entrypoint: '/workspace/.ci/scripts/build-environment/downstream-builder/generate_downstream.sh'
+      id: tf-oics-base
       secretEnv: ["GITHUB_TOKEN"]
       waitFor: ["merged"]
       env:
@@ -166,7 +177,8 @@ steps:
           - 'beta'
           - $_PR_NUMBER
 
-    - name: 'gcr.io/graphite-docker-images/github-differ'
+    - name: 'gcr.io/graphite-docker-images/go-plus'
+      entrypoint: '/workspace/.ci/scripts/go-plus/github-differ/generate_comment.sh'
       id: diff
       secretEnv: ["GITHUB_TOKEN"]
       env:
@@ -178,7 +190,8 @@ steps:
       args:
           - $_PR_NUMBER
 
-    - name: 'gcr.io/graphite-docker-images/terraform-validator-tester'
+    - name: 'gcr.io/graphite-docker-images/go-plus'
+      entrypoint: '/workspace/.ci/scripts/go-plus/terraform-validator-tester/test_terraform_validator.sh'
       id: tgc-test
       allowFailure: true
       secretEnv: ["GITHUB_TOKEN"]
@@ -196,8 +209,9 @@ steps:
           - "17"  # Build step
           - terraform-google-conversion
 
-    - name: 'gcr.io/graphite-docker-images/terraform-validator-tester-integration'
+    - name: 'gcr.io/graphite-docker-images/go-plus'
       id: tgc-test-integration-0.12.31
+      entrypoint: '/workspace/.ci/scripts/go-plus/terraform-validator-tester-integration/test_terraform_validator_integration.sh'
       allowFailure: true
       secretEnv: ["GITHUB_TOKEN"]
       waitFor: ["tgc-head", "tgc-base"]
@@ -215,8 +229,9 @@ steps:
           - "18"  # Build step
           - terraform-google-conversion
 
-    - name: 'gcr.io/graphite-docker-images/terraform-validator-tester-integration'
+    - name: 'gcr.io/graphite-docker-images/go-plus'
       id: tgc-test-integration-0.13.7
+      entrypoint: '/workspace/.ci/scripts/go-plus/terraform-validator-tester-integration/test_terraform_validator_integration.sh'
       allowFailure: true
       secretEnv: ["GITHUB_TOKEN"]
       waitFor: ["tgc-head", "tgc-base"]
@@ -234,25 +249,19 @@ steps:
           - "19"  # Build step
           - terraform-google-conversion
 
-    - name: 'gcr.io/graphite-docker-images/terraform-tester'
+    - name: 'gcr.io/graphite-docker-images/bash-plus'
       id: tpgb-test
+      entrypoint: '/workspace/.ci/scripts/bash-plus/terraform-tester/test_terraform.sh'
       secretEnv: ["GITHUB_TOKEN"]
       waitFor: ["tpgb-head", "tpgb-base"]
-      args:
-          - 'beta'                            # remove after 07/2023
-          - $_PR_NUMBER                       # remove after 07/2023
-          - $COMMIT_SHA                       # remove after 07/2023
-          - $BUILD_ID                         # remove after 07/2023
-          - $PROJECT_ID                       # remove after 07/2023
-          - GoogleCloudPlatform/magic-modules # remove after 07/2023
-          - "20"                              # remove after 07/2023
       env:
         - VERSION=beta
         - COMMIT_SHA=$COMMIT_SHA
         - PR_NUMBER=$_PR_NUMBER
 
-    - name: 'gcr.io/graphite-docker-images/terraform-tester'
+    - name: 'gcr.io/graphite-docker-images/bash-plus'
       id: tpg-test
+      entrypoint: '/workspace/.ci/scripts/bash-plus/terraform-tester/test_terraform.sh'
       secretEnv: ["GITHUB_TOKEN"]
       waitFor: ["tpg-head", "tpg-base"]
       args:
@@ -268,8 +277,9 @@ steps:
         - COMMIT_SHA=$COMMIT_SHA
         - PR_NUMBER=$_PR_NUMBER
 
-    - name: 'gcr.io/graphite-docker-images/gcb-terraform-vcr-tester'
+    - name: 'gcr.io/graphite-docker-images/go-plus'
       id: gcb-tpg-vcr-test
+      entrypoint: '/workspace/.ci/scripts/go-plus/gcb-terraform-vcr-tester/test_terraform_vcr.sh'
       secretEnv: ["GITHUB_TOKEN", "GOOGLE_BILLING_ACCOUNT", "GOOGLE_CUST_ID", "GOOGLE_FIRESTORE_PROJECT", "GOOGLE_IDENTITY_USER", "GOOGLE_MASTER_BILLING_ACCOUNT", "GOOGLE_ORG", "GOOGLE_ORG_2", "GOOGLE_ORG_DOMAIN", "GOOGLE_PROJECT", "GOOGLE_PROJECT_NUMBER", "GOOGLE_SERVICE_ACCOUNT", "SA_KEY", "GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION"]
       waitFor: ["diff"]
       env:

.ci/scripts/bash-plus/downstream-waiter/wait_for_commit.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+set -e
+if [ $# -lt 3 ]; then
+    echo "Usage: $0 (sync-branch) (base-branch) (sha)"
+    exit 1
+fi
+
+SYNC_BRANCH_PREFIX=$1
+BASE_BRANCH=$2
+SHA=$3
+
+if [ "$BASE_BRANCH" == "main" ]; then
+    SYNC_BRANCH=$SYNC_BRANCH_PREFIX
+else
+    SYNC_BRANCH=$SYNC_BRANCH_PREFIX-$BASE_BRANCH
+fi
+
+echo "SYNC_BRANCH: $SYNC_BRANCH"
+
+if git merge-base --is-ancestor $SHA origin/$SYNC_BRANCH; then
+    echo "Found $SHA in history of $SYNC_BRANCH - dying to avoid double-generating that commit."
+    exit 1
+fi
+
+while true; do
+    commits="$(git log --pretty=%H origin/$SYNC_BRANCH..origin/$BASE_BRANCH | tail -n 1)"
+    if [ "$commits" == "$SHA" ]; then
+        break
+    else
+        echo "git log says waiting on: $commits"
+        echo "command says waiting on $SHA"
+        git fetch origin $SYNC_BRANCH
+    fi
+    sleep 5
+done

.ci/scripts/bash-plus/pre-build-validator/validate.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+set -e
+
+gh_repo=magic-modules
+
+post_body=$(jq -n \
+  --arg owner "GoogleCloudPlatform" \
+  --arg repo "$gh_repo" \
+  --arg sha "$COMMIT_SHA" \
+  '{
+    ref: "main",
+    inputs: {
+      owner: $owner,
+      repo: $repo,
+      sha: $sha,
+    }
+  }')
+
+curl \
+  -X POST \
+  -u "modular-magician:$GITHUB_TOKEN" \
+  -H "Accept: application/vnd.github.v3+json" \
+  "https://api.github.com/repos/GoogleCloudPlatform/magic-modules/actions/workflows/pre-build-validation.yml/dispatches" \
+  -d "$post_body"

.ci/scripts/bash-plus/terraform-tester/test_terraform.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+
+set -e
+
+version=${VERSION:-$1} # fallback for old variable based declaration
+pr_number=${PR_NUMBER:-$2}
+mm_commit_sha=${COMMIT_SHA:-$3}
+github_username=modular-magician
+
+if [ "$version" == "ga" ]; then
+    gh_repo=terraform-provider-google
+elif [ "$version" == "beta" ]; then
+    gh_repo=terraform-provider-google-beta
+else
+    echo "no repo, dying."
+    exit 1
+fi
+
+new_branch="auto-pr-$pr_number"
+
+post_body=$(jq -n \
+  --arg owner "$github_username" \
+  --arg branch "$new_branch" \
+  --arg repo "$gh_repo" \
+  --arg sha "$mm_commit_sha" \
+  '{
+    ref: "main",
+    inputs: {
+      owner: $owner,
+      repo: $repo,
+      branch: $branch,
+      sha: $sha
+    }
+  }')
+
+curl \
+  -X POST \
+  -u "$github_username:$GITHUB_TOKEN" \
+  -H "Accept: application/vnd.github.v3+json" \
+  "https://api.github.com/repos/GoogleCloudPlatform/magic-modules/actions/workflows/test-tpg.yml/dispatches" \
+  -d "$post_body"