Run conformance tests on s390x using kubetest2

config/jobs/kubernetes/cloud-provider-ibmcloud/cloud-provider-ibmcloud-s390x-periodics.yaml

@@ -137,3 +137,136 @@ periodics:
           requests:
             cpu: 6
             memory: 20Gi
+
+  - name: ci-kubernetes-s390x-conformance-latest-kubetest2
+    cron: "0 2-23/3 * * *" # every 3h starting at 02:00 UTC
+    cluster: k8s-infra-s390x-prow-build
+    labels:
+      preset-ibmcloud-cred-z: "true"
+    decorate: true
+    decoration_config:
+      timeout: 220m
+    extra_refs:
+      - base_ref: main
+        org: kubernetes-sigs
+        repo: provider-ibmcloud-test-infra
+        workdir: true
+    annotations:
+      description: Runs conformance tests using kubetest2 against kubernetes ci latest on IBM VPC
+      testgrid-dashboards: ibm-s390x-k8s, conformance-s390x, ibm-k8s-e2e
+      testgrid-tab-name: ci-kubernetes-s390x-conformance-latest-kubetest2
+    spec:
+      containers:
+        - image: us-central1-docker.pkg.dev/k8s-staging-test-infra/images/kubekins-e2e:v20251021-e2c2c9806f-master
+          securityContext:
+            privileged: true
+          env:
+            - name: "BOSKOS_HOST"
+              value: "boskos.test-pods.svc.cluster.local"
+            - name: "USER"
+              value: "ci-kubernetes-s390x-conformance-latest-kubetest2"
+          resources:
+            requests:
+              cpu: 4
+              memory: "14Gi"
+            limits:
+              cpu: 4
+              memory: "14Gi"
+          command:
+            - runner.sh
+          args:
+            - bash
+            - -c
+            - |
+              set -o xtrace
+              set +o errexit
+              export PATH=$GOPATH/bin:$PATH
+              export GO111MODULE=on
+              RESOURCE_TYPE="vpc-service"
+
+              #Call to boskos to checkout resource
+              heartbeat_account(){
+                count=0
+                url="http://${BOSKOS_HOST}/update?name=${BOSKOS_RESOURCE_NAME}&state=busy&owner=${USER}"
+                while [ ${count} -lt 120 ]
+                do
+                  status_code=$(curl -s -o /dev/null -w '%{http_code}' -X POST ${url})
+                  if [[ ${status_code} != 200 ]]; then
+                    echo "Heart beat to resource '${BOSKOS_RESOURCE_NAME}' failed due to invalid response, status code: ${status_code}"
+                    exit 1
+                  fi
+                  count=$(( $count + 1 ))
+                  sleep 60
+                done
+              }
+
+              if [ -n "${BOSKOS_HOST:-}" ]; then
+                set +o xtrace
+                url="http://${BOSKOS_HOST}/acquire?type=${RESOURCE_TYPE}&state=free&dest=busy&owner=${USER}"
+                output=$(curl -v -X POST ${url})
+                [ $? = 0 ] && status_code=200
+
+                if [[ ${status_code} == 200 && ${output} =~ "failed" ]]; then
+                  echo "Failed to acquire resource from Boskos of type ${RESOURCE_TYPE}"
+                  exit 1
+                elif [[ ${status_code} == 200 ]]; then
+                  export BOSKOS_RESOURCE_NAME=$(echo ${output} | jq -r '.name')
+                  export BOSKOS_REGION=$(echo ${output} | jq -r '.userdata["region"]')
+                  export BOSKOS_RESOURCE_GROUP=$(echo ${output} | jq -r '.userdata["resource-group-name"]')
+                  export BOSKOS_ZONE=$(echo ${output} | jq -r '.userdata["zone"]')
+                  rm -rf output.json
+                else
+                  echo "Failed to acquire resource due to invalid response, status code : ${status_code}"
+                  exit 1
+                fi
+                heartbeat_account >> "$ARTIFACTS/boskos.log" 2>&1 &
+                HEART_BEAT_PID=$(echo $!)
+              fi
+
+              #Setup of kubetest2 tf deployer and ginkgo tester
+              make install-deployer-tf
+              go install sigs.k8s.io/kubetest2/kubetest2-tester-ginkgo@latest
+
+              CLUSTER_NAME="config1-$(date +%s)"
+              kubetest2 tf  --target-provider vpc  --vpc-node-image-name ibm-ubuntu-22-04-4-minimal-s390x-3 \
+              --vpc-region ${BOSKOS_REGION} --vpc-zone ${BOSKOS_ZONE} \
+              --vpc-name ${BOSKOS_RESOURCE_NAME} \
+              --vpc-resource-group ${BOSKOS_RESOURCE_GROUP} \
+              --vpc-ssh-key k8s-s390x-ssh-key \
+              --vpc-node-profile bz2-4x16 \
+              --ssh-private-key /etc/secret-volume/ssh-privatekey \
+              --extra-vars=k8s_tar_bundles:kubernetes-server-linux-s390x.tar.gz \
+              --extra-vars=pod_subnet:10.244.0.0/16 \
+              --extra-vars=cni_provider:flannel \
+              --build-version $(curl -Ls https://dl.k8s.io/ci/latest.txt) \
+              --cluster-name $CLUSTER_NAME \
+              --workers-count 2 \
+              --up --auto-approve --retry-on-tf-failure 3 \
+              --break-kubetest-on-upfail true \
+              --build-version $(curl -Ls https://dl.k8s.io/ci/latest.txt) \
+              --test=ginkgo -- --parallel 10 --test-package-dir ci --test-package-marker latest.txt --focus-regex='\[Conformance\]' --skip-regex='\[Serial\]'; rc1=$?
+              export KUBECONFIG="$(pwd)/$CLUSTER_NAME/kubeconfig"
+              export ARTIFACTS=$ARTIFACTS/serial_tests_artifacts
+              #Run Serial Conformance tests
+              kubetest2 tf --target-provider vpc --vpc-region ${BOSKOS_REGION} --vpc-zone ${BOSKOS_ZONE} \
+                --ignore-cluster-dir true \
+                --cluster-name $CLUSTER_NAME \
+                --down --auto-approve --ignore-destroy-errors \
+                --test=ginkgo -- --test-package-dir ci \
+                --test-package-marker latest.txt --focus-regex='\[Serial\].*\[Conformance\]'; rc2=$?
+              if [ -n "${BOSKOS_HOST:-}" ]; then
+                url="http://${BOSKOS_HOST}/release?name=${BOSKOS_RESOURCE_NAME}&dest=dirty&owner=${USER}"
+                status_code=$(curl -w '%{http_code}' -X POST ${url})
+                if [[ ${status_code} != 200 ]]; then
+                  echo "Failed to release resource: ${BOSKOS_RESOURCE_NAME}"
+                  exit 1
+                fi
+              fi
+
+              if [ $rc1 != 0 ]; then
+                  echo "ERROR: Non Serial k8s Conformance tests exited with code: $rc1"
+                  exit $rc1
+              elif [ $rc2 != 0 ]; then
+                  echo "ERROR: Serial k8s Conformance tests exited with code: $rc2"
+                  exit $rc2
+              fi

config/prow/config.yaml

@@ -1123,6 +1123,23 @@ presets:
       secret:
         defaultMode: 256
         secretName: prow-job-ssh-private-key
+- labels:
+    preset-ibmcloud-cred-z: "true"
+  volumeMounts:
+    - mountPath: /etc/secret-volume
+      name: prow-job-ssh-private-key
+      readOnly: true
+  env:
+    - name: TF_VAR_vpc_api_key
+      valueFrom:
+        secretKeyRef:
+          name: prow-job-api-key
+          key: key
+  volumes:
+    - name: prow-job-ssh-private-key
+      secret:
+        defaultMode: 256
+        secretName: prow-job-ssh-private-key
 # enable GOPROXY by default
 - env:
   - name: GOPROXY

config/testgrids/conformance/conformance-all.yaml

@@ -231,9 +231,6 @@ dashboards:
 # s390x dashboard
 - name: conformance-s390x
   dashboard_tab:
-    - name: Periodic s390x conformance test on local cluster
-      test_group_name: s390x-conformance
-      description: Runs conformance tests by using kubetest2 against latest master version of kubernetes on local s390x cluster
     - name: Periodic s390x conformance test on local cluster , v1.33
       test_group_name: s390x-conformance-v1.33
       description: Runs conformance tests by using kubetest2 against latest v1.33 branch HEAD of kubernetes on local s390x cluster
@@ -263,8 +260,6 @@ test_groups:
   gcs_prefix: cel-conformance/test-logs
 
 #IBM s390x test results
-- name: s390x-conformance
-  gcs_prefix: k8s-conform-s390x-k8s/logs/ci-k8s-conformance-s390x
 - name: s390x-conformance-v1.33
   gcs_prefix: k8s-conform-s390x-k8s/logs/ci-k8s-conformance-s390x-v1.33
 - name: s390x-conformance-v1.32