Validate no WAL corruption when both nodes shutdown gracefully

Author: kasturinarra

test/extended/two_node/tnf_recovery.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"math/rand"
 	"os"
+	"strings"
 	"time"
 
 	g "github.com/onsi/ginkgo/v2"
@@ -407,6 +408,36 @@ var _ = g.Describe("[sig-etcd][apigroup:config.openshift.io][OCPFeatureGate:Dual
 			&targetNode, true, false, // targetNode expected started == true, learner == false
 			6*time.Minute, 45*time.Second)
 	})
+
+	g.It("should recover after simultaneous graceful shutdown of both nodes", func() {
+		g.GinkgoT().Printf("Gracefully rebooting both nodes: %s and %s\n",
+			targetNode.Name, peerNode.Name)
+
+		g.By(fmt.Sprintf("Triggering graceful reboot on %s", targetNode.Name))
+		err := exutil.TriggerNodeRebootGraceful(oc.KubeClient(), targetNode.Name)
+		o.Expect(err).To(o.BeNil(), fmt.Sprintf("Expected to trigger graceful reboot on %s without error", targetNode.Name))
+
+		g.By(fmt.Sprintf("Triggering graceful reboot on %s", peerNode.Name))
+		err = exutil.TriggerNodeRebootGraceful(oc.KubeClient(), peerNode.Name)
+		o.Expect(err).To(o.BeNil(), fmt.Sprintf("Expected to trigger graceful reboot on %s without error", peerNode.Name))
+
+		g.By("Waiting for graceful shutdown to take effect (shutdown -r 1 schedules reboot in 1 minute)")
+		time.Sleep(90 * time.Second)
+
+		g.By(fmt.Sprintf("Waiting for both etcd members to become healthy (timeout: %v)", membersHealthyAfterDoubleReboot))
+		validateEtcdRecoveryState(oc, etcdClientFactory,
+			&targetNode,
+			&peerNode, true, false,
+			membersHealthyAfterDoubleReboot, utils.FiveSecondPollInterval)
+
+		g.By("Verifying etcd containers are running on both nodes")
+		for _, node := range []corev1.Node{targetNode, peerNode} {
+			got, err := exutil.DebugNodeRetryWithOptionsAndChroot(oc, node.Name, "openshift-etcd",
+				strings.Split(ensurePodmanEtcdContainerIsRunning, " ")...)
+			o.Expect(err).To(o.BeNil(), fmt.Sprintf("Expected no error checking etcd on %s", node.Name))
+			o.Expect(got).To(o.Equal("'true'"), fmt.Sprintf("Expected etcd container running on %s", node.Name))
+		}
+	})
 })
 
 func validateEtcdRecoveryState(