Merge pull request #622 from agebhar1/feature/tc2

build(deps-dev): Bump org.testcontainers:testcontainers from 1.21.3 to 2.0.1

Author: embano1

.github/workflows/e2e.yaml

@@ -43,6 +43,10 @@ jobs:
           distribution: "temurin"
           java-version: ${{ matrix.java }}
           cache: "maven"
+      - name: Set up Docker Compose
+        uses: docker/setup-compose-action@364cc21a5de5b1ee4a7f5f9d3fa374ce0ccde746 # v1.2.0
+        with:
+          version: v2.40.3
       - name: Run Kafka Connect ${{ matrix.kafka }} E2E Tests with Apache Kafka
         run: |
           echo "Running integration tests with Docker Compose file: ${COMPOSE_FILE}"
@@ -69,6 +73,10 @@ jobs:
           distribution: "temurin"
           java-version: ${{ matrix.java }}
           cache: "maven"
+      - name: Set up Docker Compose
+        uses: docker/setup-compose-action@364cc21a5de5b1ee4a7f5f9d3fa374ce0ccde746 # v1.2.0
+        with:
+          version: v2.40.3
       - name: Run Confluent Kafka Platform ${{ matrix.kafka }} E2E Tests
         run: |
           echo "Running integration tests with Docker Compose file: ${COMPOSE_FILE}"
@@ -97,6 +105,10 @@ jobs:
           distribution: "temurin"
           java-version: ${{ matrix.java }}
           cache: "maven"
+      - name: Set up Docker Compose
+        uses: docker/setup-compose-action@364cc21a5de5b1ee4a7f5f9d3fa374ce0ccde746 # v1.2.0
+        with:
+          version: v2.40.3
       - name: Run Kafka Connect ${{ matrix.kafka }} E2E Tests with Redpanda ${{ matrix.redpanda }}
         run: |
           echo "Running integration tests with Docker Compose file: ${COMPOSE_FILE}"

CONTRIBUTING.md

@@ -85,10 +85,10 @@ mvn clean package -Drevision=$(git describe --tags --always)
 
 #### Integration Tests with Docker Compose
 
-To execute the integration tests (requires [Docker](https://docker.com/)) run:
+To execute the integration tests (requires [Docker](https://docker.com/) with [compose](https://github.com/docker/compose) plugin) run:
 
 ```bash
-export KAFKA_VERSION=4.0.0
+export KAFKA_VERSION=4.1.0
 export COMPOSE_FILE=e2e/docker_compose.yaml
 mvn clean verify -Drevision=$(git describe --tags --always)
 ```
@@ -116,9 +116,6 @@ locally e.g., to create the local environment:
 ```bash
 # create the JAR which will be loaded into Kafka Connect (Docker Volume)
 mvn clean package -Drevision=$(git describe --tags --always)
-
-# move to the e2e directory
-cd e2e
 ```
 
 Skip the next step if you already have these credentials configured in your environment:
@@ -139,7 +136,7 @@ export AWS_SESSION_TOKEN=ABCDEFGHIJKLMNOPQRST
 The following command removes any previous resources and creates the local test environment:
 
 ```bash
-docker compose -f docker_compose.yaml down --remove-orphans -v && docker compose -f docker_compose.yaml up
+docker compose -f e2e/docker_compose.yaml down --remove-orphans -v && docker compose -f e2e/docker_compose.yaml up
 ```
 
 > [!NOTE]  
@@ -153,10 +150,10 @@ e2e-connect-1     | [2023-07-12 09:07:29,263] INFO Kafka Connect started (org.ap
 ```
 
 Change the JSON configuration example `connect-config.json` (uses LocalStack defaults) according to your
-environment. In a separate terminal (within the `e2e` folder) deploy the connector:
+environment. In a separate terminal (within the project root folder) deploy the connector:
 
 ```bash
-curl -i --fail-with-body -X POST -H "Accept:application/json" -H  "Content-Type:application/json" http://$(docker compose -f docker_compose.yaml port connect 8083)/connectors/ -d @connect-config.json
+curl -i --fail-with-body -X POST -H "Accept:application/json" -H  "Content-Type:application/json" http://$(docker compose -f e2e/docker_compose.yaml port connect 8083)/connectors/ -d @e2e/connect-config.json
 ```
 
 The output should look like:
@@ -184,7 +181,7 @@ Produce a Kafka record to invoke the EventBridge sink connector:
 
 ```bash
 # open a shell in the Kafka broker container
-docker compose -f docker_compose.yaml exec -w /opt/kafka/bin kafka /bin/bash
+docker compose -f e2e/docker_compose.yaml exec -w /opt/kafka/bin kafka /bin/bash
 
 # produce an event
 # replace the topic with your connector settings if needed
@@ -231,7 +228,7 @@ The output event should look similar to the below:
 To tear down the environment run:
 
 ```bash
-docker compose -f docker_compose.yaml down --remove-orphans -v
+docker compose -f e2e/docker_compose.yaml down --remove-orphans -v
 ```
 
 ##### Remote Debug the Connector

e2e/docker_compose.yaml

@@ -33,8 +33,8 @@ services:
       KAFKA_LOG4J_OPTS: -Dlog4j.configuration=file:/mnt/connect-log4j.properties
       # KAFKA_OPTS: -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005
     volumes:
-      - ${PWD}:/mnt
-      - ${PWD}/../target:/opt/connectors/kafka-eventbridge-sink
+      - ${PWD}/e2e:/mnt
+      - ${PWD}/target:/opt/connectors/kafka-eventbridge-sink
     command: ["/bin/bash","-c","env && /opt/kafka/bin/connect-standalone.sh /mnt/connect-standalone.properties"]
 
   localstack:

e2e/docker_compose_confluent.yaml

@@ -52,7 +52,7 @@ services:
       AWS_SESSION_TOKEN: ${AWS_SESSION_TOKEN}
       # KAFKA_OPTS: -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005
     volumes:
-      - ${PWD}/../target:/usr/local/share/java/connect/kafka-eventbridge-sink
+      - ${PWD}/target:/usr/local/share/java/connect/kafka-eventbridge-sink
 
   localstack:
     image: docker.io/localstack/localstack:4.9.2

e2e/docker_compose_redpanda.yaml

@@ -30,8 +30,8 @@ services:
       KAFKA_LOG4J_OPTS: -Dlog4j.configuration=file:/mnt/connect-log4j.properties
       # KAFKA_OPTS: -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005
     volumes:
-      - ${PWD}:/mnt
-      - ${PWD}/../target:/opt/connectors/kafka-eventbridge-sink
+      - ${PWD}/e2e:/mnt
+      - ${PWD}/target:/opt/connectors/kafka-eventbridge-sink
     command: ["/bin/bash","-c","env && /opt/kafka/bin/connect-standalone.sh /mnt/connect-standalone.properties"]
 
   localstack:

pom.xml

@@ -39,7 +39,7 @@
         <org.mockito.version>5.20.0</org.mockito.version>
         <org.slf4j.version>2.0.17</org.slf4j.version>
         <org.skyscreamer.version>1.5.3</org.skyscreamer.version>
-        <org.testcontainers.version>1.21.3</org.testcontainers.version>
+        <org.testcontainers.version>2.0.1</org.testcontainers.version>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     </properties>
 
@@ -158,8 +158,7 @@
         </dependency>
         <dependency>
             <groupId>org.testcontainers</groupId>
-            <artifactId>junit-jupiter</artifactId>
-            <version>1.21.3</version>
+            <artifactId>testcontainers-junit-jupiter</artifactId>
             <scope>test</scope>
         </dependency>
     </dependencies>
@@ -274,8 +273,10 @@
             </dependency>
             <dependency>
                 <groupId>org.testcontainers</groupId>
-                <artifactId>testcontainers</artifactId>
+                <artifactId>testcontainers-bom</artifactId>
                 <version>${org.testcontainers.version}</version>
+                <scope>import</scope>
+                <type>pom</type>
             </dependency>
             <dependency>
                 <groupId>org.skyscreamer</groupId>

src/test/java/software/amazon/event/kafkaconnector/AbstractEventBridgeSinkConnectorIT.java

@@ -36,7 +36,7 @@
 import org.junit.jupiter.api.TestInstance;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.testcontainers.containers.DockerComposeContainer;
+import org.testcontainers.containers.ComposeContainer;
 import org.testcontainers.containers.output.Slf4jLogConsumer;
 import org.testcontainers.containers.wait.strategy.Wait;
 import org.testcontainers.junit.jupiter.Testcontainers;
@@ -64,28 +64,28 @@ public abstract class AbstractEventBridgeSinkConnectorIT {
   private static final String KAFKA_VERSION_ENV = "KAFKA_VERSION";
   private static final String REDPANDA_VERSION_ENV = "REDPANDA_VERSION";
 
-  private static final String CONNECT_SERVICE = "connect_1";
+  private static final String CONNECT_SERVICE = "connect";
   private static final int CONNECT_EXPOSED_SERVICE_PORT = 8083;
 
-  private static final String LOCALSTACK_SERVICE = "localstack_1";
+  private static final String LOCALSTACK_SERVICE = "localstack";
   private static final int LOCALSTACK_EXPOSED_SERVICE_PORT = 4566;
 
   private static final String AWS_ACCESS_KEY_ID = "test";
   private static final String AWS_SECRET_ACCESS_KEY = "test";
 
-  private final String RUNNING_STATE = "RUNNING";
+  private static final String RUNNING_STATE = "RUNNING";
 
-  protected final String TEST_RESOURCE_NAME = "eventbridge-e2e";
+  protected static final String TEST_RESOURCE_NAME = "eventbridge-e2e";
 
-  protected final String TEST_BUCKET_NAME = "test-bucket";
+  protected static final String TEST_BUCKET_NAME = "test-bucket";
 
   private String connectorName = null;
 
   protected static final Logger log =
       LoggerFactory.getLogger(AbstractEventBridgeSinkConnectorIT.class);
 
-  private final DockerComposeContainer<?> environment =
-      new DockerComposeContainer<>("e2e", getComposeFile())
+  private final ComposeContainer environment =
+      new ComposeContainer("e2e", getComposeFile())
           .withLogConsumer("connect", new Slf4jLogConsumer(log).withSeparateOutputStreams())
           .withEnv("AWS_ACCESS_KEY_ID", AWS_ACCESS_KEY_ID)
           .withEnv("AWS_SECRET_ACCESS_KEY", AWS_SECRET_ACCESS_KEY)