From 1176da6169e5ed2bc148ab48480de418acf1c7ec Mon Sep 17 00:00:00 2001 From: Sergei Egorov Date: Fri, 10 May 2019 20:41:11 +0200 Subject: [PATCH 01/11] Simplify Kafka container by deferring the Kafka command --- .../containers/KafkaContainer.java | 85 +++++++++++++------ 1 file changed, 57 insertions(+), 28 deletions(-) diff --git a/modules/kafka/src/main/java/org/testcontainers/containers/KafkaContainer.java b/modules/kafka/src/main/java/org/testcontainers/containers/KafkaContainer.java index 6396d6bcd7f..790a55ac71b 100644 --- a/modules/kafka/src/main/java/org/testcontainers/containers/KafkaContainer.java +++ b/modules/kafka/src/main/java/org/testcontainers/containers/KafkaContainer.java @@ -1,9 +1,13 @@ package org.testcontainers.containers; -import org.testcontainers.utility.Base58; +import com.github.dockerjava.api.command.ExecCreateCmdResponse; +import com.github.dockerjava.api.command.InspectContainerResponse; +import com.github.dockerjava.core.command.ExecStartResultCallback; +import lombok.NonNull; +import lombok.SneakyThrows; import org.testcontainers.utility.TestcontainersConfiguration; -import java.util.stream.Stream; +import java.util.concurrent.TimeUnit; /** * This container wraps Confluent Kafka and Zookeeper (optionally) @@ -17,7 +21,7 @@ public class KafkaContainer extends GenericContainer { protected String externalZookeeperConnect = null; - protected SocatContainer proxy; + private int port = -1; public KafkaContainer() { this("4.0.0"); @@ -26,8 +30,6 @@ public KafkaContainer() { public KafkaContainer(String confluentPlatformVersion) { super(TestcontainersConfiguration.getInstance().getKafkaImage() + ":" + confluentPlatformVersion); - withNetwork(Network.newNetwork()); - withNetworkAliases("kafka-" + Base58.randomString(6)); withExposedPorts(KAFKA_PORT); // Use two listeners with different names, it will force Kafka to communicate with itself via internal @@ -54,40 +56,67 @@ public KafkaContainer withExternalZookeeper(String connectString) { } public String getBootstrapServers() { - return String.format("PLAINTEXT://%s:%s", proxy.getContainerIpAddress(), proxy.getFirstMappedPort()); + if (port == -1) { + throw new IllegalStateException("You should start Kafka container first"); + } + return String.format("PLAINTEXT://%s:%s", getContainerIpAddress(), port); } @Override - protected void doStart() { - String networkAlias = getNetworkAliases().get(0); - proxy = new SocatContainer() - .withNetwork(getNetwork()) - .withTarget(KAFKA_PORT, networkAlias) - .withTarget(ZOOKEEPER_PORT, networkAlias); + @NonNull + public synchronized Network getNetwork() { + if (super.getNetwork() == null) { + // Backward compatibility + withNetwork(Network.newNetwork()); + } + return super.getNetwork(); + } - proxy.start(); - withEnv("KAFKA_ADVERTISED_LISTENERS", "BROKER://" + networkAlias + ":9092" + "," + getBootstrapServers()); + @Override + protected void doStart() { + withCommand("sleep infinity"); - if (externalZookeeperConnect != null) { - withEnv("KAFKA_ZOOKEEPER_CONNECT", externalZookeeperConnect); - } else { + if (externalZookeeperConnect == null) { addExposedPort(ZOOKEEPER_PORT); - withEnv("KAFKA_ZOOKEEPER_CONNECT", "localhost:2181"); - withCommand( - "sh", - "-c", - // Use command to create the file to avoid file mounting (useful when you run your tests against a remote Docker daemon) - "printf 'clientPort=2181\ndataDir=/var/lib/zookeeper/data\ndataLogDir=/var/lib/zookeeper/log' > /zookeeper.properties" + - " && zookeeper-server-start /zookeeper.properties" + - " & /etc/confluent/docker/run" - ); } super.doStart(); } @Override - public void stop() { - Stream.of(super::stop, proxy::stop).parallel().forEach(Runnable::run); + @SneakyThrows + protected void containerIsStarting(InspectContainerResponse containerInfo) { + super.containerIsStarting(containerInfo); + + port = getMappedPort(KAFKA_PORT); + + final String zookeeperConnect; + if (externalZookeeperConnect != null) { + zookeeperConnect = externalZookeeperConnect; + } else { + zookeeperConnect = "localhost:" + ZOOKEEPER_PORT; + + // Start ZooKeeper + ExecCreateCmdResponse execCreateCmdResponse = dockerClient.execCreateCmd(getContainerId()) + .withCmd("sh", "-c", "" + + "printf 'clientPort=" + ZOOKEEPER_PORT + "\ndataDir=/var/lib/zookeeper/data\ndataLogDir=/var/lib/zookeeper/log' > /zookeeper.properties\n" + + "zookeeper-server-start /zookeeper.properties\n" + ) + .exec(); + + dockerClient.execStartCmd(execCreateCmdResponse.getId()).exec(new ExecStartResultCallback()).awaitStarted(10, TimeUnit.SECONDS); + } + + String internalIp = containerInfo.getNetworkSettings().getIpAddress(); + + ExecCreateCmdResponse execCreateCmdResponse = dockerClient.execCreateCmd(getContainerId()) + .withCmd("sh", "-c", "" + + "export KAFKA_ZOOKEEPER_CONNECT=" + zookeeperConnect + "\n" + + "export KAFKA_ADVERTISED_LISTENERS=" + getBootstrapServers() + "," + String.format("BROKER://%s:9092", internalIp) + "\n" + + "/etc/confluent/docker/run" + ) + .exec(); + + dockerClient.execStartCmd(execCreateCmdResponse.getId()).exec(new ExecStartResultCallback()).awaitStarted(10, TimeUnit.SECONDS); } } From 7dba2fbaa998151858133f6d98b56ed788be0bab Mon Sep 17 00:00:00 2001 From: Sergei Egorov Date: Sat, 24 Aug 2019 19:14:08 +0200 Subject: [PATCH 02/11] WIP: reusable containers based on labels --- .../containers/GenericContainer.java | 133 ++++++++++-- .../containers/LazyDockerClient.java | 15 ++ .../images/builder/ImageFromDockerfile.java | 3 + .../utility/TestcontainersConfiguration.java | 5 + .../containers/ReusabilityUnitTests.java | 199 ++++++++++++++++++ .../JdbcDatabaseContainerProvider.java | 16 +- .../testcontainers/jdbc/ConnectionUrl.java | 4 + .../jdbc/ContainerDatabaseDriver.java | 2 +- 8 files changed, 346 insertions(+), 31 deletions(-) create mode 100644 core/src/main/java/org/testcontainers/containers/LazyDockerClient.java create mode 100644 core/src/test/java/org/testcontainers/containers/ReusabilityUnitTests.java diff --git a/core/src/main/java/org/testcontainers/containers/GenericContainer.java b/core/src/main/java/org/testcontainers/containers/GenericContainer.java index 14ce129c9a5..d1132c185b8 100644 --- a/core/src/main/java/org/testcontainers/containers/GenericContainer.java +++ b/core/src/main/java/org/testcontainers/containers/GenericContainer.java @@ -1,5 +1,9 @@ package org.testcontainers.containers; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.MapperFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; import com.github.dockerjava.api.DockerClient; import com.github.dockerjava.api.command.CreateContainerCmd; import com.github.dockerjava.api.command.InspectContainerResponse; @@ -13,12 +17,13 @@ import com.github.dockerjava.api.model.Volume; import com.github.dockerjava.api.model.VolumesFrom; import com.google.common.base.Strings; +import com.google.common.collect.ImmutableMap; import lombok.AccessLevel; import lombok.Data; -import lombok.EqualsAndHashCode; import lombok.NonNull; import lombok.Setter; import lombok.SneakyThrows; +import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; import org.apache.commons.compress.utils.IOUtils; @@ -62,11 +67,13 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; +import java.lang.reflect.Method; import java.nio.charset.Charset; import java.nio.file.Path; import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -100,6 +107,8 @@ public class GenericContainer> public static final String INTERNAL_HOST_HOSTNAME = "host.testcontainers.internal"; + static final String HASH_LABEL = "org.testcontainers.hash"; + /* * Default settings */ @@ -168,11 +177,12 @@ public class GenericContainer> protected final Set dependencies = new HashSet<>(); - /* + /** * Unique instance of DockerClient for use by this container object. + * We use {@link LazyDockerClient} here to avoid eager client creation */ @Setter(AccessLevel.NONE) - protected DockerClient dockerClient = DockerClientFactory.instance().client(); + protected DockerClient dockerClient = LazyDockerClient.INSTANCE; /* * Info about the Docker server; lazily fetched. @@ -222,6 +232,8 @@ public class GenericContainer> @Nullable private Map tmpFsMapping; + @Setter(AccessLevel.NONE) + private boolean shouldBeReused = false; public GenericContainer() { this(TestcontainersConfiguration.getInstance().getTinyImage()); @@ -291,6 +303,23 @@ protected void doStart() { } } + @SneakyThrows + protected boolean canBeReused() { + for (Class type = getClass(); type != GenericContainer.class; type = type.getSuperclass()) { + try { + Method method = type.getDeclaredMethod("containerIsCreated", String.class); + if (method.getDeclaringClass() != GenericContainer.class) { + logger().warn("{} can't be reused because it overrides {}", getClass(), method.getName()); + return false; + } + } catch (NoSuchMethodException e) { + // ignore + } + } + + return true; + } + private void tryStart() { try { String dockerImageName = image.get(); @@ -300,36 +329,73 @@ private void tryStart() { CreateContainerCmd createCommand = dockerClient.createContainerCmd(dockerImageName); applyConfiguration(createCommand); - containerId = createCommand.exec().getId(); + createCommand.getLabels().put(DockerClientFactory.TESTCONTAINERS_LABEL, "true"); - connectToPortForwardingNetwork(createCommand.getNetworkMode()); + if (shouldBeReused) { + if (!canBeReused()) { + throw new IllegalStateException("This container does not support reuse"); + } + + if (TestcontainersConfiguration.getInstance().environmentSupportsReuse()) { + String hash = hash(createCommand); + + containerId = findContainerForReuse(hash).orElse(null); - copyToFileContainerPathMap.forEach(this::copyFileToContainer); + if (containerId != null) { + logger().info("Reusing container with ID: {} and hash: {}", containerId, hash); + containerInfo = dockerClient.inspectContainerCmd(containerId).exec(); + } else { + logger().debug("Can't find a reusable running container with hash: {}", hash); - containerIsCreated(containerId); + createCommand.getLabels().put(HASH_LABEL, hash); + } + } else { + logger().info("Reuse was requested but the environment does not support the reuse of containers"); + } + } else { + createCommand.getLabels().put(DockerClientFactory.TESTCONTAINERS_SESSION_ID_LABEL, DockerClientFactory.SESSION_ID); + } - logger().info("Starting container with ID: {}", containerId); - dockerClient.startContainerCmd(containerId).exec(); + if (containerInfo == null) { + containerId = createCommand.exec().getId(); + + copyToFileContainerPathMap.forEach(this::copyFileToContainer); + } + + connectToPortForwardingNetwork(createCommand.getNetworkMode()); + + if (containerInfo == null) { + containerIsCreated(containerId); + + logger().info("Starting container with ID: {}", containerId); + dockerClient.startContainerCmd(containerId).exec(); + } logger().info("Container {} is starting: {}", dockerImageName, containerId); // For all registered output consumers, start following as close to container startup as possible this.logConsumers.forEach(this::followOutput); - // Tell subclasses that we're starting - containerInfo = dockerClient.inspectContainerCmd(containerId).exec(); + boolean reused = containerInfo != null; + + if (containerInfo == null) { + // Tell subclasses that we're starting + containerInfo = dockerClient.inspectContainerCmd(containerId).exec(); + } containerName = containerInfo.getName(); containerIsStarting(containerInfo); - // Wait until the container has reached the desired running state - if (!this.startupCheckStrategy.waitUntilStartupSuccessful(dockerClient, containerId)) { - // Bail out, don't wait for the port to start listening. - // (Exception thrown here will be caught below and wrapped) - throw new IllegalStateException("Container did not start correctly."); - } + if (!reused) { + // Wait until the container has reached the desired running state + if (!this.startupCheckStrategy.waitUntilStartupSuccessful(dockerClient, containerId)) { + // Bail out, don't wait for the port to start listening. + // (Exception thrown here will be caught below and wrapped) + throw new IllegalStateException("Container did not start correctly."); + } - // Wait until the process within the container has become ready for use (e.g. listening on network, log message emitted, etc). - waitUntilContainerStarted(); + // Wait until the process within the container has become ready for use (e.g. listening on network, log message emitted, etc). + waitUntilContainerStarted(); + } logger().info("Container {} started", dockerImageName); containerIsStarted(containerInfo); @@ -351,6 +417,29 @@ private void tryStart() { } } + @SneakyThrows(JsonProcessingException.class) + final String hash(CreateContainerCmd createCommand) { + // TODO add Testcontainers' version to the hash + String commandJson = new ObjectMapper() + .enable(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY) + .enable(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS) + .writeValueAsString(createCommand); + + return DigestUtils.sha1Hex(commandJson); + } + + Optional findContainerForReuse(String hash) { + // TODO locking + return dockerClient.listContainersCmd() + .withLabelFilter(ImmutableMap.of(HASH_LABEL, hash)) + .withLimit(1) + .withStatusFilter(Arrays.asList("running")) + .exec() + .stream() + .findAny() + .map(it -> it.getId()); + } + /** * Set any custom settings for the create command such as shared memory size. */ @@ -613,7 +702,6 @@ private void applyConfiguration(CreateContainerCmd createCommand) { if (createCommand.getLabels() != null) { combinedLabels.putAll(createCommand.getLabels()); } - combinedLabels.putAll(DockerClientFactory.DEFAULT_LABELS); createCommand.withLabels(combinedLabels); } @@ -1246,6 +1334,11 @@ public SELF withTmpFs(Map mapping) { return self(); } + public SELF withReuse(boolean reusable) { + this.shouldBeReused = reusable; + return self(); + } + @Override public boolean equals(Object o) { return this == o; diff --git a/core/src/main/java/org/testcontainers/containers/LazyDockerClient.java b/core/src/main/java/org/testcontainers/containers/LazyDockerClient.java new file mode 100644 index 00000000000..b1341b52755 --- /dev/null +++ b/core/src/main/java/org/testcontainers/containers/LazyDockerClient.java @@ -0,0 +1,15 @@ +package org.testcontainers.containers; + +import com.github.dockerjava.api.DockerClient; +import lombok.experimental.Delegate; +import org.testcontainers.DockerClientFactory; + +enum LazyDockerClient implements DockerClient { + + INSTANCE; + + @Delegate + final DockerClient getDockerClient() { + return DockerClientFactory.instance().client(); + } +} diff --git a/core/src/main/java/org/testcontainers/images/builder/ImageFromDockerfile.java b/core/src/main/java/org/testcontainers/images/builder/ImageFromDockerfile.java index 8c653718a69..8d0a16b7b44 100644 --- a/core/src/main/java/org/testcontainers/images/builder/ImageFromDockerfile.java +++ b/core/src/main/java/org/testcontainers/images/builder/ImageFromDockerfile.java @@ -46,6 +46,9 @@ public class ImageFromDockerfile extends LazyFuture implements static { Runtime.getRuntime().addShutdownHook(new Thread(DockerClientFactory.TESTCONTAINERS_THREAD_GROUP, () -> { + if (imagesToDelete.isEmpty()) { + return; + } DockerClient dockerClientForCleaning = DockerClientFactory.instance().client(); try { for (String dockerImageName : imagesToDelete) { diff --git a/core/src/main/java/org/testcontainers/utility/TestcontainersConfiguration.java b/core/src/main/java/org/testcontainers/utility/TestcontainersConfiguration.java index 608d02716d0..dca15796731 100644 --- a/core/src/main/java/org/testcontainers/utility/TestcontainersConfiguration.java +++ b/core/src/main/java/org/testcontainers/utility/TestcontainersConfiguration.java @@ -74,6 +74,11 @@ public boolean isDisableChecks() { return Boolean.parseBoolean((String) properties.getOrDefault("checks.disable", "false")); } + public boolean environmentSupportsReuse() { + // FIXME read it only from global properties + return Boolean.parseBoolean((String) properties.getOrDefault("testcontainers.reuse.enable", "false")); + } + public String getDockerClientStrategyClassName() { return (String) properties.get("docker.client.strategy"); } diff --git a/core/src/test/java/org/testcontainers/containers/ReusabilityUnitTests.java b/core/src/test/java/org/testcontainers/containers/ReusabilityUnitTests.java new file mode 100644 index 00000000000..8f7648190f4 --- /dev/null +++ b/core/src/test/java/org/testcontainers/containers/ReusabilityUnitTests.java @@ -0,0 +1,199 @@ +package org.testcontainers.containers; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.github.dockerjava.api.DockerClient; +import com.github.dockerjava.api.command.*; +import com.github.dockerjava.api.model.Container; +import com.github.dockerjava.core.command.CreateContainerCmdImpl; +import com.github.dockerjava.core.command.InspectContainerCmdImpl; +import com.github.dockerjava.core.command.ListContainersCmdImpl; +import com.github.dockerjava.core.command.StartContainerCmdImpl; +import lombok.RequiredArgsConstructor; +import lombok.experimental.FieldDefaults; +import org.junit.Assume; +import org.junit.Test; +import org.junit.experimental.runners.Enclosed; +import org.junit.runner.RunWith; +import org.junit.runners.BlockJUnit4ClassRunner; +import org.junit.runners.Parameterized; +import org.mockito.Mockito; +import org.mockito.stubbing.Answer; +import org.rnorth.visibleassertions.VisibleAssertions; +import org.testcontainers.containers.startupcheck.StartupCheckStrategy; +import org.testcontainers.containers.wait.strategy.AbstractWaitStrategy; +import org.testcontainers.containers.wait.strategy.WaitStrategy; +import org.testcontainers.utility.TestcontainersConfiguration; + +import java.util.Collections; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; + +@RunWith(Enclosed.class) +public class ReusabilityUnitTests { + + static final CompletableFuture IMAGE_FUTURE = CompletableFuture.completedFuture( + TestcontainersConfiguration.getInstance().getTinyImage() + ); + + @RunWith(Parameterized.class) + @RequiredArgsConstructor + @FieldDefaults(makeFinal = true) + public static class CanBeReusedTest { + + @Parameterized.Parameters(name = "{0}") + public static Object[][] data() { + return new Object[][] { + { "generic", new GenericContainer(IMAGE_FUTURE), true }, + { "anonymous generic", new GenericContainer(IMAGE_FUTURE) {}, true }, + { "custom", new CustomContainer(), true }, + { "anonymous custom", new CustomContainer() {}, true }, + { "custom with containerIsCreated", new CustomContainerWithContainerIsCreated(), false }, + }; + } + + String name; + + GenericContainer container; + + boolean reusable; + + @Test + public void shouldBeReusable() { + if (reusable) { + VisibleAssertions.assertTrue("Is reusable", container.canBeReused()); + } else { + VisibleAssertions.assertFalse("Is not reusable", container.canBeReused()); + } + } + + static class CustomContainer extends GenericContainer { + CustomContainer() { + super(IMAGE_FUTURE); + } + } + + static class CustomContainerWithContainerIsCreated extends GenericContainer { + + CustomContainerWithContainerIsCreated() { + super(IMAGE_FUTURE); + } + + @Override + protected void containerIsCreated(String containerId) { + super.containerIsCreated(containerId); + } + } + } + + @RunWith(BlockJUnit4ClassRunner.class) + @FieldDefaults(makeFinal = true) + public static class HashTest { + + StartupCheckStrategy startupCheckStrategy = new StartupCheckStrategy() { + @Override + public StartupStatus checkStartupState(DockerClient dockerClient, String containerId) { + return StartupStatus.SUCCESSFUL; + } + }; + + WaitStrategy waitStrategy = new AbstractWaitStrategy() { + @Override + protected void waitUntilReady() { + + } + }; + + DockerClient client = Mockito.mock(DockerClient.class); + + GenericContainer container = new GenericContainer(IMAGE_FUTURE) + .withStartupCheckStrategy(startupCheckStrategy) + .waitingFor(waitStrategy) + .withReuse(true); + + { + container.dockerClient = client; + } + + @Test + public void shouldStartIfListReturnsEmpty() { + String containerId = randomContainerId(); + when(client.createContainerCmd(any())).then(createContainerAnswer(containerId)); + when(client.listContainersCmd()).then(listContainersAnswer()); + when(client.startContainerCmd(containerId)).then(startContainerAnswer()); + when(client.inspectContainerCmd(containerId)).then(inspectContainerAnswer()); + + container.start(); + + Mockito.verify(client, Mockito.atLeastOnce()).startContainerCmd(containerId); + } + + @Test + public void shouldReuseIfListReturnsID() { + // TODO mock TestcontainersConfiguration + Assume.assumeTrue("supports reuse", TestcontainersConfiguration.getInstance().environmentSupportsReuse()); + when(client.createContainerCmd(any())).then(createContainerAnswer(randomContainerId())); + String existingContainerId = randomContainerId(); + when(client.listContainersCmd()).then(listContainersAnswer(existingContainerId)); + when(client.inspectContainerCmd(existingContainerId)).then(inspectContainerAnswer()); + + container.start(); + + Mockito.verify(client, Mockito.never()).startContainerCmd(existingContainerId); + } + + private String randomContainerId() { + return UUID.randomUUID().toString(); + } + + private Answer listContainersAnswer(String... ids) { + return invocation -> { + ListContainersCmd.Exec exec = command -> { + return new ObjectMapper().convertValue( + Stream.of(ids) + .map(id -> Collections.singletonMap("Id", id)) + .collect(Collectors.toList()), + new TypeReference>() {} + ); + }; + return new ListContainersCmdImpl(exec); + }; + } + + private Answer createContainerAnswer(String containerId) { + return invocation -> { + CreateContainerCmd.Exec exec = command -> { + CreateContainerResponse response = new CreateContainerResponse(); + response.setId(containerId); + return response; + }; + return new CreateContainerCmdImpl(exec, null, "image:latest"); + }; + } + + private Answer startContainerAnswer() { + return invocation -> { + StartContainerCmd.Exec exec = command -> { + return null; + }; + return new StartContainerCmdImpl(exec, invocation.getArgument(0)); + }; + } + + private Answer inspectContainerAnswer() { + return invocation -> { + InspectContainerCmd.Exec exec = command -> { + return new InspectContainerResponse(); + }; + return new InspectContainerCmdImpl(exec, invocation.getArgument(0)); + }; + } + + } +} diff --git a/modules/jdbc/src/main/java/org/testcontainers/containers/JdbcDatabaseContainerProvider.java b/modules/jdbc/src/main/java/org/testcontainers/containers/JdbcDatabaseContainerProvider.java index f2ebde857c5..2195a9c343c 100644 --- a/modules/jdbc/src/main/java/org/testcontainers/containers/JdbcDatabaseContainerProvider.java +++ b/modules/jdbc/src/main/java/org/testcontainers/containers/JdbcDatabaseContainerProvider.java @@ -45,11 +45,14 @@ public JdbcDatabaseContainer newInstance() { * @return Instance of {@link JdbcDatabaseContainer} */ public JdbcDatabaseContainer newInstance(ConnectionUrl url) { + final JdbcDatabaseContainer result; if (url.getImageTag().isPresent()) { - return newInstance(url.getImageTag().get()); + result = newInstance(url.getImageTag().get()); } else { - return newInstance(); + result = newInstance(); } + result.withReuse(url.isReusable()); + return result; } protected JdbcDatabaseContainer newInstanceFromConnectionUrl(ConnectionUrl connectionUrl, final String userParamName, final String pwdParamName) { @@ -59,14 +62,7 @@ protected JdbcDatabaseContainer newInstanceFromConnectionUrl(ConnectionUrl conne final String user = connectionUrl.getQueryParameters().getOrDefault(userParamName, "test"); final String password = connectionUrl.getQueryParameters().getOrDefault(pwdParamName, "test"); - final JdbcDatabaseContainer instance; - if (connectionUrl.getImageTag().isPresent()) { - instance = newInstance(connectionUrl.getImageTag().get()); - } else { - instance = newInstance(); - } - - return instance + return newInstance(connectionUrl) .withDatabaseName(databaseName) .withUsername(user) .withPassword(password); diff --git a/modules/jdbc/src/main/java/org/testcontainers/jdbc/ConnectionUrl.java b/modules/jdbc/src/main/java/org/testcontainers/jdbc/ConnectionUrl.java index e236d723c82..62f9ec08958 100644 --- a/modules/jdbc/src/main/java/org/testcontainers/jdbc/ConnectionUrl.java +++ b/modules/jdbc/src/main/java/org/testcontainers/jdbc/ConnectionUrl.java @@ -49,6 +49,8 @@ public class ConnectionUrl { private Optional initScriptPath = Optional.empty(); + private boolean reusable = false; + private Optional initFunction = Optional.empty(); private Optional queryString; @@ -132,6 +134,8 @@ private void parseUrl() { initScriptPath = Optional.ofNullable(containerParameters.get("TC_INITSCRIPT")); + reusable = Boolean.parseBoolean(containerParameters.get("TC_REUSABLE")); + Matcher funcMatcher = Patterns.INITFUNCTION_MATCHING_PATTERN.matcher(this.getUrl()); if (funcMatcher.matches()) { initFunction = Optional.of(new InitFunctionDef(funcMatcher.group(2), funcMatcher.group(4))); diff --git a/modules/jdbc/src/main/java/org/testcontainers/jdbc/ContainerDatabaseDriver.java b/modules/jdbc/src/main/java/org/testcontainers/jdbc/ContainerDatabaseDriver.java index 6d3df6c7cc5..c5413fe7efb 100644 --- a/modules/jdbc/src/main/java/org/testcontainers/jdbc/ContainerDatabaseDriver.java +++ b/modules/jdbc/src/main/java/org/testcontainers/jdbc/ContainerDatabaseDriver.java @@ -151,7 +151,7 @@ public synchronized Connection connect(String url, final Properties info) throws */ private Connection wrapConnection(final Connection connection, final JdbcDatabaseContainer container, final ConnectionUrl connectionUrl) { - final boolean isDaemon = connectionUrl.isInDaemonMode(); + final boolean isDaemon = connectionUrl.isInDaemonMode() || connectionUrl.isReusable(); Set connections = containerConnections.computeIfAbsent(container.getContainerId(), k -> new HashSet<>()); From 480b9d076bdbb910383761eecb0d4a2f5e177da4 Mon Sep 17 00:00:00 2001 From: Sergei Egorov Date: Sun, 25 Aug 2019 10:29:15 +0200 Subject: [PATCH 03/11] do not call `newInstance` from `newInstanceFromConnectionUrl` --- .../containers/JdbcDatabaseContainerProvider.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/modules/jdbc/src/main/java/org/testcontainers/containers/JdbcDatabaseContainerProvider.java b/modules/jdbc/src/main/java/org/testcontainers/containers/JdbcDatabaseContainerProvider.java index 2195a9c343c..f8694306384 100644 --- a/modules/jdbc/src/main/java/org/testcontainers/containers/JdbcDatabaseContainerProvider.java +++ b/modules/jdbc/src/main/java/org/testcontainers/containers/JdbcDatabaseContainerProvider.java @@ -62,7 +62,15 @@ protected JdbcDatabaseContainer newInstanceFromConnectionUrl(ConnectionUrl conne final String user = connectionUrl.getQueryParameters().getOrDefault(userParamName, "test"); final String password = connectionUrl.getQueryParameters().getOrDefault(pwdParamName, "test"); - return newInstance(connectionUrl) + final JdbcDatabaseContainer instance; + if (connectionUrl.getImageTag().isPresent()) { + instance = newInstance(connectionUrl.getImageTag().get()); + } else { + instance = newInstance(); + } + + return instance + .withReuse(connectionUrl.isReusable()) .withDatabaseName(databaseName) .withUsername(user) .withPassword(password); From 3b0adb36f15df10fc0c198d239b310034a910a8d Mon Sep 17 00:00:00 2001 From: Sergei Egorov Date: Sun, 25 Aug 2019 11:12:53 +0200 Subject: [PATCH 04/11] disable the port forwarding in `ReusabilityUnitTests` --- .../java/org/testcontainers/containers/ReusabilityUnitTests.java | 1 + 1 file changed, 1 insertion(+) diff --git a/core/src/test/java/org/testcontainers/containers/ReusabilityUnitTests.java b/core/src/test/java/org/testcontainers/containers/ReusabilityUnitTests.java index 8f7648190f4..b05db5b5077 100644 --- a/core/src/test/java/org/testcontainers/containers/ReusabilityUnitTests.java +++ b/core/src/test/java/org/testcontainers/containers/ReusabilityUnitTests.java @@ -113,6 +113,7 @@ protected void waitUntilReady() { DockerClient client = Mockito.mock(DockerClient.class); GenericContainer container = new GenericContainer(IMAGE_FUTURE) + .withNetworkMode("none") // to disable the port forwarding .withStartupCheckStrategy(startupCheckStrategy) .waitingFor(waitStrategy) .withReuse(true); From 66f481e0b770c8f636e2a13045eb5570db3271d2 Mon Sep 17 00:00:00 2001 From: Sergei Egorov Date: Sun, 25 Aug 2019 12:13:46 +0200 Subject: [PATCH 05/11] Simplify the startup sequence --- .../containers/GenericContainer.java | 45 +++++++++---------- 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/core/src/main/java/org/testcontainers/containers/GenericContainer.java b/core/src/main/java/org/testcontainers/containers/GenericContainer.java index d1132c185b8..7ae9258e658 100644 --- a/core/src/main/java/org/testcontainers/containers/GenericContainer.java +++ b/core/src/main/java/org/testcontainers/containers/GenericContainer.java @@ -71,6 +71,7 @@ import java.nio.charset.Charset; import java.nio.file.Path; import java.time.Duration; +import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -288,13 +289,15 @@ protected void doStart() { try { configure(); + Instant startedAt = Instant.now(); + logger().debug("Starting container: {}", getDockerImageName()); logger().debug("Trying to start container: {}", image.get()); AtomicInteger attempt = new AtomicInteger(0); Unreliables.retryUntilSuccess(startupAttempts, () -> { logger().debug("Trying to start container: {} (attempt {}/{})", image.get(), attempt.incrementAndGet(), startupAttempts); - tryStart(); + tryStart(startedAt); return true; }); @@ -320,7 +323,7 @@ protected boolean canBeReused() { return true; } - private void tryStart() { + private void tryStart(Instant startedAt) { try { String dockerImageName = image.get(); logger().debug("Starting container: {}", dockerImageName); @@ -331,6 +334,7 @@ private void tryStart() { createCommand.getLabels().put(DockerClientFactory.TESTCONTAINERS_LABEL, "true"); + boolean reused = false; if (shouldBeReused) { if (!canBeReused()) { throw new IllegalStateException("This container does not support reuse"); @@ -343,7 +347,7 @@ private void tryStart() { if (containerId != null) { logger().info("Reusing container with ID: {} and hash: {}", containerId, hash); - containerInfo = dockerClient.inspectContainerCmd(containerId).exec(); + reused = true; } else { logger().debug("Can't find a reusable running container with hash: {}", hash); @@ -356,15 +360,16 @@ private void tryStart() { createCommand.getLabels().put(DockerClientFactory.TESTCONTAINERS_SESSION_ID_LABEL, DockerClientFactory.SESSION_ID); } - if (containerInfo == null) { + if (!reused) { containerId = createCommand.exec().getId(); + // TODO add to the hash copyToFileContainerPathMap.forEach(this::copyFileToContainer); } connectToPortForwardingNetwork(createCommand.getNetworkMode()); - if (containerInfo == null) { + if (!reused) { containerIsCreated(containerId); logger().info("Starting container with ID: {}", containerId); @@ -376,28 +381,22 @@ private void tryStart() { // For all registered output consumers, start following as close to container startup as possible this.logConsumers.forEach(this::followOutput); - boolean reused = containerInfo != null; - - if (containerInfo == null) { - // Tell subclasses that we're starting - containerInfo = dockerClient.inspectContainerCmd(containerId).exec(); - } + // Tell subclasses that we're starting + containerInfo = dockerClient.inspectContainerCmd(containerId).exec(); containerName = containerInfo.getName(); containerIsStarting(containerInfo); - if (!reused) { - // Wait until the container has reached the desired running state - if (!this.startupCheckStrategy.waitUntilStartupSuccessful(dockerClient, containerId)) { - // Bail out, don't wait for the port to start listening. - // (Exception thrown here will be caught below and wrapped) - throw new IllegalStateException("Container did not start correctly."); - } - - // Wait until the process within the container has become ready for use (e.g. listening on network, log message emitted, etc). - waitUntilContainerStarted(); + // Wait until the container has reached the desired running state + if (!this.startupCheckStrategy.waitUntilStartupSuccessful(dockerClient, containerId)) { + // Bail out, don't wait for the port to start listening. + // (Exception thrown here will be caught below and wrapped) + throw new IllegalStateException("Container did not start correctly."); } - logger().info("Container {} started", dockerImageName); + // Wait until the process within the container has become ready for use (e.g. listening on network, log message emitted, etc). + waitUntilContainerStarted(); + + logger().info("Container {} started in {}", dockerImageName, Duration.between(startedAt, Instant.now())); containerIsStarted(containerInfo); } catch (Exception e) { logger().error("Could not start container", e); @@ -1302,7 +1301,7 @@ public SELF withStartupAttempts(int attempts) { } /** - * Allow low level modifications of {@link CreateContainerCmd} after it was pre-configured in {@link #tryStart()}. + * Allow low level modifications of {@link CreateContainerCmd} after it was pre-configured in {@link #tryStart(Instant)}. * Invocation happens eagerly on a moment when container is created. * Warning: this does expose the underlying docker-java API so might change outside of our control. * From 111ae0fcf9344df46c5aea209106e8b9cd82e29c Mon Sep 17 00:00:00 2001 From: Sergei Egorov Date: Sat, 24 Aug 2019 22:26:38 +0200 Subject: [PATCH 06/11] speed up port detection by running the checks as a single command --- .../internal/ExternalPortListeningCheck.java | 4 +- .../InternalCommandPortListeningCheck.java | 45 ++++++++++--------- .../wait/strategy/HostPortWaitStrategy.java | 4 +- ...InternalCommandPortListeningCheckTest.java | 10 ++--- 4 files changed, 33 insertions(+), 30 deletions(-) diff --git a/core/src/main/java/org/testcontainers/containers/wait/internal/ExternalPortListeningCheck.java b/core/src/main/java/org/testcontainers/containers/wait/internal/ExternalPortListeningCheck.java index 3283880b569..2aae12bab8d 100644 --- a/core/src/main/java/org/testcontainers/containers/wait/internal/ExternalPortListeningCheck.java +++ b/core/src/main/java/org/testcontainers/containers/wait/internal/ExternalPortListeningCheck.java @@ -20,13 +20,13 @@ public class ExternalPortListeningCheck implements Callable { public Boolean call() { String address = containerState.getContainerIpAddress(); - for (Integer externalPort : externalLivenessCheckPorts) { + externalLivenessCheckPorts.parallelStream().forEach(externalPort -> { try { new Socket(address, externalPort).close(); } catch (IOException e) { throw new IllegalStateException("Socket not listening yet: " + externalPort); } - } + }); return true; } } diff --git a/core/src/main/java/org/testcontainers/containers/wait/internal/InternalCommandPortListeningCheck.java b/core/src/main/java/org/testcontainers/containers/wait/internal/InternalCommandPortListeningCheck.java index bad30fef653..43ed94bc84d 100644 --- a/core/src/main/java/org/testcontainers/containers/wait/internal/InternalCommandPortListeningCheck.java +++ b/core/src/main/java/org/testcontainers/containers/wait/internal/InternalCommandPortListeningCheck.java @@ -1,9 +1,13 @@ package org.testcontainers.containers.wait.internal; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.testcontainers.containers.Container.ExecResult; import org.testcontainers.containers.ExecInContainerPattern; import org.testcontainers.containers.wait.strategy.WaitStrategyTarget; +import java.time.Duration; +import java.time.Instant; import java.util.Set; import static java.lang.String.format; @@ -12,6 +16,7 @@ * Mechanism for testing that a socket is listening when run from the container being checked. */ @RequiredArgsConstructor +@Slf4j public class InternalCommandPortListeningCheck implements java.util.concurrent.Callable { private final WaitStrategyTarget waitStrategyTarget; @@ -19,30 +24,26 @@ public class InternalCommandPortListeningCheck implements java.util.concurrent.C @Override public Boolean call() { - for (Integer internalPort : internalPorts) { - tryPort(internalPort); + String command = "true"; + + for (int internalPort : internalPorts) { + command += " && "; + command += " ("; + command += format("cat /proc/net/tcp{,6} | awk '{print $2}' | grep -i :%x", internalPort); + command += " || "; + command += format("nc -vz -w 1 localhost %d", internalPort); + command += " || "; + command += format("/bin/bash -c ' externalLivenessCheckPorts = getLivenessCheckPorts(); if (externalLivenessCheckPorts.isEmpty()) { - log.debug("Liveness check ports of {} is empty. Not waiting.", waitStrategyTarget.getContainerInfo().getName()); + if (log.isDebugEnabled()) { + log.debug("Liveness check ports of {} is empty. Not waiting.", waitStrategyTarget.getContainerInfo().getName()); + } return; } diff --git a/core/src/test/java/org/testcontainers/containers/wait/internal/InternalCommandPortListeningCheckTest.java b/core/src/test/java/org/testcontainers/containers/wait/internal/InternalCommandPortListeningCheckTest.java index c1cd341943e..71a14c3c79e 100644 --- a/core/src/test/java/org/testcontainers/containers/wait/internal/InternalCommandPortListeningCheckTest.java +++ b/core/src/test/java/org/testcontainers/containers/wait/internal/InternalCommandPortListeningCheckTest.java @@ -6,7 +6,7 @@ import org.testcontainers.containers.BindMode; import org.testcontainers.containers.GenericContainer; -import static org.rnorth.visibleassertions.VisibleAssertions.assertThrows; +import static org.rnorth.visibleassertions.VisibleAssertions.assertFalse; import static org.rnorth.visibleassertions.VisibleAssertions.assertTrue; public class InternalCommandPortListeningCheckTest { @@ -30,8 +30,8 @@ public void singleListening() { public void nonListening() { final InternalCommandPortListeningCheck check = new InternalCommandPortListeningCheck(nginx, ImmutableSet.of(8080, 1234)); - assertThrows("InternalCommandPortListeningCheck detects a non-listening port among many", - IllegalStateException.class, - (Runnable) check::call); + final Boolean result = check.call(); + + assertFalse("InternalCommandPortListeningCheck detects a non-listening port among many", result); } -} \ No newline at end of file +} From 45f972d65929c432edfe9fcf2569b30254ffbfde Mon Sep 17 00:00:00 2001 From: Sergei Egorov Date: Sun, 25 Aug 2019 17:08:31 +0200 Subject: [PATCH 07/11] Separate "environment" and "classpath" properties (for global things) --- .../DockerClientProviderStrategy.java | 18 +--- .../utility/TestcontainersConfiguration.java | 96 +++++++++---------- .../TestcontainersConfigurationTest.java | 43 +++++++++ 3 files changed, 94 insertions(+), 63 deletions(-) create mode 100644 core/src/test/java/org/testcontainers/utility/TestcontainersConfigurationTest.java diff --git a/core/src/main/java/org/testcontainers/dockerclient/DockerClientProviderStrategy.java b/core/src/main/java/org/testcontainers/dockerclient/DockerClientProviderStrategy.java index a9321276fd6..5604246573e 100644 --- a/core/src/main/java/org/testcontainers/dockerclient/DockerClientProviderStrategy.java +++ b/core/src/main/java/org/testcontainers/dockerclient/DockerClientProviderStrategy.java @@ -165,20 +165,10 @@ public DockerClient getClient() { } protected DockerClient getClientForConfig(DockerClientConfig config) { - DockerClientBuilder clientBuilder = DockerClientBuilder - .getInstance(new AuthDelegatingDockerClientConfig(config)); - - String transportType = TestcontainersConfiguration.getInstance().getTransportType(); - if ("okhttp".equals(transportType)) { - clientBuilder - .withDockerCmdExecFactory(new OkHttpDockerCmdExecFactory()); - } else { - throw new IllegalArgumentException("Unknown transport type: " + transportType); - } - - LOGGER.info("Will use '{}' transport", transportType); - - return clientBuilder.build(); + return DockerClientBuilder + .getInstance(new AuthDelegatingDockerClientConfig(config)) + .withDockerCmdExecFactory(new OkHttpDockerCmdExecFactory()) + .build(); } protected void ping(DockerClient client, int timeoutInSeconds) { diff --git a/core/src/main/java/org/testcontainers/utility/TestcontainersConfiguration.java b/core/src/main/java/org/testcontainers/utility/TestcontainersConfiguration.java index 608d02716d0..3fbfd8db0a8 100644 --- a/core/src/main/java/org/testcontainers/utility/TestcontainersConfiguration.java +++ b/core/src/main/java/org/testcontainers/utility/TestcontainersConfiguration.java @@ -5,6 +5,7 @@ import java.io.*; import java.net.MalformedURLException; +import java.net.URL; import java.util.Objects; import java.util.Properties; import java.util.stream.Stream; @@ -19,12 +20,22 @@ public class TestcontainersConfiguration { private static String PROPERTIES_FILE_NAME = "testcontainers.properties"; - private static File GLOBAL_CONFIG_FILE = new File(System.getProperty("user.home"), "." + PROPERTIES_FILE_NAME); + private static File ENVIRONMENT_CONFIG_FILE = new File(System.getProperty("user.home"), "." + PROPERTIES_FILE_NAME); @Getter(lazy = true) private static final TestcontainersConfiguration instance = loadConfiguration(); - private final Properties properties; + @Getter(AccessLevel.NONE) + private final Properties environmentProperties; + + private final Properties properties = new Properties(); + + TestcontainersConfiguration(Properties environmentProperties, Properties classpathProperties) { + this.environmentProperties = environmentProperties; + + this.properties.putAll(classpathProperties); + this.properties.putAll(environmentProperties); + } public String getAmbassadorContainerImage() { return (String) properties.getOrDefault("ambassador.container.image", "richnorth/ambassador:latest"); @@ -71,15 +82,11 @@ public String getPulsarImage() { } public boolean isDisableChecks() { - return Boolean.parseBoolean((String) properties.getOrDefault("checks.disable", "false")); + return Boolean.parseBoolean((String) environmentProperties.getOrDefault("checks.disable", "false")); } public String getDockerClientStrategyClassName() { - return (String) properties.get("docker.client.strategy"); - } - - public String getTransportType() { - return properties.getProperty("transport.type", "okhttp"); + return (String) environmentProperties.get("docker.client.strategy"); } public Integer getImagePullPauseTimeout() { @@ -89,64 +96,55 @@ public Integer getImagePullPauseTimeout() { @Synchronized public boolean updateGlobalConfig(@NonNull String prop, @NonNull String value) { try { - Properties globalProperties = new Properties(); - GLOBAL_CONFIG_FILE.createNewFile(); - try (InputStream inputStream = new FileInputStream(GLOBAL_CONFIG_FILE)) { - globalProperties.load(inputStream); - } - - if (value.equals(globalProperties.get(prop))) { + if (value.equals(environmentProperties.get(prop))) { return false; } - globalProperties.setProperty(prop, value); + environmentProperties.setProperty(prop, value); - try (OutputStream outputStream = new FileOutputStream(GLOBAL_CONFIG_FILE)) { - globalProperties.store(outputStream, "Modified by Testcontainers"); + ENVIRONMENT_CONFIG_FILE.createNewFile(); + try (OutputStream outputStream = new FileOutputStream(ENVIRONMENT_CONFIG_FILE)) { + environmentProperties.store(outputStream, "Modified by Testcontainers"); } - // Update internal state only if global config was successfully updated + // Update internal state only if environment config was successfully updated properties.setProperty(prop, value); return true; } catch (Exception e) { - log.debug("Can't store global property {} in {}", prop, GLOBAL_CONFIG_FILE); + log.debug("Can't store environment property {} in {}", prop, ENVIRONMENT_CONFIG_FILE); return false; } } @SneakyThrows(MalformedURLException.class) private static TestcontainersConfiguration loadConfiguration() { - final TestcontainersConfiguration config = new TestcontainersConfiguration( - Stream - .of( - TestcontainersConfiguration.class.getClassLoader().getResource(PROPERTIES_FILE_NAME), - Thread.currentThread().getContextClassLoader().getResource(PROPERTIES_FILE_NAME), - GLOBAL_CONFIG_FILE.toURI().toURL() - ) - .filter(Objects::nonNull) - .map(it -> { - log.debug("Testcontainers configuration overrides will be loaded from {}", it); - - final Properties subProperties = new Properties(); - try (final InputStream inputStream = it.openStream()) { - subProperties.load(inputStream); - } catch (FileNotFoundException e) { - log.trace("Testcontainers config override was found on " + it + " but the file was not found", e); - } catch (IOException e) { - log.warn("Testcontainers config override was found on " + it + " but could not be loaded", e); - } - return subProperties; - }) - .reduce(new Properties(), (a, b) -> { - a.putAll(b); - return a; - }) + return new TestcontainersConfiguration( + readProperties(ENVIRONMENT_CONFIG_FILE.toURI().toURL()), + Stream + .of( + TestcontainersConfiguration.class.getClassLoader(), + Thread.currentThread().getContextClassLoader() + ) + .map(it -> it.getResource(PROPERTIES_FILE_NAME)) + .filter(Objects::nonNull) + .map(TestcontainersConfiguration::readProperties) + .reduce(new Properties(), (a, b) -> { + a.putAll(b); + return a; + }) ); + } - if (!config.getProperties().isEmpty()) { - log.debug("Testcontainers configuration overrides loaded from {}", config); + private static Properties readProperties(URL url) { + log.debug("Testcontainers configuration overrides will be loaded from {}", url); + Properties properties = new Properties(); + try (InputStream inputStream = url.openStream()) { + properties.load(inputStream); + } catch (FileNotFoundException e) { + log.trace("Testcontainers config override was found on {} but the file was not found", url, e); + } catch (IOException e) { + log.warn("Testcontainers config override was found on {} but could not be loaded", url, e); } - - return config; + return properties; } } diff --git a/core/src/test/java/org/testcontainers/utility/TestcontainersConfigurationTest.java b/core/src/test/java/org/testcontainers/utility/TestcontainersConfigurationTest.java new file mode 100644 index 00000000000..ad2a0be6cd3 --- /dev/null +++ b/core/src/test/java/org/testcontainers/utility/TestcontainersConfigurationTest.java @@ -0,0 +1,43 @@ +package org.testcontainers.utility; + +import org.junit.Test; + +import java.util.Properties; +import java.util.UUID; + +import static org.rnorth.visibleassertions.VisibleAssertions.assertEquals; +import static org.rnorth.visibleassertions.VisibleAssertions.assertFalse; +import static org.rnorth.visibleassertions.VisibleAssertions.assertTrue; + +public class TestcontainersConfigurationTest { + + final Properties environmentProperties = new Properties(); + + final Properties classpathProperties = new Properties(); + + @Test + public void shouldReadChecksFromEnvironmentOnly() { + assertFalse("checks enabled by default", newConfig().isDisableChecks()); + + classpathProperties.setProperty("checks.disable", "true"); + assertFalse("checks are not affected by classpath properties", newConfig().isDisableChecks()); + + environmentProperties.setProperty("checks.disable", "true"); + assertTrue("checks disabled", newConfig().isDisableChecks()); + } + + @Test + public void shouldReadDockerClientStrategyFromEnvironmentOnly() { + String currentValue = newConfig().getDockerClientStrategyClassName(); + + classpathProperties.setProperty("docker.client.strategy", UUID.randomUUID().toString()); + assertEquals("Docker client strategy is not affected by classpath properties", currentValue, newConfig().getDockerClientStrategyClassName()); + + environmentProperties.setProperty("docker.client.strategy", "foo"); + assertEquals("Docker client strategy is changed", "foo", newConfig().getDockerClientStrategyClassName()); + } + + private TestcontainersConfiguration newConfig() { + return new TestcontainersConfiguration(environmentProperties, classpathProperties); + } +} From 173f66ee54f7ad51dec8edca44ca913b412dbb1f Mon Sep 17 00:00:00 2001 From: Sergei Egorov Date: Sun, 25 Aug 2019 17:12:51 +0200 Subject: [PATCH 08/11] Update TestcontainersConfiguration.java --- .../utility/TestcontainersConfiguration.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/core/src/main/java/org/testcontainers/utility/TestcontainersConfiguration.java b/core/src/main/java/org/testcontainers/utility/TestcontainersConfiguration.java index 3fbfd8db0a8..b7ef772fcf5 100644 --- a/core/src/main/java/org/testcontainers/utility/TestcontainersConfiguration.java +++ b/core/src/main/java/org/testcontainers/utility/TestcontainersConfiguration.java @@ -89,6 +89,15 @@ public String getDockerClientStrategyClassName() { return (String) environmentProperties.get("docker.client.strategy"); } + /** + * + * @deprecated we no longer have different transport types + */ + @Deprecated + public String getTransportType() { + return properties.getProperty("transport.type", "okhttp"); + } + public Integer getImagePullPauseTimeout() { return Integer.parseInt((String) properties.getOrDefault("pull.pause.timeout", "30")); } From bf790d2c15cc0bfb688955e0d5660f80336b8bae Mon Sep 17 00:00:00 2001 From: Sergei Egorov Date: Mon, 26 Aug 2019 10:49:09 +0200 Subject: [PATCH 09/11] Add test for the reuse env property --- .../utility/TestcontainersConfigurationTest.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/core/src/test/java/org/testcontainers/utility/TestcontainersConfigurationTest.java b/core/src/test/java/org/testcontainers/utility/TestcontainersConfigurationTest.java index ad2a0be6cd3..49a378554f1 100644 --- a/core/src/test/java/org/testcontainers/utility/TestcontainersConfigurationTest.java +++ b/core/src/test/java/org/testcontainers/utility/TestcontainersConfigurationTest.java @@ -37,6 +37,17 @@ public void shouldReadDockerClientStrategyFromEnvironmentOnly() { assertEquals("Docker client strategy is changed", "foo", newConfig().getDockerClientStrategyClassName()); } + @Test + public void shouldReadReuseFromEnvironmentOnly() { + assertFalse("no reuse by default", newConfig().environmentSupportsReuse()); + + classpathProperties.setProperty("testcontainers.reuse.enable", "true"); + assertFalse("reuse is not affected by classpath properties", newConfig().environmentSupportsReuse()); + + environmentProperties.setProperty("testcontainers.reuse.enable", "true"); + assertTrue("reuse enabled", newConfig().environmentSupportsReuse()); + } + private TestcontainersConfiguration newConfig() { return new TestcontainersConfiguration(environmentProperties, classpathProperties); } From 63b276ea742f0599a66f2da43e87171cb1541c56 Mon Sep 17 00:00:00 2001 From: Sergei Egorov Date: Wed, 23 Oct 2019 21:45:49 +0200 Subject: [PATCH 10/11] Add `@UnstableAPI` annotation --- .../java/org/testcontainers/UnstableAPI.java | 21 +++++++++++++++++++ .../containers/GenericContainer.java | 4 ++++ .../utility/TestcontainersConfiguration.java | 2 ++ .../testcontainers/jdbc/ConnectionUrl.java | 2 ++ 4 files changed, 29 insertions(+) create mode 100644 core/src/main/java/org/testcontainers/UnstableAPI.java diff --git a/core/src/main/java/org/testcontainers/UnstableAPI.java b/core/src/main/java/org/testcontainers/UnstableAPI.java new file mode 100644 index 00000000000..21fd2c3176c --- /dev/null +++ b/core/src/main/java/org/testcontainers/UnstableAPI.java @@ -0,0 +1,21 @@ +package org.testcontainers; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Marks that the annotated API is a subject to change and SHOULD NOT be considered + * a stable API. + */ +@Retention(RetentionPolicy.SOURCE) +@Target({ + ElementType.TYPE, + ElementType.METHOD, + ElementType.FIELD, +}) +@Documented +public @interface UnstableAPI { +} diff --git a/core/src/main/java/org/testcontainers/containers/GenericContainer.java b/core/src/main/java/org/testcontainers/containers/GenericContainer.java index 7ae9258e658..0dea429bde8 100644 --- a/core/src/main/java/org/testcontainers/containers/GenericContainer.java +++ b/core/src/main/java/org/testcontainers/containers/GenericContainer.java @@ -38,6 +38,7 @@ import org.rnorth.visibleassertions.VisibleAssertions; import org.slf4j.Logger; import org.testcontainers.DockerClientFactory; +import org.testcontainers.UnstableAPI; import org.testcontainers.containers.output.OutputFrame; import org.testcontainers.containers.startupcheck.IsRunningStartupCheckStrategy; import org.testcontainers.containers.startupcheck.MinimumDurationRunningStartupCheckStrategy; @@ -306,6 +307,7 @@ protected void doStart() { } } + @UnstableAPI @SneakyThrows protected boolean canBeReused() { for (Class type = getClass(); type != GenericContainer.class; type = type.getSuperclass()) { @@ -416,6 +418,7 @@ private void tryStart(Instant startedAt) { } } + @UnstableAPI @SneakyThrows(JsonProcessingException.class) final String hash(CreateContainerCmd createCommand) { // TODO add Testcontainers' version to the hash @@ -1333,6 +1336,7 @@ public SELF withTmpFs(Map mapping) { return self(); } + @UnstableAPI public SELF withReuse(boolean reusable) { this.shouldBeReused = reusable; return self(); diff --git a/core/src/main/java/org/testcontainers/utility/TestcontainersConfiguration.java b/core/src/main/java/org/testcontainers/utility/TestcontainersConfiguration.java index 14ccf23b0df..1187d1b3a44 100644 --- a/core/src/main/java/org/testcontainers/utility/TestcontainersConfiguration.java +++ b/core/src/main/java/org/testcontainers/utility/TestcontainersConfiguration.java @@ -2,6 +2,7 @@ import lombok.*; import lombok.extern.slf4j.Slf4j; +import org.testcontainers.UnstableAPI; import java.io.*; import java.net.MalformedURLException; @@ -85,6 +86,7 @@ public boolean isDisableChecks() { return Boolean.parseBoolean((String) environmentProperties.getOrDefault("checks.disable", "false")); } + @UnstableAPI public boolean environmentSupportsReuse() { return Boolean.parseBoolean((String) environmentProperties.getOrDefault("testcontainers.reuse.enable", "false")); } diff --git a/modules/jdbc/src/main/java/org/testcontainers/jdbc/ConnectionUrl.java b/modules/jdbc/src/main/java/org/testcontainers/jdbc/ConnectionUrl.java index 62f9ec08958..4c590474bf7 100644 --- a/modules/jdbc/src/main/java/org/testcontainers/jdbc/ConnectionUrl.java +++ b/modules/jdbc/src/main/java/org/testcontainers/jdbc/ConnectionUrl.java @@ -13,6 +13,7 @@ import lombok.AllArgsConstructor; import lombok.EqualsAndHashCode; import lombok.Getter; +import org.testcontainers.UnstableAPI; import static java.util.stream.Collectors.toMap; @@ -49,6 +50,7 @@ public class ConnectionUrl { private Optional initScriptPath = Optional.empty(); + @UnstableAPI private boolean reusable = false; private Optional initFunction = Optional.empty(); From e793eb0b82963f785fcc802734eb1e7f994b8225 Mon Sep 17 00:00:00 2001 From: Sergei Egorov Date: Wed, 23 Oct 2019 21:53:28 +0200 Subject: [PATCH 11/11] Add `VisibleForTesting` on `findContainerForReuse` --- .../java/org/testcontainers/containers/GenericContainer.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/src/main/java/org/testcontainers/containers/GenericContainer.java b/core/src/main/java/org/testcontainers/containers/GenericContainer.java index 0dea429bde8..5b34a6e40c9 100644 --- a/core/src/main/java/org/testcontainers/containers/GenericContainer.java +++ b/core/src/main/java/org/testcontainers/containers/GenericContainer.java @@ -16,6 +16,7 @@ import com.github.dockerjava.api.model.PortBinding; import com.github.dockerjava.api.model.Volume; import com.github.dockerjava.api.model.VolumesFrom; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Strings; import com.google.common.collect.ImmutableMap; import lombok.AccessLevel; @@ -430,6 +431,7 @@ final String hash(CreateContainerCmd createCommand) { return DigestUtils.sha1Hex(commandJson); } + @VisibleForTesting Optional findContainerForReuse(String hash) { // TODO locking return dockerClient.listContainersCmd()