-
-
Notifications
You must be signed in to change notification settings - Fork 1.6k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Make it possible to run TestContainers inside a container #267
Changes from all commits
398a616
55d93ab
7471ffe
ab9a570
e6d8be0
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
distributionUrl=https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.3.9/apache-maven-3.3.9-bin.zip | ||
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -19,8 +19,8 @@ before_install: | |
- docker pull mysql:5.6 | ||
- docker pull mysql:5.5 | ||
- docker pull postgres:latest | ||
- docker pull selenium/standalone-chrome-debug:2.45.0 | ||
- docker pull selenium/standalone-firefox-debug:2.45.0 | ||
- docker pull selenium/standalone-chrome-debug:2.52.0 | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. classpath contains 2.52, but we were pulling 2.45 and then 2.52 There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 👍 |
||
- docker pull selenium/standalone-firefox-debug:2.52.0 | ||
- docker pull richnorth/vnc-recorder:latest | ||
- docker pull nginx:1.9.4 | ||
- docker pull dduportal/docker-compose:1.6.0 | ||
|
@@ -32,6 +32,15 @@ before_install: | |
|
||
script: | ||
- mvn -B test | ||
# Run Docker-in-Docker tests | ||
- | | ||
DOCKER_HOST=unix:///var/run/docker.sock DOCKER_TLS_VERIFY= docker run --rm \ | ||
-v "$HOME/.m2":/root/.m2/ \ | ||
-v /var/run/docker.sock:/var/run/docker.sock \ | ||
-v "$(pwd)":"$(pwd)" \ | ||
-w "$(pwd)" \ | ||
openjdk:8-jre \ | ||
./mvnw -B -pl core test -Dtest=*GenericContainerRuleTest | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. GenericContainerRuleTest covers most of the cases. Right now we can't use all the tests because some of them make incorrect assumptions There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Is the selenium container image just used here because it includes Java? It's potentially a bit misleading.. If so, I wouldn't worry about pulling an openjdk image and using that. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Ok, I'll change it :) Just wanted to save a few seconds of pulling |
||
- mvn -B test -f shade-test/pom.xml | ||
|
||
cache: | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,36 +1,35 @@ | ||
package org.testcontainers; | ||
|
||
import com.github.dockerjava.api.DockerClient; | ||
import com.github.dockerjava.api.command.CreateContainerResponse; | ||
import com.github.dockerjava.api.command.CreateContainerCmd; | ||
import com.github.dockerjava.api.exception.InternalServerErrorException; | ||
import com.github.dockerjava.api.exception.NotFoundException; | ||
import com.github.dockerjava.api.model.Frame; | ||
import com.github.dockerjava.api.model.Image; | ||
import com.github.dockerjava.api.model.Info; | ||
import com.github.dockerjava.api.model.Version; | ||
import com.github.dockerjava.core.command.LogContainerResultCallback; | ||
import com.github.dockerjava.core.command.PullImageResultCallback; | ||
import com.github.dockerjava.core.command.WaitContainerResultCallback; | ||
|
||
import lombok.Synchronized; | ||
import org.slf4j.Logger; | ||
import lombok.extern.slf4j.Slf4j; | ||
import org.testcontainers.dockerclient.*; | ||
|
||
import java.util.List; | ||
import java.util.Optional; | ||
import java.util.function.BiFunction; | ||
import java.util.function.Consumer; | ||
|
||
import static java.util.Arrays.asList; | ||
import static org.slf4j.LoggerFactory.getLogger; | ||
|
||
/** | ||
* Singleton class that provides initialized Docker clients. | ||
* <p> | ||
* The correct client configuration to use will be determined on first use, and cached thereafter. | ||
*/ | ||
@Slf4j | ||
public class DockerClientFactory { | ||
|
||
private static final String TINY_IMAGE = "alpine:3.2"; | ||
private static DockerClientFactory instance; | ||
private static final Logger LOGGER = getLogger(DockerClientFactory.class); | ||
|
||
// Cached client configuration | ||
private DockerClientProviderStrategy strategy; | ||
|
@@ -80,20 +79,29 @@ public DockerClient client() { | |
} | ||
|
||
strategy = DockerClientProviderStrategy.getFirstValidStrategy(CONFIGURATION_STRATEGIES); | ||
|
||
log.info("Docker host IP address is {}", strategy.getDockerHostIpAddress()); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. moved here to avoid the cycling dependency |
||
DockerClient client = strategy.getClient(); | ||
|
||
if (!preconditionsChecked) { | ||
Info dockerInfo = client.infoCmd().exec(); | ||
Version version = client.versionCmd().exec(); | ||
activeApiVersion = version.getApiVersion(); | ||
activeExecutionDriver = dockerInfo.getExecutionDriver(); | ||
LOGGER.info("Connected to docker: \n" + | ||
log.info("Connected to docker: \n" + | ||
" Server Version: " + dockerInfo.getServerVersion() + "\n" + | ||
" API Version: " + activeApiVersion + "\n" + | ||
" Operating System: " + dockerInfo.getOperatingSystem() + "\n" + | ||
" Total Memory: " + dockerInfo.getMemTotal() / (1024 * 1024) + " MB"); | ||
|
||
checkVersion(version.getVersion()); | ||
|
||
List<Image> images = client.listImagesCmd().exec(); | ||
// Pull the image we use to perform some checks | ||
if (images.stream().noneMatch(it -> it.getRepoTags() != null && asList(it.getRepoTags()).contains(TINY_IMAGE))) { | ||
client.pullImageCmd(TINY_IMAGE).exec(new PullImageResultCallback()).awaitSuccess(); | ||
} | ||
|
||
checkDiskSpaceAndHandleExceptions(client); | ||
preconditionsChecked = true; | ||
} | ||
|
@@ -121,7 +129,7 @@ private void checkDiskSpaceAndHandleExceptions(DockerClient client) { | |
} catch (NotEnoughDiskSpaceException e) { | ||
throw e; | ||
} catch (Exception e) { | ||
LOGGER.warn("Encountered and ignored error while checking disk space", e); | ||
log.warn("Encountered and ignored error while checking disk space", e); | ||
} | ||
} | ||
|
||
|
@@ -130,44 +138,47 @@ private void checkDiskSpaceAndHandleExceptions(DockerClient client) { | |
* @param client an active Docker client | ||
*/ | ||
private void checkDiskSpace(DockerClient client) { | ||
DiskSpaceUsage df = runInsideDocker(client, cmd -> cmd.withCmd("df", "-P"), (dockerClient, id) -> { | ||
String logResults = dockerClient.logContainerCmd(id) | ||
.withStdOut(true) | ||
.exec(new LogToStringContainerCallback()) | ||
.toString(); | ||
|
||
return parseAvailableDiskSpace(logResults); | ||
}); | ||
|
||
log.info("Disk utilization in Docker environment is {} ({} )", | ||
df.usedPercent.map(x -> x + "%").orElse("unknown"), | ||
df.availableMB.map(x -> x + " MB available").orElse("unknown available")); | ||
|
||
if (df.availableMB.orElseThrow(NotAbleToGetDiskSpaceUsageException::new) < 2048) { | ||
log.error("Docker environment has less than 2GB free - execution is unlikely to succeed so will be aborted."); | ||
throw new NotEnoughDiskSpaceException("Not enough disk space in Docker environment"); | ||
} | ||
} | ||
|
||
List<Image> images = client.listImagesCmd().exec(); | ||
if (!images.stream().anyMatch(it -> it.getRepoTags() != null && asList(it.getRepoTags()).contains("alpine:3.2"))) { | ||
PullImageResultCallback callback = client.pullImageCmd("alpine:3.2").exec(new PullImageResultCallback()); | ||
callback.awaitSuccess(); | ||
public <T> T runInsideDocker(Consumer<CreateContainerCmd> createContainerCmdConsumer, BiFunction<DockerClient, String, T> block) { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. wish I could make it module-level, but I can't because the consumer is in |
||
if (strategy == null) { | ||
client(); | ||
} | ||
// We can't use client() here because it might create an infinite loop | ||
return runInsideDocker(strategy.getClient(), createContainerCmdConsumer, block); | ||
} | ||
|
||
CreateContainerResponse createContainerResponse = client.createContainerCmd("alpine:3.2").withCmd("df", "-P").exec(); | ||
String id = createContainerResponse.getId(); | ||
private <T> T runInsideDocker(DockerClient client, Consumer<CreateContainerCmd> createContainerCmdConsumer, BiFunction<DockerClient, String, T> block) { | ||
CreateContainerCmd createContainerCmd = client.createContainerCmd(TINY_IMAGE); | ||
createContainerCmdConsumer.accept(createContainerCmd); | ||
String id = createContainerCmd.exec().getId(); | ||
|
||
client.startContainerCmd(id).exec(); | ||
|
||
LogContainerResultCallback callback = client.logContainerCmd(id).withStdOut(true).exec(new LogContainerCallback()); | ||
try { | ||
|
||
WaitContainerResultCallback waitCallback = new WaitContainerResultCallback(); | ||
client.waitContainerCmd(id).exec(waitCallback); | ||
waitCallback.awaitStarted(); | ||
|
||
callback.awaitCompletion(); | ||
String logResults = callback.toString(); | ||
|
||
DiskSpaceUsage df = parseAvailableDiskSpace(logResults); | ||
LOGGER.info("Disk utilization in Docker environment is {} ({} )", | ||
df.usedPercent.map(x -> x.toString() + "%").orElse("unknown"), | ||
df.availableMB.map(x -> x.toString() + " MB available").orElse("unknown available")); | ||
|
||
if (df.availableMB.orElseThrow(NotAbleToGetDiskSpaceUsageException::new) < 2048) { | ||
LOGGER.error("Docker environment has less than 2GB free - execution is unlikely to succeed so will be aborted."); | ||
throw new NotEnoughDiskSpaceException("Not enough disk space in Docker environment"); | ||
} | ||
} catch (InterruptedException e) { | ||
throw new RuntimeException(e); | ||
return block.apply(client, id); | ||
} finally { | ||
try { | ||
client.removeContainerCmd(id).withRemoveVolumes(true).withForce(true).exec(); | ||
} catch (NotFoundException | InternalServerErrorException ignored) { | ||
|
||
log.debug("", ignored); | ||
} | ||
} | ||
} | ||
|
@@ -231,18 +242,3 @@ private static class NotAbleToGetDiskSpaceUsageException extends RuntimeExceptio | |
} | ||
} | ||
} | ||
|
||
class LogContainerCallback extends LogContainerResultCallback { | ||
private final StringBuffer log = new StringBuffer(); | ||
|
||
@Override | ||
public void onNext(Frame frame) { | ||
log.append(new String(frame.getPayload())); | ||
super.onNext(frame); | ||
} | ||
|
||
@Override | ||
public String toString() { | ||
return log.toString(); | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,18 +1,53 @@ | ||
package org.testcontainers.dockerclient; | ||
|
||
import com.github.dockerjava.core.DockerClientConfig; | ||
import lombok.Getter; | ||
import lombok.extern.slf4j.Slf4j; | ||
import org.apache.commons.lang.StringUtils; | ||
import org.testcontainers.DockerClientFactory; | ||
|
||
import java.io.File; | ||
import java.util.Optional; | ||
|
||
@Slf4j | ||
public class DockerClientConfigUtils { | ||
|
||
// See https://github.com/docker/docker/blob/a9fa38b1edf30b23cae3eade0be48b3d4b1de14b/daemon/initlayer/setup_unix.go#L25 | ||
public static final boolean IN_A_CONTAINER = new File("/.dockerenv").exists(); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. the best way to check I found, libnetwork uses it as well, so it's here to stay |
||
|
||
@Getter(lazy = true) | ||
private static final Optional<String> detectedDockerHostIp = Optional | ||
.of(IN_A_CONTAINER) | ||
.filter(it -> it) | ||
.map(file -> DockerClientFactory.instance().runInsideDocker( | ||
cmd -> cmd.withCmd("sh", "-c", "ip route|awk '/default/ { print $3 }'"), | ||
(client, id) -> { | ||
try { | ||
return client.logContainerCmd(id) | ||
.withStdOut(true) | ||
.exec(new LogToStringContainerCallback()) | ||
.toString(); | ||
} catch (Exception e) { | ||
log.warn("Can't parse the default gateway IP", e); | ||
return null; | ||
} | ||
} | ||
)) | ||
.map(StringUtils::trimToEmpty) | ||
.filter(StringUtils::isNotBlank); | ||
|
||
public static String getDockerHostIpAddress(DockerClientConfig config) { | ||
switch (config.getDockerHost().getScheme()) { | ||
case "http": | ||
case "https": | ||
case "tcp": | ||
return config.getDockerHost().getHost(); | ||
case "unix": | ||
return "localhost"; | ||
default: | ||
return null; | ||
} | ||
return getDetectedDockerHostIp().orElseGet(() -> { | ||
switch (config.getDockerHost().getScheme()) { | ||
case "http": | ||
case "https": | ||
case "tcp": | ||
return config.getDockerHost().getHost(); | ||
case "unix": | ||
return "localhost"; | ||
default: | ||
return null; | ||
} | ||
}); | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -27,7 +27,6 @@ public void test() throws InvalidConfigurationException { | |
} | ||
|
||
LOGGER.info("Found docker client settings from environment"); | ||
LOGGER.info("Docker host IP address is {}", DockerClientConfigUtils.getDockerHostIpAddress(config)); | ||
} | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. moved to DockerClientFactory |
||
|
||
@Override | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,24 @@ | ||
package org.testcontainers.dockerclient; | ||
|
||
import com.github.dockerjava.api.model.Frame; | ||
import com.github.dockerjava.core.command.LogContainerResultCallback; | ||
|
||
public class LogToStringContainerCallback extends LogContainerResultCallback { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Am I right in thinking that this exists because we can't use our container logging abstraction (with If that's the case, should we reduce visibility of this class from public so as to reduce confusion with our higher-level logging API? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Wish we could :) This class is shared between different packages :( There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. No problem then - the type system will at least help people not use the wrong thing. |
||
private final StringBuffer log = new StringBuffer(); | ||
|
||
@Override | ||
public void onNext(Frame frame) { | ||
log.append(new String(frame.getPayload())); | ||
super.onNext(frame); | ||
} | ||
|
||
@Override | ||
public String toString() { | ||
try { | ||
awaitCompletion(); | ||
} catch (InterruptedException e) { | ||
throw new RuntimeException(e); | ||
} | ||
return log.toString(); | ||
} | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I've added Maven Wrapper ( https://github.com/takari/maven-wrapper ) to make it possible to run TC build inside any container with Java 8. It's also nice to have anyway :)
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Yep - good idea!