Skip to content

Commit

Permalink
Use local disk for GenericContainer#withCopyFileToContainer()
Browse files Browse the repository at this point in the history
This is an attempt to fix `OutOfMemoryError` (out of heap memory) when copying large files into a container via `GenericContainer#withCopyFileToContainer()`.

The previous implementation relied on an in-memory array backing a `ByteArrayInputStream` which represents the tar archive sent to the Docker daemon which is limited by the available heap memory of the JVM at the moment of execution.

By using the local disk (temporary directory), we can copy files into the container which are larger than the available heap memory of the JVM.
The caveat being that we are now limited by the available disk space in the temporary directory of the machine running Testcontainers.

Fixes #4203
  • Loading branch information
joschi committed Mar 5, 2024
1 parent ce3315d commit 3051f1f
Show file tree
Hide file tree
Showing 2 changed files with 65 additions and 10 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -22,14 +22,17 @@
import org.testcontainers.utility.MountableFile;
import org.testcontainers.utility.ThrowingFunction;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
Expand Down Expand Up @@ -345,21 +348,32 @@ default void copyFileToContainer(Transferable transferable, String containerPath
throw new IllegalStateException("copyFileToContainer can only be used with created / running container");
}

Path tempFile = Files.createTempFile("tc-", "-copy");

try (
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
TarArchiveOutputStream tarArchive = new TarArchiveOutputStream(byteArrayOutputStream)
OutputStream os = Files.newOutputStream(tempFile);
BufferedOutputStream bos = new BufferedOutputStream(os);
TarArchiveOutputStream tarArchive = new TarArchiveOutputStream(bos)
) {
tarArchive.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX);
tarArchive.setBigNumberMode(TarArchiveOutputStream.BIGNUMBER_POSIX);

transferable.transferTo(tarArchive, containerPath);
tarArchive.finish();

getDockerClient()
.copyArchiveToContainerCmd(getContainerId())
.withTarInputStream(new ByteArrayInputStream(byteArrayOutputStream.toByteArray()))
.withRemotePath("/")
.exec();
bos.flush();

try (
InputStream is = Files.newInputStream(tempFile);
BufferedInputStream bis = new BufferedInputStream(is)
) {
getDockerClient()
.copyArchiveToContainerCmd(getContainerId())
.withTarInputStream(bis)
.withRemotePath("/")
.exec();
}
} finally {
Files.deleteIfExists(tempFile);
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,15 +3,19 @@
import com.github.dockerjava.api.exception.NotFoundException;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.io.output.CountingOutputStream;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.testcontainers.TestImages;
import org.testcontainers.containers.Container;
import org.testcontainers.containers.GenericContainer;
import org.testcontainers.containers.startupcheck.OneShotStartupCheckStrategy;
import org.testcontainers.utility.MountableFile;

import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;

import static org.assertj.core.api.Assertions.assertThat;
Expand Down Expand Up @@ -40,6 +44,43 @@ public void copyFileToContainerFileTest() throws Exception {
}
}

@Test
public void copyLargeFilesToContainer() throws Exception {
File tempFile = temporaryFolder.newFile();
try (
GenericContainer alpineCopyToContainer = new GenericContainer(TestImages.ALPINE_IMAGE) //
.withCommand("sleep", "infinity")
) {
alpineCopyToContainer.start();
final long byteCount;
try (
FileOutputStream fos = new FileOutputStream(tempFile);
CountingOutputStream cos = new CountingOutputStream(fos);
BufferedOutputStream bos = new BufferedOutputStream(cos)
) {
for (int i = 0; i < 0x4000; i++) {
byte[] bytes = new byte[0xFFFF];
bos.write(bytes);
}
bos.flush();
byteCount = cos.getByteCount();
}
final MountableFile mountableFile = MountableFile.forHostPath(tempFile.getPath());
final String containerPath = "/test.bin";
alpineCopyToContainer.copyFileToContainer(mountableFile, containerPath);

final Container.ExecResult execResult = alpineCopyToContainer.execInContainer( //
"stat",
"-c",
"%s",
containerPath
);
assertThat(execResult.getStdout()).isEqualToIgnoringNewLines(Long.toString(byteCount));
} finally {
tempFile.delete();
}
}

@Test
public void copyFileToContainerFolderTest() throws Exception {
try (
Expand Down

0 comments on commit 3051f1f

Please sign in to comment.