Skip to content

Commit

Permalink
Adds entries for all directories in layer tarball. (#891)
Browse files Browse the repository at this point in the history
  • Loading branch information
@coollog
coollog authored Aug 29, 2018
1 parent 17425fc commit f67186e
Show file tree
Hide file tree
Showing 4 changed files with 147 additions and 53 deletions.
99 changes: 77 additions & 22 deletions jib-core/src/main/java/com/google/cloud/tools/jib/image/ReproducibleLayerBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,19 @@

import com.google.cloud.tools.jib.filesystem.DirectoryWalker;
import com.google.cloud.tools.jib.tar.TarStreamBuilder;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Verify;
import com.google.common.collect.ImmutableList;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;

/**
Expand All @@ -34,41 +40,80 @@
*/
public class ReproducibleLayerBuilder {

/**
* Holds a list of {@link TarArchiveEntry}s with unique extraction paths. The list also includes
* all parent directories for each extraction path.
*/
private static class UniqueTarArchiveEntries {

/**
* Uses the current directory to act as the file input to TarArchiveEntry (since all directories
* are treated the same in {@link TarArchiveEntry#TarArchiveEntry(File, String)}, except for
* modification time, which is wiped away in {@link #build}).
*/
private static final File DIRECTORY_FILE = Paths.get(".").toFile();

private final List<TarArchiveEntry> entries = new ArrayList<>();
private final Set<String> names = new HashSet<>();

/**
* Adds a {@link TarArchiveEntry} if its extraction path does not exist yet. Also adds all of
* the parent directories on the extraction path.
*
* @param tarArchiveEntry the {@link TarArchiveEntry}
*/
private void add(TarArchiveEntry tarArchiveEntry) {
if (names.contains(tarArchiveEntry.getName())) {
return;
}

// Adds all directories along extraction paths to explicitly set permissions for those
// directories.
Path namePath = Paths.get(tarArchiveEntry.getName());
if (namePath.getParent() != namePath.getRoot()) {
add(new TarArchiveEntry(DIRECTORY_FILE, namePath.getParent().toString()));
}

entries.add(tarArchiveEntry);
names.add(tarArchiveEntry.getName());
}

private List<TarArchiveEntry> getSortedEntries() {
List<TarArchiveEntry> sortedEntries = new ArrayList<>(entries);
sortedEntries.sort(Comparator.comparing(TarArchiveEntry::getName));
return sortedEntries;
}
}

/**
* Builds the {@link TarArchiveEntry}s for adding this {@link LayerEntry} to a tarball archive.
*
* @return the list of {@link TarArchiveEntry}
* @throws IOException if walking a source file that is a directory failed
*/
private static List<TarArchiveEntry> buildAsTarArchiveEntries(LayerEntry layerEntry)
throws IOException {
@VisibleForTesting
static List<TarArchiveEntry> buildAsTarArchiveEntries(LayerEntry layerEntry) throws IOException {
List<TarArchiveEntry> tarArchiveEntries = new ArrayList<>();

// Adds the files to extract relative to the extraction path.
for (Path sourceFile : layerEntry.getSourceFiles()) {
if (Files.isDirectory(sourceFile)) {
new DirectoryWalker(sourceFile)
.filterRoot()
.filter(path -> !Files.isDirectory(path))
.walk(
path -> {
/*
* Builds the same file path as in the source file for extraction. The iteration
* is necessary because the path needs to be in Unix-style.
*/
StringBuilder subExtractionPath =
new StringBuilder(layerEntry.getExtractionPath());
Path sourceFileRelativePath = sourceFile.getParent().relativize(path);
for (Path sourceFileRelativePathComponent : sourceFileRelativePath) {
subExtractionPath.append('/').append(sourceFileRelativePathComponent);
}
Path extractionPath =
Paths.get(layerEntry.getExtractionPath()).resolve(sourceFileRelativePath);
tarArchiveEntries.add(
new TarArchiveEntry(path.toFile(), subExtractionPath.toString()));
new TarArchiveEntry(path.toFile(), extractionPath.toString()));
});

} else {
Path extractionPath =
Paths.get(layerEntry.getExtractionPath()).resolve(sourceFile.getFileName());
TarArchiveEntry tarArchiveEntry =
new TarArchiveEntry(
sourceFile.toFile(),
layerEntry.getExtractionPath() + "/" + sourceFile.getFileName());
new TarArchiveEntry(sourceFile.toFile(), extractionPath.toString());
tarArchiveEntries.add(tarArchiveEntry);
}
}
Expand Down Expand Up @@ -101,26 +146,36 @@ public ReproducibleLayerBuilder addFiles(List<Path> sourceFiles, String extracti
* @throws IOException if walking the source files fails
*/
public UnwrittenLayer build() throws IOException {
List<TarArchiveEntry> filesystemEntries = new ArrayList<>();
UniqueTarArchiveEntries uniqueTarArchiveEntries = new UniqueTarArchiveEntries();

// Adds all the layer entries as tar entries.
for (LayerEntry layerEntry : layerEntries.build()) {
filesystemEntries.addAll(buildAsTarArchiveEntries(layerEntry));
List<LayerEntry> layerEntries = this.layerEntries.build();
for (LayerEntry layerEntry : layerEntries) {
// Converts layerEntry to list of TarArchiveEntrys.
List<TarArchiveEntry> tarArchiveEntries = buildAsTarArchiveEntries(layerEntry);
// Adds the entries to uniqueTarArchiveEntries, which makes sure all entries are unique and
// adds parent directories for each extraction path.
tarArchiveEntries.forEach(uniqueTarArchiveEntries::add);
}

// Sorts the entries by name.
filesystemEntries.sort(Comparator.comparing(TarArchiveEntry::getName));
// Gets the entries sorted by extraction path.
List<TarArchiveEntry> sortedFilesystemEntries = uniqueTarArchiveEntries.getSortedEntries();

Set<String> names = new HashSet<>();

// Adds all the files to a tar stream.
TarStreamBuilder tarStreamBuilder = new TarStreamBuilder();
for (TarArchiveEntry entry : filesystemEntries) {
for (TarArchiveEntry entry : sortedFilesystemEntries) {
// Strips out all non-reproducible elements from tar archive entries.
entry.setModTime(0);
entry.setGroupId(0);
entry.setUserId(0);
entry.setUserName("");
entry.setGroupName("");

Verify.verify(!names.contains(entry.getName()));
names.add(entry.getName());

tarStreamBuilder.addTarArchiveEntry(entry);
}

Expand Down
97 changes: 66 additions & 31 deletions jib-core/src/test/java/com/google/cloud/tools/jib/image/ReproducibleLayerBuilderTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@

import com.google.cloud.tools.jib.blob.Blob;
import com.google.cloud.tools.jib.blob.Blobs;
import com.google.cloud.tools.jib.filesystem.DirectoryWalker;
import com.google.common.collect.ImmutableList;
import com.google.common.io.CharStreams;
import com.google.common.io.Resources;
Expand All @@ -35,6 +34,7 @@
import java.nio.file.attribute.FileTime;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
import org.hamcrest.CoreMatchers;
Expand Down Expand Up @@ -72,8 +72,53 @@ private static void verifyNextTarArchiveEntry(
Assert.assertEquals(expectedString, extractedString);
}

/**
* Verifies that the next {@link TarArchiveEntry} in the {@link TarArchiveInputStream} is a
* directory with correct permissions.
*
* @param tarArchiveInputStream the {@link TarArchiveInputStream} to read from
* @param expectedExtractionPath the expected extraction path of the next entry
* @throws IOException if an I/O exception occurs
*/
private static void verifyNextTarArchiveEntryIsDirectory(
TarArchiveInputStream tarArchiveInputStream, String expectedExtractionPath)
throws IOException {
TarArchiveEntry extractionPathEntry = tarArchiveInputStream.getNextTarEntry();
Assert.assertEquals(expectedExtractionPath, extractionPathEntry.getName());
Assert.assertTrue(extractionPathEntry.isDirectory());
Assert.assertEquals(TarArchiveEntry.DEFAULT_DIR_MODE, extractionPathEntry.getMode());
}

@Rule public TemporaryFolder temporaryFolder = new TemporaryFolder();

@Test
public void testBuildAsTarArchiveEntries() throws URISyntaxException, IOException {
Path testDirectory = Paths.get(Resources.getResource("layer").toURI());
Path testFile = Paths.get(Resources.getResource("fileA").toURI());

List<TarArchiveEntry> tarArchiveEntries =
ReproducibleLayerBuilder.buildAsTarArchiveEntries(
new LayerEntry(ImmutableList.of(testDirectory, testFile), "/app/"));

List<TarArchiveEntry> expectedTarArchiveEntries =
ImmutableList.of(
new TarArchiveEntry(
testDirectory.resolve("a").resolve("b").resolve("bar").toFile(),
"/app/layer/a/b/bar"),
new TarArchiveEntry(
testDirectory.resolve("c").resolve("cat").toFile(), "/app/layer/c/cat"),
new TarArchiveEntry(testDirectory.resolve("foo").toFile(), "/app/layer/foo"),
new TarArchiveEntry(testFile.toFile(), "/app/fileA"));

Assert.assertEquals(expectedTarArchiveEntries.size(), tarArchiveEntries.size());
for (int entryIndex = 0; entryIndex < expectedTarArchiveEntries.size(); entryIndex++) {
TarArchiveEntry expectedTarArchiveEntry = expectedTarArchiveEntries.get(entryIndex);
TarArchiveEntry tarArchiveEntry = tarArchiveEntries.get(entryIndex);
Assert.assertEquals(expectedTarArchiveEntry.getFile(), tarArchiveEntry.getFile());
Assert.assertEquals(expectedTarArchiveEntry.getName(), tarArchiveEntry.getName());
}
}

@Test
public void testBuild() throws URISyntaxException, IOException {
Path layerDirectory = Paths.get(Resources.getResource("layer").toURI());
Expand All @@ -95,37 +140,27 @@ public void testBuild() throws URISyntaxException, IOException {
// Reads the file back.
try (TarArchiveInputStream tarArchiveInputStream =
new TarArchiveInputStream(Files.newInputStream(temporaryFile))) {
// Verifies that blobA was added.
verifyNextTarArchiveEntryIsDirectory(tarArchiveInputStream, "extract/");
verifyNextTarArchiveEntryIsDirectory(tarArchiveInputStream, "extract/here/");
verifyNextTarArchiveEntryIsDirectory(tarArchiveInputStream, "extract/here/apple/");
verifyNextTarArchiveEntry(tarArchiveInputStream, "extract/here/apple/blobA", blobA);

// Verifies that all the files have been added to the tarball stream.
ImmutableList<Path> layerDirectoryPaths =
new DirectoryWalker(layerDirectory).filter(path -> !path.equals(layerDirectory)).walk();
for (Path path : layerDirectoryPaths) {
TarArchiveEntry header = tarArchiveInputStream.getNextTarEntry();

StringBuilder expectedExtractionPath = new StringBuilder("extract/here/apple");
for (Path pathComponent : layerDirectory.getParent().relativize(path)) {
expectedExtractionPath.append("/").append(pathComponent);
}
// Check path-equality because there might be an appended backslash in the header
// filename.
Assert.assertEquals(
Paths.get(expectedExtractionPath.toString()), Paths.get(header.getName()));

// If is a normal file, checks that the file contents match.
if (Files.isRegularFile(path)) {
String expectedFileString = new String(Files.readAllBytes(path), StandardCharsets.UTF_8);

String extractedFileString =
CharStreams.toString(
new InputStreamReader(tarArchiveInputStream, StandardCharsets.UTF_8));

Assert.assertEquals(expectedFileString, extractedFileString);
}
}

// Verifies that blobA was added to the other location.
verifyNextTarArchiveEntryIsDirectory(tarArchiveInputStream, "extract/here/apple/layer/");
verifyNextTarArchiveEntryIsDirectory(tarArchiveInputStream, "extract/here/apple/layer/a/");
verifyNextTarArchiveEntryIsDirectory(tarArchiveInputStream, "extract/here/apple/layer/a/b/");
verifyNextTarArchiveEntry(
tarArchiveInputStream,
"extract/here/apple/layer/a/b/bar",
Paths.get(Resources.getResource("layer/a/b/bar").toURI()));
verifyNextTarArchiveEntryIsDirectory(tarArchiveInputStream, "extract/here/apple/layer/c/");
verifyNextTarArchiveEntry(
tarArchiveInputStream,
"extract/here/apple/layer/c/cat",
Paths.get(Resources.getResource("layer/c/cat").toURI()));
verifyNextTarArchiveEntry(
tarArchiveInputStream,
"extract/here/apple/layer/foo",
Paths.get(Resources.getResource("layer/foo").toURI()));
verifyNextTarArchiveEntryIsDirectory(tarArchiveInputStream, "extract/here/banana/");
verifyNextTarArchiveEntry(tarArchiveInputStream, "extract/here/banana/blobA", blobA);
}
}
Expand Down
2 changes: 2 additions & 0 deletions jib-gradle-plugin/CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ All notable changes to this project will be documented in this file.

### Fixed

- Corrects permissions for directories in the container filesystem ([#772](https://github.com/GoogleContainerTools/jib/pull/772))

## 0.9.9

### Added
Expand Down
2 changes: 2 additions & 0 deletions jib-maven-plugin/CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ All notable changes to this project will be documented in this file.

### Fixed

- Corrects permissions for directories in the container filesystem ([#772](https://github.com/GoogleContainerTools/jib/pull/772))

## 0.9.9

### Added
Expand Down

0 comments on commit f67186e

Please sign in to comment.