Skip to content

Commit

Permalink
Repositories can only be accessed in projects that define them in the…
Browse files Browse the repository at this point in the history
…ir WORKSPACE file

This is prep for #1943 - hierarchical workspace loading.

RELNOTES[INC]: Remote repositories must define any remote repositories they
themselves use (e.g., if @x//:foo depends on @y//:bar, @y must be defined
in @x's WORKSPACE file).

PiperOrigin-RevId: 154295762
  • Loading branch information
kchodorow authored and vladmos committed Apr 26, 2017
1 parent 4db102c commit d5ee3b5
Show file tree
Hide file tree
Showing 8 changed files with 251 additions and 3 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,127 @@
// Copyright 2017 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package com.google.devtools.build.lib.rules.repository;

import com.google.auto.value.AutoValue;
import com.google.devtools.build.lib.cmdline.Label;
import com.google.devtools.build.lib.cmdline.RepositoryName;
import com.google.devtools.build.lib.packages.Rule;
import com.google.devtools.build.lib.skyframe.PackageLookupValue;
import com.google.devtools.build.lib.skyframe.SkyFunctions;
import com.google.devtools.build.lib.skyframe.WorkspaceFileValue;
import com.google.devtools.build.lib.util.Preconditions;
import com.google.devtools.build.lib.vfs.RootedPath;
import com.google.devtools.build.skyframe.SkyFunction;
import com.google.devtools.build.skyframe.SkyKey;
import com.google.devtools.build.skyframe.SkyValue;
import javax.annotation.Nullable;

/**
* Used for repositories that are declared in other repositories' WORKSPACE files.
*/
public class RepositoryVisibilityFunction implements SkyFunction {
public static SkyKey key(RepositoryName parent, RepositoryName child) {
return SkyKey.create(SkyFunctions.REPOSITORY_DEPENDENCY, Key.create(parent, child));
}

@Nullable
@Override
public SkyValue compute(SkyKey skyKey, Environment env) throws InterruptedException {
Key key = (Key) skyKey.argument();
RepositoryDirectoryValue parentDir = (RepositoryDirectoryValue) env.getValue(
RepositoryDirectoryValue.key(key.parent()));
if (env.valuesMissing()) {
return null;
}

SkyKey parentWorkspaceKey;
if (parentDir.repositoryExists()) {
parentWorkspaceKey = WorkspaceFileValue.key(
RootedPath.toRootedPath(parentDir.getPath(), Label.EXTERNAL_PACKAGE_FILE_NAME));
} else {
// This is kind of hacky: the main repository won't exist under output_base/external, so RDF
// won't find it above. However, we know that the parent repository has to exist, so we'll
// just assume it's the main repository if RDF couldn't find it.
PackageLookupValue packageLookupValue = (PackageLookupValue) env.getValue(
PackageLookupValue.key(Label.EXTERNAL_PACKAGE_IDENTIFIER));
if (env.valuesMissing()) {
return null;
}
Preconditions.checkState(packageLookupValue.packageExists());
parentWorkspaceKey = WorkspaceFileValue.key(
RootedPath.toRootedPath(packageLookupValue.getRoot(), Label.EXTERNAL_PACKAGE_FILE_NAME));
}

// This just looks for the child repo name. It doesn't care if the name is used for a different
// repository (either a different type or a different path/url/commit/whichever) than is
// actually being used. For example, if someone has a copy of Boost on their system that
// they're using but a library they're depending downloads Boost, we just want "everyone" to
// use the local copy of Boost, not error out. This is the check that the library actually
// declares a repository "dependency" on @boost.
while (true) {
WorkspaceFileValue parentWorkspace = (WorkspaceFileValue) env.getValue(parentWorkspaceKey);
if (env.valuesMissing()) {
return null;
}

Rule child = parentWorkspace.getPackage().getRule(key.child().strippedName());
if (child == null) {
if (parentWorkspace.hasNext()) {
parentWorkspaceKey = parentWorkspace.next();
} else {
break;
}
} else {
return RepositoryVisibilityValue.OK;
}
}
return RepositoryVisibilityValue.NOT_FOUND;
}

@Nullable
@Override
public String extractTag(SkyKey skyKey) {
return null;
}

/**
* Represents a parent repository and a child repository that we expect to be defined in the
* parent's WORKSPACE file.
*/
@AutoValue
public abstract static class Key {
public static Key create(RepositoryName parent, RepositoryName child) {
return new AutoValue_RepositoryVisibilityFunction_Key(parent, child);
}

public abstract RepositoryName parent();
public abstract RepositoryName child();
}

/**
* Returns if the repository definition was found or not.
*/
public static class RepositoryVisibilityValue implements SkyValue {
private static RepositoryVisibilityValue OK = new RepositoryVisibilityValue();
private static RepositoryVisibilityValue NOT_FOUND = new RepositoryVisibilityValue();

private RepositoryVisibilityValue() {
}

public boolean ok() {
return this == OK;
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import com.google.common.base.VerifyException;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.common.collect.LinkedHashMultimap;
import com.google.common.collect.LinkedListMultimap;
Expand Down Expand Up @@ -50,6 +51,7 @@
import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException;
import com.google.devtools.build.lib.analysis.config.PatchTransition;
import com.google.devtools.build.lib.cmdline.Label;
import com.google.devtools.build.lib.cmdline.RepositoryName;
import com.google.devtools.build.lib.collect.nestedset.NestedSet;
import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
import com.google.devtools.build.lib.collect.nestedset.Order;
Expand All @@ -69,6 +71,8 @@
import com.google.devtools.build.lib.packages.SkylarkProviderIdentifier;
import com.google.devtools.build.lib.packages.Target;
import com.google.devtools.build.lib.packages.TargetUtils;
import com.google.devtools.build.lib.rules.repository.RepositoryVisibilityFunction;
import com.google.devtools.build.lib.rules.repository.RepositoryVisibilityFunction.RepositoryVisibilityValue;
import com.google.devtools.build.lib.skyframe.AspectFunction.AspectCreationException;
import com.google.devtools.build.lib.skyframe.AspectValue.AspectKey;
import com.google.devtools.build.lib.skyframe.SkyframeExecutor.BuildViewProvider;
Expand All @@ -91,6 +95,7 @@
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.Semaphore;
Expand Down Expand Up @@ -336,6 +341,35 @@ static OrderedSetMultimap<Attribute, ConfiguredTarget> computeDependencies(
throw new DependencyEvaluationException(e);
}

RepositoryName ctgRepository = ctgValue.getLabel().getPackageIdentifier().getRepository();
ImmutableSet.Builder<SkyKey> pairsToCheck = ImmutableSet.builder();
for (Dependency dep : depValueNames.values()) {
RepositoryName depRepository = dep.getLabel().getPackageIdentifier().getRepository();
if (ctgRepository.equals(depRepository) || depRepository.isMain()) {
continue;
}
pairsToCheck.add(RepositoryVisibilityFunction.key(ctgRepository, depRepository));
}
Map<SkyKey, SkyValue> pairs = env.getValues(pairsToCheck.build());
if (env.valuesMissing()) {
return null;
}
boolean hadError = false;
for (Entry<SkyKey, SkyValue> entry : pairs.entrySet()) {
if (!((RepositoryVisibilityValue) entry.getValue()).ok()) {
RepositoryVisibilityFunction.Key key =
(RepositoryVisibilityFunction.Key) entry.getKey().argument();
String message = ctgValue.getLabel() + " has a dependency on "
+ key.child() + " but does not define " + key.child() + " in its WORKSPACE";
env.getListener().handle(Event.error(message));
hadError = true;
}
}
if (hadError) {
throw new DependencyEvaluationException(
new ConfiguredValueCreationException("Missing external repository definitions"));
}

// Trim each dep's configuration so it only includes the fragments needed by its transitive
// closure (only dynamic configurations support this).
if (useDynamicConfigurations(ctgValue.getConfiguration())) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,8 @@ public final class SkyFunctions {
SkyFunctionName.create("ACTION_TEMPLATE_EXPANSION");
public static final SkyFunctionName LOCAL_REPOSITORY_LOOKUP =
SkyFunctionName.create("LOCAL_REPOSITORY_LOOKUP");
public static final SkyFunctionName REPOSITORY_DEPENDENCY =
SkyFunctionName.create("REPOSITORY_DEPENDENCY");

public static Predicate<SkyKey> isSkyFunction(final SkyFunctionName functionName) {
return new Predicate<SkyKey>() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,7 @@
import com.google.devtools.build.lib.pkgcache.TestFilter;
import com.google.devtools.build.lib.pkgcache.TransitivePackageLoader;
import com.google.devtools.build.lib.profiler.AutoProfiler;
import com.google.devtools.build.lib.rules.repository.RepositoryVisibilityFunction;
import com.google.devtools.build.lib.skyframe.AspectValue.AspectValueKey;
import com.google.devtools.build.lib.skyframe.DirtinessCheckerUtils.FileDirtinessChecker;
import com.google.devtools.build.lib.skyframe.ExternalFilesHelper.ExternalFileAction;
Expand Down Expand Up @@ -446,6 +447,7 @@ private ImmutableMap<SkyFunctionName, SkyFunction> skyFunctions(
SkyFunctions.ACTION_TEMPLATE_EXPANSION,
new ActionTemplateExpansionFunction(removeActionsAfterEvaluation));
map.put(SkyFunctions.LOCAL_REPOSITORY_LOOKUP, new LocalRepositoryLookupFunction());
map.put(SkyFunctions.REPOSITORY_DEPENDENCY, new RepositoryVisibilityFunction());
map.putAll(extraSkyFunctions);
return map.build();
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -551,7 +551,9 @@ public void testCcLibraryExternalIncludesNotWarned() throws Exception {
new ModifiedFileSet.Builder().modify(PathFragment.create("WORKSPACE")).build(),
rootDirectory);
FileSystemUtils.createDirectoryAndParents(scratch.resolve("/foo/bar"));
scratch.file("/foo/WORKSPACE", "workspace(name = 'pkg')");
scratch.file("/foo/WORKSPACE",
"workspace(name = 'pkg')",
"local_repository(name = 'bazel_tools', path = '/whatever')");
scratch.file(
"/foo/bar/BUILD",
"cc_library(name = 'lib',",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -149,7 +149,9 @@ public void outputDirectoryForProtoCompileAction_externalRepos() throws Exceptio
scratch.file(
"x/BUILD", "cc_proto_library(name = 'foo_cc_proto', deps = ['@bla//foo:bar_proto'])");

scratch.file("/bla/WORKSPACE");
scratch.file("/bla/WORKSPACE",
"local_repository(name = 'com_google_protobuf', path = '/whatever')",
"local_repository(name = 'com_google_protobuf_cc', path = '/whatever')");
// Create the rule '@bla//foo:bar_proto'.
scratch.file(
"/bla/foo/BUILD",
Expand Down
5 changes: 4 additions & 1 deletion src/test/shell/bazel/external_correctness_test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -176,12 +176,15 @@ function test_refs_btwn_repos() {
REMOTE1=$TEST_TMPDIR/remote1
REMOTE2=$TEST_TMPDIR/remote2
mkdir -p $REMOTE1 $REMOTE2
touch $REMOTE1/WORKSPACE $REMOTE2/WORKSPACE
touch $REMOTE1/WORKSPACE
cat > $REMOTE1/input <<EOF
1.0
EOF
cat > $REMOTE1/BUILD <<EOF
exports_files(['input'])
EOF
cat > $REMOTE2/WORKSPACE <<EOF
local_repository(name = "remote1", path = "/whatever")
EOF
cat > $REMOTE2/BUILD <<EOF
genrule(
Expand Down
76 changes: 76 additions & 0 deletions src/test/shell/bazel/workspace_test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -190,6 +190,9 @@ EOF
touch foo/baz
cat > bar/WORKSPACE <<EOF
workspace(name = "bar")
# Needs to be defined, since @foo is referenced from the genrule.
local_repository(name = "foo", path = "/whatever")
EOF
cat > bar/BUILD <<EOF
genrule(
Expand Down Expand Up @@ -243,4 +246,77 @@ EOF
assert_contains "override" bazel-genfiles/external/o/gen.out
}

function test_direct_deps() {
REPO1="$PWD/repo1"
REPO2="$PWD/repo2"
mkdir -p "$REPO1" "$REPO2"

# repo1 has dependencies on the main repo and repo2.
cat > "$REPO1/WORKSPACE" <<EOF
workspace(name = "repo1")
local_repository(name = "repo2", path = "/whatever")
# Definition of main_repo purposely omitted to make build fail.
EOF
cat > "$REPO1/BUILD" << EOF
genrule(
name = "bar",
srcs = ["@repo2//:baz", "@main_repo//:qux"],
outs = ["bar.out"],
cmd = "echo \$(SRCS) > \$@",
visibility = ["//visibility:public"],
)
EOF

# repo2 has no dependencies.
touch "$REPO2/WORKSPACE"
cat > "$REPO2/BUILD" << EOF
genrule(
name = "baz",
outs = ["baz.out"],
cmd = "echo 'baz' > \$@",
visibility = ["//visibility:public"],
)
EOF

# The main repo has dependencies on repo1.
cat > WORKSPACE << EOF
workspace(name = "main_repo")
local_repository(name = "repo1", path = "$REPO1")
# TODO: move this to repo1/WORKSPACE once hierarchical workspaces work.
local_repository(name = "repo2", path = "$REPO2")
EOF
cat > BUILD <<EOF
exports_files(["qux"])
genrule(
name = "foo",
srcs = ["@repo1//:bar"],
outs = ["foo.out"],
cmd = "echo 'hi' > \$@",
)
EOF
touch qux

bazel build //:foo &> $TEST_log && fail "Expected missing main_repo"
expect_log "@repo1//:bar has a dependency on @main_repo but does not define @main_repo in its WORKSPACE"

cat > "$REPO1/WORKSPACE" <<EOF
workspace(name = "repo1")
local_repository(name = "main_repo", path = "/whatever")
# Definition of repo2 purposely omitted to make build fail.
EOF
bazel build //:foo &> $TEST_log && fail "Expected missing repo2"
expect_log "@repo1//:bar has a dependency on @repo2 but does not define @repo2 in its WORKSPACE"

cat > "$REPO1/WORKSPACE" <<EOF
workspace(name = "repo1")
local_repository(name = "main_repo", path = "/whatever")
local_repository(name = "repo2", path = "/whatever")
EOF
bazel build //:foo &> $TEST_log || fail "Expected success"
}

run_suite "workspace tests"

0 comments on commit d5ee3b5

Please sign in to comment.