Add a module collecting the results of repository rules

We allow repository rules to return a reproducible version of themselves,
or a list of fully reproducible rules they expand to. Add a module collecting
all those answers, logging them at each invocation, if requested; this
collection can also be used by the upcoming 'sync' command to generate
a WORKSPACE.resolved file.

Change-Id: Iac1358de1b74633810d300ba2bf45bba8b3992dc
PiperOrigin-RevId: 195427096

src/main/java/com/google/devtools/build/lib/bazel/Bazel.java

@@ -42,6 +42,7 @@ public final class Bazel {
           com.google.devtools.build.lib.bazel.BazelWorkspaceStatusModule.class,
           com.google.devtools.build.lib.bazel.BazelDiffAwarenessModule.class,
           com.google.devtools.build.lib.bazel.BazelRepositoryModule.class,
+          com.google.devtools.build.lib.bazel.repository.RepositoryResolvedModule.class,
           com.google.devtools.build.lib.bazel.SpawnLogModule.class,
           com.google.devtools.build.lib.ssd.SsdModule.class,
           com.google.devtools.build.lib.worker.WorkerModule.class,

src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryResolvedEvent.java

@@ -0,0 +1,109 @@
+// Copyright 2018 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.bazel.repository;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.events.ExtendedEventHandler.Postable;
+import com.google.devtools.build.lib.packages.Attribute;
+import com.google.devtools.build.lib.packages.Info;
+import com.google.devtools.build.lib.packages.Rule;
+import com.google.devtools.build.lib.syntax.EvalException;
+import com.google.devtools.build.lib.syntax.Runtime;
+import java.util.Map;
+
+/**
+ * Event indicating that a repository rule was executed, together with the return value of the rule.
+ */
+public class RepositoryResolvedEvent implements Postable {
+  public static final String ORIGINAL_RULE_CLASS = "original_rule_class";
+  public static final String ORIGINAL_ATTRIBUTES = "original_attributes";
+  public static final String RULE_CLASS = "rule_class";
+  public static final String ATTRIBUTES = "attributes";
+  public static final String REPOSITORIES = "repositories";
+
+  /**
+   * The entry for WORSPACE.resolved corresponding to that rule invocation.
+   *
+   * <p>It will always be a dict with three entries <ul>
+   *  <li> the original rule class (as String, e.g., "@bazel_tools//:git.bzl%git_repository")
+   *  <li> the original attributes (as dict, e.g., mapping "name" to "build_bazel"
+   *       and "remote" to "https://github.com/bazelbuild/bazel.git"), and
+   *  <li> a "repositories" entry; this is a list, often a single entry, of fully resolved
+   *       repositories the rule call expanded to (in the above example, the attributes entry
+   *       would have an additional "commit" and "shallow-since" entry).
+   * </ul>
+   */
+  private final Object resolvedInformation;
+
+  public RepositoryResolvedEvent(Rule rule, Info attrs, Object result) {
+    ImmutableMap.Builder<String, Object> builder = ImmutableMap.builder();
+
+    String originalClass =
+        rule.getRuleClassObject().getRuleDefinitionEnvironmentLabel() + "%" + rule.getRuleClass();
+    builder.put(ORIGINAL_RULE_CLASS, originalClass);
+
+    ImmutableMap.Builder<String, Object> origAttrBuilder = ImmutableMap.builder();
+    for (Attribute attr : rule.getAttributes()) {
+      String name = attr.getPublicName();
+      if (!name.startsWith("_")) {
+        // TODO(aehlig): filter out remaining attributes that cannot be set in a
+        // WORKSPACE file.
+        try {
+          Object value = attrs.getValue(name, Object.class);
+          // Only record explicit values, skip computed defaults
+          if (!(value instanceof Attribute.ComputedDefault)) {
+            origAttrBuilder.put(name, value);
+          }
+        } catch (EvalException e) {
+          // Do nothing, just ignore the value.
+        }
+      }
+    }
+    ImmutableMap<String, Object> origAttr = origAttrBuilder.build();
+    builder.put(ORIGINAL_ATTRIBUTES, origAttr);
+
+    if (result == Runtime.NONE) {
+      // Rule claims to be already reproducible, so wants to be called as is.
+      builder.put(
+          REPOSITORIES,
+          ImmutableList.<Object>of(
+              ImmutableMap.<String, Object>builder()
+                  .put(RULE_CLASS, originalClass)
+                  .put(ATTRIBUTES, origAttr)
+                  .build()));
+    } else if (result instanceof Map) {
+      // Rule claims that the returned (probably changed) arguments are a reproducible
+      // version of itself.
+      builder.put(
+          REPOSITORIES,
+          ImmutableList.<Object>of(
+              ImmutableMap.<String, Object>builder()
+                  .put(RULE_CLASS, originalClass)
+                  .put(ATTRIBUTES, result)
+                  .build()));
+    } else {
+      // TODO(aehlig): handle strings specially to allow encodings of the former
+      // values to be accepted as well.
+      builder.put(REPOSITORIES, result);
+    }
+
+    this.resolvedInformation = builder.build();
+  }
+
+  /** Return the entry for the given rule invocation in a format suitable for WORKSPACE.resolved. */
+  public Object getResolvedInformation() {
+    return resolvedInformation;
+  }
+}

src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryResolvedModule.java

@@ -0,0 +1,82 @@
+// Copyright 2018 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.bazel.repository;
+
+import com.google.common.base.Strings;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.eventbus.Subscribe;
+import com.google.common.io.Files;
+import com.google.devtools.build.lib.runtime.BlazeModule;
+import com.google.devtools.build.lib.runtime.Command;
+import com.google.devtools.build.lib.runtime.CommandEnvironment;
+import com.google.devtools.build.lib.syntax.Printer;
+import com.google.devtools.common.options.OptionsBase;
+import java.io.File;
+import java.io.IOException;
+import java.io.Writer;
+import java.nio.charset.StandardCharsets;
+import java.util.logging.Logger;
+
+/** Module providing the collection of the resolved values for the repository rules executed. */
+public final class RepositoryResolvedModule extends BlazeModule {
+  public static final String EXPORTED_NAME = "resolved";
+
+  private static final Logger logger = Logger.getLogger(RepositoryResolvedModule.class.getName());
+  private ImmutableList.Builder<Object> resultBuilder;
+  private String resolvedFile;
+
+  @Override
+  public Iterable<Class<? extends OptionsBase>> getCommandOptions(Command command) {
+    return ImmutableSet.of("fetch", "build", "query").contains(command.name())
+        ? ImmutableList.<Class<? extends OptionsBase>>of(RepositoryResolvedOptions.class)
+        : ImmutableList.<Class<? extends OptionsBase>>of();
+  }
+
+  @Override
+  public void beforeCommand(CommandEnvironment env) {
+    RepositoryResolvedOptions options =
+        env.getOptions().getOptions(RepositoryResolvedOptions.class);
+    if (options != null && !Strings.isNullOrEmpty(options.repositoryResolvedFile)) {
+      this.resolvedFile = options.repositoryResolvedFile;
+      env.getEventBus().register(this);
+      this.resultBuilder = new ImmutableList.Builder<>();
+    } else {
+      this.resolvedFile = null;
+    }
+  }
+
+  @Override
+  public void afterCommand() {
+    if (resolvedFile != null) {
+      try {
+        Writer writer = Files.newWriter(new File(resolvedFile), StandardCharsets.UTF_8);
+        // TODO(aehlig): pretty print
+        writer.write(EXPORTED_NAME + " = " + Printer.repr(resultBuilder.build()));
+        writer.close();
+      } catch (IOException e) {
+        logger.warning("IO Error writing to file " + resolvedFile + ": " + e);
+      }
+    }
+
+    this.resultBuilder = null;
+  }
+
+  @Subscribe
+  public void repositoryResolved(RepositoryResolvedEvent event) {
+    if (resultBuilder != null) {
+      resultBuilder.add(event.getResolvedInformation());
+    }
+  }
+}

src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryResolvedOptions.java

@@ -0,0 +1,35 @@
+// Copyright 2018 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.repository;
+
+import com.google.devtools.common.options.Option;
+import com.google.devtools.common.options.OptionDocumentationCategory;
+import com.google.devtools.common.options.OptionEffectTag;
+import com.google.devtools.common.options.OptionsBase;
+
+/** Options for handling the repository resolution */
+public class RepositoryResolvedOptions extends OptionsBase {
+
+  @Option(
+    name = "experimental_repository_resolved_file",
+    defaultValue = "",
+    documentationCategory = OptionDocumentationCategory.LOGGING,
+    effectTags = {OptionEffectTag.AFFECTS_OUTPUTS},
+    help =
+        "If non-empty, write a Skylark value with the resolved information of all Skylark"
+            + " respository rules that were executed."
+  )
+  public String repositoryResolvedFile;
+}

src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryFunction.java

@@ -18,6 +18,7 @@
 import com.google.common.collect.ImmutableMap;
 import com.google.devtools.build.lib.analysis.BlazeDirectories;
 import com.google.devtools.build.lib.analysis.RuleDefinition;
+import com.google.devtools.build.lib.bazel.repository.RepositoryResolvedEvent;
 import com.google.devtools.build.lib.bazel.repository.downloader.HttpDownloader;
 import com.google.devtools.build.lib.cmdline.Label;
 import com.google.devtools.build.lib.events.Event;
@@ -87,7 +88,10 @@ public RepositoryDirectoryValue.Builder fetch(Rule rule, Path outputDirectory,
         // means the rule might get restarted for legitimate reasons.
       }
 
-      // This has side-effect, we don't care about the output.
+      // This rule is mainly executed for its side effect. Nevertheless, the return value is
+      // of importance, as it provides information on how the call has to be modified to be a
+      // reproducible rule.
+      //
       // Also we do a lot of stuff in there, maybe blocking operations and we should certainly make
       // it possible to return null and not block but it doesn't seem to be easy with Skylark
       // structure as it is.
@@ -101,6 +105,8 @@ public RepositoryDirectoryValue.Builder fetch(Rule rule, Path outputDirectory,
         env.getListener()
             .handle(Event.info("Repository rule '" + rule.getName() + "' returned: " + retValue));
       }
+      env.getListener()
+          .post(new RepositoryResolvedEvent(rule, skylarkRepositoryContext.getAttr(), retValue));
     } catch (EvalException e) {
       if (e.getCause() instanceof RepositoryMissingDependencyException) {
         // A dependency is missing, cleanup and returns null

src/test/shell/bazel/BUILD

@@ -385,6 +385,13 @@ sh_test(
     data = [":test-deps"],
 )
 
+sh_test(
+    name = "workspace_resolved_test",
+    size = "medium",
+    srcs = ["workspace_resolved_test.sh"],
+    data = [":test-deps"],
+)
+
 sh_test(
     name = "cc_integration_test",
     size = "medium",

src/test/shell/bazel/workspace_resolved_test.sh

@@ -0,0 +1,80 @@
+#!/bin/bash
+#
+# Copyright 2018 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Load the test setup defined in the parent directory
+CURRENT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "${CURRENT_DIR}/../integration_test_setup.sh" \
+  || { echo "integration_test_setup.sh not found!" >&2; exit 1; }
+
+test_result_recorded() {
+  mkdir fetchrepo
+  cd fetchrepo
+  cat > rule.bzl <<'EOF'
+def _rule_impl(ctx):
+  ctx.symlink(ctx.attr.build_file, "BUILD")
+  return {"build_file": ctx.attr.build_file, "extra_arg": "foobar"}
+
+trivial_rule = repository_rule(
+  implementation = _rule_impl,
+  attrs = { "build_file" : attr.label() },
+)
+
+EOF
+  cat > ext.BUILD <<'EOF'
+genrule(
+  name = "foo",
+  outs = ["foo.txt"],
+  cmd = "echo bar > $@",
+)
+EOF
+  touch BUILD
+  cat  > WORKSPACE <<'EOF'
+load("//:rule.bzl", "trivial_rule")
+trivial_rule(
+  name = "ext",
+  build_file = "//:ext.BUILD",
+)
+EOF
+
+  bazel clean --expunge
+  bazel build --experimental_repository_resolved_file=../repo.bzl @ext//... \
+      || fail "Expected success"
+
+  # Verify that bazel can read the generated repo.bzl file and that it contains
+  # the expected information
+  cd ..
+  mkdir analysisrepo
+  mv repo.bzl analysisrepo
+  cd analysisrepo
+  touch WORKSPACE
+  cat > BUILD <<'EOF'
+load("//:repo.bzl", "resolved")
+
+[ genrule(
+    name = "out",
+    outs = ["out.txt"],
+    cmd = "echo %s > $@" % entry["repositories"][0]["attributes"]["extra_arg"],
+  ) for entry in resolved if entry["original_rule_class"] == "//:rule.bzl%trivial_rule"
+]
+EOF
+  cat BUILD
+  bazel build //:out || fail "Expected success"
+  grep "foobar" `bazel info bazel-genfiles`/out.txt \
+      || fail "Did not find the expected value"
+
+}
+
+run_suite "workspace_resolved_test tests"