From 83b2a1e5c966356658a87e11bf43f258ce4c222f Mon Sep 17 00:00:00 2001 From: Benjy Weinberger Date: Thu, 16 Aug 2018 18:00:26 -0600 Subject: [PATCH 1/2] Make requirements on codegen products optional. We declare product requirements in various places to force codegen to run before, e.g., dependency resolution. However currently these requirements are mandatory, so deregistering all codegen backends will cause the round engine to find no producers of these products, and fail. This commit changes these requirements to optional, so that if there are no codegen backends, we don't attempt to depend on their products. --- src/python/pants/backend/jvm/tasks/coursier_resolve.py | 6 ++++-- src/python/pants/backend/jvm/tasks/ivy_resolve.py | 6 ++++-- src/python/pants/backend/python/tasks/gather_sources.py | 2 +- .../pants/backend/python/tasks/python_binary_create.py | 3 ++- .../backend/python/tasks/resolve_requirements_task_base.py | 3 +++ 5 files changed, 14 insertions(+), 6 deletions(-) diff --git a/src/python/pants/backend/jvm/tasks/coursier_resolve.py b/src/python/pants/backend/jvm/tasks/coursier_resolve.py index ba6846eb5a4..a8f1f4cd3c9 100644 --- a/src/python/pants/backend/jvm/tasks/coursier_resolve.py +++ b/src/python/pants/backend/jvm/tasks/coursier_resolve.py @@ -661,8 +661,10 @@ def product_types(cls): @classmethod def prepare(cls, options, round_manager): super(CoursierResolve, cls).prepare(options, round_manager) - round_manager.require_data('java') - round_manager.require_data('scala') + # Codegen may inject extra resolvable deps, so make sure we have a product dependency + # on relevant codegen tasks, if any. + round_manager.optional_data('java') + round_manager.optional_data('scala') @classmethod def register_options(cls, register): diff --git a/src/python/pants/backend/jvm/tasks/ivy_resolve.py b/src/python/pants/backend/jvm/tasks/ivy_resolve.py index d75a9a07a81..593af582d88 100644 --- a/src/python/pants/backend/jvm/tasks/ivy_resolve.py +++ b/src/python/pants/backend/jvm/tasks/ivy_resolve.py @@ -69,8 +69,10 @@ def product_types(cls): @classmethod def prepare(cls, options, round_manager): super(IvyResolve, cls).prepare(options, round_manager) - round_manager.require_data('java') - round_manager.require_data('scala') + # Codegen may inject extra resolvable deps, so make sure we have a product dependency + # on relevant codegen tasks, if any. + round_manager.optional_data('java') + round_manager.optional_data('scala') def __init__(self, *args, **kwargs): super(IvyResolve, self).__init__(*args, **kwargs) diff --git a/src/python/pants/backend/python/tasks/gather_sources.py b/src/python/pants/backend/python/tasks/gather_sources.py index f41b688e5ba..de58c887b98 100644 --- a/src/python/pants/backend/python/tasks/gather_sources.py +++ b/src/python/pants/backend/python/tasks/gather_sources.py @@ -40,7 +40,7 @@ def product_types(cls): @classmethod def prepare(cls, options, round_manager): round_manager.require_data(PythonInterpreter) - round_manager.require_data('python') # For codegen. + round_manager.optional_data('python') # For codegen. def execute(self): targets = self._collect_source_targets() diff --git a/src/python/pants/backend/python/tasks/python_binary_create.py b/src/python/pants/backend/python/tasks/python_binary_create.py index a788f187222..1c40610ba92 100644 --- a/src/python/pants/backend/python/tasks/python_binary_create.py +++ b/src/python/pants/backend/python/tasks/python_binary_create.py @@ -51,8 +51,9 @@ def cache_target_dirs(self): @classmethod def prepare(cls, options, round_manager): + # See comment below for why we don't use the GatherSources.PYTHON_SOURCES product. round_manager.require_data(PythonInterpreter) - round_manager.require_data('python') # For codegen. + round_manager.optional_data('python') # For codegen. round_manager.optional_product(PythonRequirementLibrary) # For local dists. @staticmethod diff --git a/src/python/pants/backend/python/tasks/resolve_requirements_task_base.py b/src/python/pants/backend/python/tasks/resolve_requirements_task_base.py index 41a100a6043..fafc116964b 100644 --- a/src/python/pants/backend/python/tasks/resolve_requirements_task_base.py +++ b/src/python/pants/backend/python/tasks/resolve_requirements_task_base.py @@ -45,6 +45,9 @@ def _python_native_code_settings(self): def prepare(cls, options, round_manager): round_manager.require_data(PythonInterpreter) round_manager.optional_product(PythonRequirementLibrary) # For local dists. + # Codegen may inject extra resolvable deps, so make sure we have a product dependency + # on relevant codegen tasks, if any. + round_manager.optional_data('python') def resolve_requirements(self, interpreter, req_libs): """Requirements resolution for PEX files. From d3be59fdc6d03915d94b3cfffced6cb378ecb781 Mon Sep 17 00:00:00 2001 From: Benjy Weinberger Date: Thu, 16 Aug 2018 18:07:50 -0600 Subject: [PATCH 2/2] Minor unrelated documentation nits. These weren't worth a separate pull request. --- src/python/pants/goal/run_tracker.py | 2 ++ src/python/pants/task/task.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/python/pants/goal/run_tracker.py b/src/python/pants/goal/run_tracker.py index 61fb3e03fdb..d9c652392e2 100644 --- a/src/python/pants/goal/run_tracker.py +++ b/src/python/pants/goal/run_tracker.py @@ -163,6 +163,8 @@ def initialize(self): raise AssertionError('RunTracker.initialize must not be called multiple times.') # Initialize the run. + + # Select a globally unique ID for the run, that sorts by time. millis = int((self._run_timestamp * 1000) % 1000) run_id = 'pants_run_{}_{}_{}'.format( time.strftime('%Y_%m_%d_%H_%M_%S', time.localtime(self._run_timestamp)), diff --git a/src/python/pants/task/task.py b/src/python/pants/task/task.py index b0749a54a12..67a30a6d94f 100644 --- a/src/python/pants/task/task.py +++ b/src/python/pants/task/task.py @@ -535,7 +535,7 @@ def check_artifact_cache(self, vts): def do_check_artifact_cache(self, vts, post_process_cached_vts=None): """Checks the artifact cache for the specified list of VersionedTargetSets. - Returns a pair (cached, uncached) of VersionedTargets that were + Returns a tuple (cached, uncached, uncached_causes) of VersionedTargets that were satisfied/unsatisfied from the cache. """ if not vts: