diff --git a/src/python/pants/backend/experimental/go/register.py b/src/python/pants/backend/experimental/go/register.py index 371bbccfb0f..fb06b84288b 100644 --- a/src/python/pants/backend/experimental/go/register.py +++ b/src/python/pants/backend/experimental/go/register.py @@ -17,7 +17,7 @@ assembly, build_go_pkg, compile, - external_module, + external_pkg, go_mod, go_pkg, import_analysis, @@ -36,7 +36,7 @@ def rules(): *assembly.rules(), *build_go_pkg.rules(), *compile.rules(), - *external_module.rules(), + *external_pkg.rules(), *golang.rules(), *import_analysis.rules(), *go_mod.rules(), diff --git a/src/python/pants/backend/go/goals/package_binary_integration_test.py b/src/python/pants/backend/go/goals/package_binary_integration_test.py index 9ed5cf87179..6a081255da7 100644 --- a/src/python/pants/backend/go/goals/package_binary_integration_test.py +++ b/src/python/pants/backend/go/goals/package_binary_integration_test.py @@ -17,7 +17,7 @@ assembly, build_go_pkg, compile, - external_module, + external_pkg, go_mod, go_pkg, import_analysis, @@ -46,7 +46,7 @@ def rule_runner() -> RuleRunner: *go_mod.rules(), *link.rules(), *target_type_rules.rules(), - *external_module.rules(), + *external_pkg.rules(), *sdk.rules(), QueryRule(BuiltPackage, (GoBinaryFieldSet,)), ], diff --git a/src/python/pants/backend/go/goals/tailor_test.py b/src/python/pants/backend/go/goals/tailor_test.py index 0b8cc7c917a..07c717eb8db 100644 --- a/src/python/pants/backend/go/goals/tailor_test.py +++ b/src/python/pants/backend/go/goals/tailor_test.py @@ -10,7 +10,7 @@ ) from pants.backend.go.goals.tailor import rules as go_tailor_rules from pants.backend.go.target_types import GoModTarget, GoPackage -from pants.backend.go.util_rules import external_module, go_mod, sdk +from pants.backend.go.util_rules import external_pkg, go_mod, sdk from pants.core.goals.tailor import ( AllOwnedSources, PutativeTarget, @@ -31,7 +31,7 @@ def rule_runner() -> RuleRunner: *go_tailor_rules(), *external_tool.rules(), *source_files.rules(), - *external_module.rules(), + *external_pkg.rules(), *go_mod.rules(), *sdk.rules(), *target_type_rules.rules(), diff --git a/src/python/pants/backend/go/target_type_rules.py b/src/python/pants/backend/go/target_type_rules.py index 04ff50d3462..a91dca47025 100644 --- a/src/python/pants/backend/go/target_type_rules.py +++ b/src/python/pants/backend/go/target_type_rules.py @@ -23,15 +23,15 @@ GoPackageSources, ) from pants.backend.go.util_rules import go_pkg, import_analysis -from pants.backend.go.util_rules.external_module import ( - ExternalModulePkgImportPaths, - ExternalModulePkgImportPathsRequest, - ResolveExternalGoPackageRequest, +from pants.backend.go.util_rules.external_pkg import ( + ExternalModuleInfo, + ExternalModuleInfoRequest, + ExternalPkgInfo, + ExternalPkgInfoRequest, ) from pants.backend.go.util_rules.go_mod import ( GoModInfo, GoModInfoRequest, - ModuleDescriptor, OwningGoMod, OwningGoModRequest, ) @@ -189,30 +189,30 @@ class InjectGoExternalPackageDependenciesRequest(InjectDependenciesRequest): inject_for = GoExternalPackageDependencies -# TODO(#12761): This duplicates first-party dependency inference but that other rule cannot operate -# on _go_external_package targets since there is no sources field in a _go_external_package. -# Consider how to merge the inference/injection rules into one. Maybe use a private Sources field? @rule async def inject_go_external_package_dependencies( request: InjectGoExternalPackageDependenciesRequest, std_lib_imports: GoStdLibImports, package_mapping: GoImportPathToPackageMapping, ) -> InjectedDependencies: - wrapped_target = await Get(WrappedTarget, Address, request.dependencies_field.address) + addr = request.dependencies_field.address + wrapped_target = await Get(WrappedTarget, Address, addr) tgt = wrapped_target.target - assert isinstance(tgt, GoExternalPackageTarget) - owning_go_mod = await Get(OwningGoMod, OwningGoModRequest(tgt.address)) + owning_go_mod = await Get(OwningGoMod, OwningGoModRequest(addr)) go_mod_info = await Get(GoModInfo, GoModInfoRequest(owning_go_mod.address)) - - this_go_package = await Get( - ResolvedGoPackage, ResolveExternalGoPackageRequest(tgt, go_mod_info.stripped_digest) + pkg_info = await Get( + ExternalPkgInfo, + ExternalPkgInfoRequest( + module_path=tgt[GoExternalModulePathField].value, + version=tgt[GoExternalModuleVersionField].value, + import_path=tgt[GoExternalPackageImportPathField].value, + go_mod_stripped_digest=go_mod_info.stripped_digest, + ), ) - # Loop through all of the imports of this package and add dependencies on other packages and - # external modules. inferred_dependencies = [] - for import_path in this_go_package.imports + this_go_package.test_imports: + for import_path in pkg_info.imports: if import_path in std_lib_imports: continue @@ -229,7 +229,7 @@ async def inject_go_external_package_dependencies( else: logger.debug( f"Unable to infer dependency for import path '{import_path}' " - f"in go_external_package at address '{this_go_package.address}'." + f"in go_external_package at address '{addr}'." ) return InjectedDependencies(inferred_dependencies) @@ -250,10 +250,10 @@ async def generate_go_external_package_targets( ) -> GeneratedTargets: generator_addr = request.generator.address go_mod_info = await Get(GoModInfo, GoModInfoRequest(generator_addr)) - all_pkg_import_paths = await MultiGet( + all_module_info = await MultiGet( Get( - ExternalModulePkgImportPaths, - ExternalModulePkgImportPathsRequest( + ExternalModuleInfo, + ExternalModuleInfoRequest( module_path=module_descriptor.path, version=module_descriptor.version, go_mod_stripped_digest=go_mod_info.stripped_digest, @@ -262,31 +262,23 @@ async def generate_go_external_package_targets( for module_descriptor in go_mod_info.modules ) - def create_tgt( - module_descriptor: ModuleDescriptor, pkg_import_path: str - ) -> GoExternalPackageTarget: + def create_tgt(pkg_info: ExternalPkgInfo) -> GoExternalPackageTarget: return GoExternalPackageTarget( { - GoExternalModulePathField.alias: module_descriptor.path, - GoExternalModuleVersionField.alias: module_descriptor.version, - GoExternalPackageImportPathField.alias: pkg_import_path, + GoExternalModulePathField.alias: pkg_info.module_path, + GoExternalModuleVersionField.alias: pkg_info.version, + GoExternalPackageImportPathField.alias: pkg_info.import_path, }, # E.g. `src/go:mod#github.com/google/uuid`. - Address( - generator_addr.spec_path, - target_name=generator_addr.target_name, - generated_name=pkg_import_path, - ), + generator_addr.create_generated(pkg_info.import_path), ) return GeneratedTargets( request.generator, ( - create_tgt(module_descriptor, pkg_import_path) - for module_descriptor, pkg_import_paths in zip( - go_mod_info.modules, all_pkg_import_paths - ) - for pkg_import_path in pkg_import_paths + create_tgt(pkg_info) + for module_info in all_module_info + for pkg_info in module_info.values() ), ) diff --git a/src/python/pants/backend/go/target_type_rules_test.py b/src/python/pants/backend/go/target_type_rules_test.py index a268ebe5dd0..a70ae582230 100644 --- a/src/python/pants/backend/go/target_type_rules_test.py +++ b/src/python/pants/backend/go/target_type_rules_test.py @@ -27,7 +27,7 @@ GoPackage, GoPackageSources, ) -from pants.backend.go.util_rules import external_module, go_mod, go_pkg, sdk +from pants.backend.go.util_rules import external_pkg, go_mod, go_pkg, sdk from pants.base.exceptions import ResolveError from pants.build_graph.address import Address from pants.core.target_types import GenericTarget @@ -53,7 +53,7 @@ def rule_runner() -> RuleRunner: rules=[ *go_mod.rules(), *go_pkg.rules(), - *external_module.rules(), + *external_pkg.rules(), *sdk.rules(), *target_type_rules.rules(), QueryRule(Addresses, [DependenciesRequest]), diff --git a/src/python/pants/backend/go/util_rules/assembly_integration_test.py b/src/python/pants/backend/go/util_rules/assembly_integration_test.py index d26ca8634ee..29d1d9d3eda 100644 --- a/src/python/pants/backend/go/util_rules/assembly_integration_test.py +++ b/src/python/pants/backend/go/util_rules/assembly_integration_test.py @@ -17,7 +17,7 @@ assembly, build_go_pkg, compile, - external_module, + external_pkg, go_mod, go_pkg, import_analysis, @@ -46,7 +46,7 @@ def rule_runner() -> RuleRunner: *go_mod.rules(), *link.rules(), *target_type_rules.rules(), - *external_module.rules(), + *external_pkg.rules(), *sdk.rules(), QueryRule(BuiltPackage, (GoBinaryFieldSet,)), ], diff --git a/src/python/pants/backend/go/util_rules/build_go_pkg.py b/src/python/pants/backend/go/util_rules/build_go_pkg.py index 6dbc59e084e..7eeebc61714 100644 --- a/src/python/pants/backend/go/util_rules/build_go_pkg.py +++ b/src/python/pants/backend/go/util_rules/build_go_pkg.py @@ -9,7 +9,7 @@ from pants.backend.go.target_types import ( GoExternalModulePathField, GoExternalModuleVersionField, - GoExternalPackageTarget, + GoExternalPackageImportPathField, GoPackageSources, ) from pants.backend.go.util_rules.assembly import ( @@ -19,11 +19,7 @@ AssemblyPreCompilationRequest, ) from pants.backend.go.util_rules.compile import CompiledGoSources, CompileGoSourcesRequest -from pants.backend.go.util_rules.external_module import ( - DownloadedModule, - DownloadedModuleRequest, - ResolveExternalGoPackageRequest, -) +from pants.backend.go.util_rules.external_pkg import ExternalPkgInfo, ExternalPkgInfoRequest from pants.backend.go.util_rules.go_mod import ( GoModInfo, GoModInfoRequest, @@ -75,40 +71,45 @@ async def build_go_package(request: BuildGoPackageRequest) -> BuiltGoPackage: target = wrapped_target.target if is_first_party_package_target(target): - source_files, resolved_package = await MultiGet( + _source_files, _resolved_package = await MultiGet( Get( SourceFiles, SourceFilesRequest((target[GoPackageSources],)), ), Get(ResolvedGoPackage, ResolveGoPackageRequest(address=target.address)), ) - source_files_digest = source_files.snapshot.digest + source_files_digest = _source_files.snapshot.digest source_files_subpath = target.address.spec_path + + original_import_path = _resolved_package.import_path + go_files = _resolved_package.go_files + s_files = _resolved_package.s_files + elif is_third_party_package_target(target): - assert isinstance(target, GoExternalPackageTarget) + original_import_path = target[GoExternalPackageImportPathField].value + _module_path = target[GoExternalModulePathField].value + source_files_subpath = original_import_path[len(_module_path) :] + _owning_go_mod = await Get(OwningGoMod, OwningGoModRequest(target.address)) _go_mod_info = await Get(GoModInfo, GoModInfoRequest(_owning_go_mod.address)) - module_path = target[GoExternalModulePathField].value - _downloaded_module, resolved_package = await MultiGet( - Get( - DownloadedModule, - DownloadedModuleRequest( - module_path, - target[GoExternalModuleVersionField].value, - _go_mod_info.stripped_digest, - ), - ), - Get( - ResolvedGoPackage, - ResolveExternalGoPackageRequest(target, _go_mod_info.stripped_digest), + _pkg_info = await Get( + ExternalPkgInfo, + ExternalPkgInfoRequest( + import_path=original_import_path, + module_path=_module_path, + version=target[GoExternalModuleVersionField].value, + go_mod_stripped_digest=_go_mod_info.stripped_digest, ), ) - source_files_digest = _downloaded_module.digest - source_files_subpath = resolved_package.import_path[len(module_path) :] + + source_files_digest = _pkg_info.digest + go_files = _pkg_info.go_files + s_files = _pkg_info.s_files + else: raise AssertionError(f"Unknown how to build target at address {request.address} with Go.") - import_path = "main" if request.is_main else resolved_package.import_path + import_path = "main" if request.is_main else original_import_path # TODO: If you use `Targets` here, then we replace the direct dep on the `go_mod` with all # of its generated targets...Figure this out. @@ -138,12 +139,10 @@ async def build_go_package(request: BuildGoPackageRequest) -> BuiltGoPackage: assembly_digests = None symabis_path = None - if resolved_package.s_files: + if s_files: assembly_setup = await Get( AssemblyPreCompilation, - AssemblyPreCompilationRequest( - input_digest, resolved_package.s_files, source_files_subpath - ), + AssemblyPreCompilationRequest(input_digest, s_files, source_files_subpath), ) input_digest = assembly_setup.merged_compilation_input_digest assembly_digests = assembly_setup.assembly_digests @@ -153,7 +152,7 @@ async def build_go_package(request: BuildGoPackageRequest) -> BuiltGoPackage: CompiledGoSources, CompileGoSourcesRequest( digest=input_digest, - sources=tuple(f"./{source_files_subpath}/{name}" for name in resolved_package.go_files), + sources=tuple(f"./{source_files_subpath}/{name}" for name in go_files), import_path=import_path, description=f"Compile Go package: {import_path}", import_config_path=import_config.CONFIG_PATH, @@ -167,7 +166,7 @@ async def build_go_package(request: BuildGoPackageRequest) -> BuiltGoPackage: AssemblyPostCompilationRequest( compilation_digest, assembly_digests, - resolved_package.s_files, + s_files, source_files_subpath, ), ) diff --git a/src/python/pants/backend/go/util_rules/build_go_pkg_test.py b/src/python/pants/backend/go/util_rules/build_go_pkg_test.py index c8a2d733b71..8e8ec5391ad 100644 --- a/src/python/pants/backend/go/util_rules/build_go_pkg_test.py +++ b/src/python/pants/backend/go/util_rules/build_go_pkg_test.py @@ -14,7 +14,7 @@ assembly, build_go_pkg, compile, - external_module, + external_pkg, go_mod, go_pkg, import_analysis, @@ -41,7 +41,7 @@ def rule_runner() -> RuleRunner: *import_analysis.rules(), *go_mod.rules(), *go_pkg.rules(), - *external_module.rules(), + *external_pkg.rules(), *target_type_rules.rules(), QueryRule(BuiltGoPackage, [BuildGoPackageRequest]), ], diff --git a/src/python/pants/backend/go/util_rules/external_module.py b/src/python/pants/backend/go/util_rules/external_pkg.py similarity index 64% rename from src/python/pants/backend/go/util_rules/external_module.py rename to src/python/pants/backend/go/util_rules/external_pkg.py index 213088c87cc..eedbca63b3d 100644 --- a/src/python/pants/backend/go/util_rules/external_module.py +++ b/src/python/pants/backend/go/util_rules/external_pkg.py @@ -1,22 +1,15 @@ # Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). -import json +from __future__ import annotations + import os from dataclasses import dataclass from typing import Tuple import ijson -from pants.backend.go.target_types import ( - GoExternalModulePathField, - GoExternalModuleVersionField, - GoExternalPackageImportPathField, - GoExternalPackageTarget, -) -from pants.backend.go.util_rules.go_pkg import ResolvedGoPackage from pants.backend.go.util_rules.sdk import GoSdkProcess -from pants.engine.collection import DeduplicatedCollection from pants.engine.engine_aware import EngineAwareParameter from pants.engine.fs import ( CreateDigest, @@ -36,8 +29,12 @@ from pants.util.frozendict import FrozenDict from pants.util.strutil import strip_v2_chroot_path +# ----------------------------------------------------------------------------------------------- +# Download modules +# ----------------------------------------------------------------------------------------------- + -class AllDownloadedModules(FrozenDict[Tuple[str, str], Digest]): +class _AllDownloadedModules(FrozenDict[Tuple[str, str], Digest]): """A mapping of each downloaded (module, version) to its digest. Each digest is stripped of the `gopath` prefix and also guaranteed to have a `go.mod` and @@ -47,7 +44,7 @@ class AllDownloadedModules(FrozenDict[Tuple[str, str], Digest]): @dataclass(frozen=True) -class AllDownloadedModulesRequest: +class _AllDownloadedModulesRequest: """Download all modules from the `go.mod`. The `go.mod` and `go.sum` must already be up-to-date. @@ -56,10 +53,32 @@ class AllDownloadedModulesRequest: go_mod_stripped_digest: Digest +@dataclass(frozen=True) +class _DownloadedModule: + """A downloaded module's directory. + + The digest is stripped of the `gopath` prefix and also guaranteed to have a `go.mod` and + `go.sum` for the particular module. This means that you can operate on the module (e.g. `go + list`) directly, without needing to set the working_dir etc. + """ + + digest: Digest + + +@dataclass(frozen=True) +class _DownloadedModuleRequest(EngineAwareParameter): + module_path: str + version: str + go_mod_stripped_digest: Digest + + def debug_hint(self) -> str: + return f"{self.module_path}@{self.version}" + + @rule async def download_external_modules( - request: AllDownloadedModulesRequest, -) -> AllDownloadedModules: + request: _AllDownloadedModulesRequest, +) -> _AllDownloadedModules: # TODO: Clean this up. input_digest_entries = await Get(DigestEntries, Digest, request.go_mod_stripped_digest) assert len(input_digest_entries) == 2 @@ -154,95 +173,75 @@ async def download_external_modules( module_paths_and_versions_to_dirs.keys(), stripped_subsets ) } - return AllDownloadedModules(module_paths_and_versions_to_digests) - - -@dataclass(frozen=True) -class DownloadedModule: - """A downloaded module's directory. - - The digest is stripped of the `gopath` prefix and also guaranteed to have a `go.mod` and - `go.sum` for the particular module. This means that you can operate on the module (e.g. `go - list`) directly, without needing to set the working_dir etc. - """ - - digest: Digest - - -@dataclass(frozen=True) -class DownloadedModuleRequest: - module_path: str - version: str - go_mod_stripped_digest: Digest + return _AllDownloadedModules(module_paths_and_versions_to_digests) @rule async def extract_module_from_downloaded_modules( - request: DownloadedModuleRequest, -) -> DownloadedModule: + request: _DownloadedModuleRequest, +) -> _DownloadedModule: all_modules = await Get( - AllDownloadedModules, AllDownloadedModulesRequest(request.go_mod_stripped_digest) + _AllDownloadedModules, _AllDownloadedModulesRequest(request.go_mod_stripped_digest) ) digest = all_modules.get((request.module_path, request.version)) if digest is None: raise AssertionError( f"The module {request.module_path}@{request.version} was not downloaded. Unless " - "you explicitly created an `_go_external_package`, this should not happen." + "you explicitly created an `_go_external_package`, this should not happen. " "Please open an issue at https://github.com/pantsbuild/pants/issues/new/choose with " "this error message." ) - return DownloadedModule(digest) + return _DownloadedModule(digest) + + +# ----------------------------------------------------------------------------------------------- +# Determine package info +# ----------------------------------------------------------------------------------------------- @dataclass(frozen=True) -class ResolveExternalGoPackageRequest(EngineAwareParameter): - tgt: GoExternalPackageTarget - go_mod_stripped_digest: Digest +class ExternalPkgInfo: + """All the info needed to build an external package. - def debug_hint(self) -> str: - return self.tgt[GoExternalPackageImportPathField].value + The digest is stripped of the `gopath` prefix. + """ + import_path: str + module_path: str + version: str -@rule -async def compute_external_go_package_info( - request: ResolveExternalGoPackageRequest, -) -> ResolvedGoPackage: - module_path = request.tgt[GoExternalModulePathField].value - module_version = request.tgt[GoExternalModuleVersionField].value + digest: Digest - downloaded_module = await Get( - DownloadedModule, - DownloadedModuleRequest(module_path, module_version, request.go_mod_stripped_digest), - ) + # Note that we don't care about test-related metadata like `TestImports`, as we'll never run + # tests directly on an external package. + imports: tuple[str, ...] + go_files: tuple[str, ...] + s_files: tuple[str, ...] - import_path = request.tgt[GoExternalPackageImportPathField].value - assert import_path.startswith(module_path) - subpath = import_path[len(module_path) :] - json_result = await Get( - ProcessResult, - GoSdkProcess( - command=("list", "-mod=readonly", "-json", f"./{subpath}"), - env={"GOPROXY": "off"}, - input_digest=downloaded_module.digest, - description=f"Determine metadata for Go external package {import_path}", - ), - ) +@dataclass(frozen=True) +class ExternalPkgInfoRequest(EngineAwareParameter): + """Request the info and digest needed to build an external package. - metadata = json.loads(json_result.stdout) - return ResolvedGoPackage.from_metadata( - metadata, - import_path=import_path, - address=request.tgt.address, - module_address=None, - module_path=module_path, - module_version=module_version, - ) + The package's module must be included in the input `go.mod`/`go.sum`. + """ + + import_path: str + module_path: str + version: str + go_mod_stripped_digest: Digest + + def debug_hint(self) -> str: + return self.import_path + + +class ExternalModuleInfo(FrozenDict[str, ExternalPkgInfo]): + """A mapping of the import path for each package in the module to its `ExternalPackageInfo`.""" @dataclass(frozen=True) -class ExternalModulePkgImportPathsRequest: - """Request the import paths for all packages belonging to an external Go module. +class ExternalModuleInfoRequest(EngineAwareParameter): + """Request info for every package contained in an external module. The module must be included in the input `go.mod`/`go.sum`. """ @@ -251,20 +250,17 @@ class ExternalModulePkgImportPathsRequest: version: str go_mod_stripped_digest: Digest - -class ExternalModulePkgImportPaths(DeduplicatedCollection[str]): - """The import paths for all packages belonging to an external Go module.""" - - sort_input = True + def debug_hint(self) -> str: + return f"{self.module_path}@{self.version}" @rule -async def compute_package_import_paths_from_external_module( - request: ExternalModulePkgImportPathsRequest, -) -> ExternalModulePkgImportPaths: +async def compute_external_module_metadata( + request: ExternalModuleInfoRequest, +) -> ExternalModuleInfo: downloaded_module = await Get( - DownloadedModule, - DownloadedModuleRequest( + _DownloadedModule, + _DownloadedModuleRequest( request.module_path, request.version, request.go_mod_stripped_digest ), ) @@ -272,19 +268,47 @@ async def compute_package_import_paths_from_external_module( ProcessResult, GoSdkProcess( input_digest=downloaded_module.digest, - # "-find" skips determining dependencies and imports for each package. - command=("list", "-find", "-mod=readonly", "-json", "./..."), + command=("list", "-mod=readonly", "-json", "./..."), env={"GOPROXY": "off"}, description=( - "Determine packages belonging to Go external module " - f"{request.module_path}@{request.version}" + f"Determine metadata for Go external module {request.module_path}@{request.version}" ), ), ) - return ExternalModulePkgImportPaths( - metadata["ImportPath"] - for metadata in ijson.items(json_result.stdout, "", multiple_values=True) + + import_path_to_info = {} + for metadata in ijson.items(json_result.stdout, "", multiple_values=True): + import_path = metadata["ImportPath"] + pkg_info = ExternalPkgInfo( + import_path=import_path, + module_path=request.module_path, + version=request.version, + digest=downloaded_module.digest, + imports=tuple(metadata.get("Imports", ())), + go_files=tuple(metadata.get("GoFiles", ())), + s_files=tuple(metadata.get("SFiles", ())), + ) + import_path_to_info[import_path] = pkg_info + return ExternalModuleInfo(import_path_to_info) + + +@rule +async def extract_package_info_from_module_info(request: ExternalPkgInfoRequest) -> ExternalPkgInfo: + module_info = await Get( + ExternalModuleInfo, + ExternalModuleInfoRequest( + request.module_path, request.version, request.go_mod_stripped_digest + ), ) + pkg_info = module_info.get(request.import_path) + if pkg_info is None: + raise AssertionError( + f"The package {request.import_path} does not belong to the module " + f"{request.module_path}@{request.version}. Unless you explicitly created an " + "`_go_external_package`, this should not happen. Please open an issue at " + "https://github.com/pantsbuild/pants/issues/new/choose with this error message." + ) + return pkg_info def rules(): diff --git a/src/python/pants/backend/go/util_rules/external_module_test.py b/src/python/pants/backend/go/util_rules/external_pkg_test.py similarity index 59% rename from src/python/pants/backend/go/util_rules/external_module_test.py rename to src/python/pants/backend/go/util_rules/external_pkg_test.py index 217d4ee07b0..1a012aa394d 100644 --- a/src/python/pants/backend/go/util_rules/external_module_test.py +++ b/src/python/pants/backend/go/util_rules/external_pkg_test.py @@ -7,20 +7,18 @@ import pytest -from pants.backend.go import target_type_rules -from pants.backend.go.target_types import GoExternalPackageTarget, GoModTarget -from pants.backend.go.util_rules import external_module, go_mod, go_pkg, sdk -from pants.backend.go.util_rules.external_module import ( - AllDownloadedModules, - AllDownloadedModulesRequest, - DownloadedModule, - DownloadedModuleRequest, - ExternalModulePkgImportPaths, - ExternalModulePkgImportPathsRequest, - ResolveExternalGoPackageRequest, +from pants.backend.go.target_types import GoModTarget +from pants.backend.go.util_rules import external_pkg, sdk +from pants.backend.go.util_rules.external_pkg import ( + ExternalModuleInfo, + ExternalModuleInfoRequest, + ExternalPkgInfo, + ExternalPkgInfoRequest, + _AllDownloadedModules, + _AllDownloadedModulesRequest, + _DownloadedModule, + _DownloadedModuleRequest, ) -from pants.backend.go.util_rules.go_pkg import ResolvedGoPackage -from pants.engine.addresses import Address from pants.engine.fs import Digest, PathGlobs, Snapshot from pants.engine.process import ProcessExecutionFailure from pants.engine.rules import QueryRule @@ -32,14 +30,11 @@ def rule_runner() -> RuleRunner: rule_runner = RuleRunner( rules=[ *sdk.rules(), - *go_mod.rules(), - *go_pkg.rules(), - *external_module.rules(), - *target_type_rules.rules(), - QueryRule(AllDownloadedModules, [AllDownloadedModulesRequest]), - QueryRule(DownloadedModule, [DownloadedModuleRequest]), - QueryRule(ExternalModulePkgImportPaths, [ExternalModulePkgImportPathsRequest]), - QueryRule(ResolvedGoPackage, [ResolveExternalGoPackageRequest]), + *external_pkg.rules(), + QueryRule(_AllDownloadedModules, [_AllDownloadedModulesRequest]), + QueryRule(_DownloadedModule, [_DownloadedModuleRequest]), + QueryRule(ExternalModuleInfo, [ExternalModuleInfoRequest]), + QueryRule(ExternalPkgInfo, [ExternalPkgInfoRequest]), ], target_types=[GoModTarget], ) @@ -94,10 +89,15 @@ def rule_runner() -> RuleRunner: ) +# ----------------------------------------------------------------------------------------------- +# Download modules +# ----------------------------------------------------------------------------------------------- + + def test_download_modules(rule_runner: RuleRunner) -> None: input_digest = rule_runner.make_snapshot({"go.mod": GO_MOD, "go.sum": GO_SUM}).digest downloaded_modules = rule_runner.request( - AllDownloadedModules, [AllDownloadedModulesRequest(input_digest)] + _AllDownloadedModules, [_AllDownloadedModulesRequest(input_digest)] ) assert len(downloaded_modules) == 7 @@ -110,7 +110,7 @@ def assert_module(module: str, version: str, sample_file: str) -> None: assert sample_file in snapshot.files extracted_module = rule_runner.request( - DownloadedModule, [DownloadedModuleRequest(module, version, input_digest)] + _DownloadedModule, [_DownloadedModuleRequest(module, version, input_digest)] ) extracted_snapshot = rule_runner.request(Snapshot, [extracted_module.digest]) assert extracted_snapshot == snapshot @@ -130,8 +130,8 @@ def test_download_modules_missing_module(rule_runner: RuleRunner) -> None: AssertionError, contains="The module some_project.org/project@v1.1 was not downloaded" ): rule_runner.request( - DownloadedModule, - [DownloadedModuleRequest("some_project.org/project", "v1.1", input_digest)], + _DownloadedModule, + [_DownloadedModuleRequest("some_project.org/project", "v1.1", input_digest)], ) @@ -154,7 +154,7 @@ def test_download_modules_invalid_go_sum(rule_runner: RuleRunner) -> None: } ).digest with engine_error(ProcessExecutionFailure, contains="SECURITY ERROR"): - rule_runner.request(AllDownloadedModules, [AllDownloadedModulesRequest(input_digest)]) + rule_runner.request(_AllDownloadedModules, [_AllDownloadedModulesRequest(input_digest)]) def test_download_modules_missing_go_sum(rule_runner: RuleRunner) -> None: @@ -177,110 +177,109 @@ def test_download_modules_missing_go_sum(rule_runner: RuleRunner) -> None: } ).digest with engine_error(contains="`go.mod` and/or `go.sum` changed!"): - rule_runner.request(AllDownloadedModules, [AllDownloadedModulesRequest(input_digest)]) + rule_runner.request(_AllDownloadedModules, [_AllDownloadedModulesRequest(input_digest)]) -def test_determine_external_package_info(rule_runner: RuleRunner) -> None: - rule_runner.write_files({"go.mod": GO_MOD, "go.sum": GO_SUM, "BUILD": "go_mod(name='mod')"}) - input_digest = rule_runner.request(Digest, [PathGlobs(["go.mod", "go.sum"])]) - - def get_pkg_info(import_path: str) -> ResolvedGoPackage: - pkg_addr = Address("", target_name="mod", generated_name=import_path) - tgt = rule_runner.get_target(pkg_addr) - assert isinstance(tgt, GoExternalPackageTarget) - result = rule_runner.request( - ResolvedGoPackage, [ResolveExternalGoPackageRequest(tgt, input_digest)] - ) - assert result.address == pkg_addr - assert result.module_address is None - assert result.import_path == import_path - return result - - cmp_info = get_pkg_info("github.com/google/go-cmp/cmp/cmpopts") - assert cmp_info.module_path == "github.com/google/go-cmp" - assert cmp_info.module_version == "v0.5.6" - assert cmp_info.package_name == "cmpopts" - assert cmp_info.imports == ( - "errors", - "fmt", - "github.com/google/go-cmp/cmp", - "github.com/google/go-cmp/cmp/internal/function", - "math", - "reflect", - "sort", - "strings", - "time", - "unicode", - "unicode/utf8", - ) - assert cmp_info.test_imports == ( - "bytes", - "errors", - "fmt", - "github.com/google/go-cmp/cmp", - "golang.org/x/xerrors", - "io", - "math", - "reflect", - "strings", - "sync", - "testing", - "time", - ) - assert cmp_info.go_files == ( - "equate.go", - "errors_go113.go", - "ignore.go", - "sort.go", - "struct_filter.go", - "xform.go", - ) - assert cmp_info.test_go_files == ("util_test.go",) - assert cmp_info.xtest_go_files == ("example_test.go",) - assert not cmp_info.c_files - assert not cmp_info.cgo_files - assert not cmp_info.cxx_files - assert not cmp_info.m_files - assert not cmp_info.h_files - assert not cmp_info.s_files - assert not cmp_info.syso_files - - # Spot check that the other modules can be analyzed. - for pkg in ( - "cloud.google.com/go/bigquery", - "github.com/google/uuid", - "golang.org/x/text/collate", - "golang.org/x/xerrors", - "rsc.io/quote", - "rsc.io/sampler", - ): - get_pkg_info(pkg) +# ----------------------------------------------------------------------------------------------- +# Determine package info +# ----------------------------------------------------------------------------------------------- -def test_determine_external_module_package_import_paths(rule_runner: RuleRunner) -> None: - input_digest = rule_runner.make_snapshot({"go.mod": GO_MOD, "go.sum": GO_SUM}).digest +def test_determine_pkg_info(rule_runner: RuleRunner) -> None: + rule_runner.write_files({"go.mod": GO_MOD, "go.sum": GO_SUM, "BUILD": "go_mod(name='mod')"}) + input_digest = rule_runner.request(Digest, [PathGlobs(["go.mod", "go.sum"])]) - def assert_packages( - module_path: str, version: str, expected: list[str], *, check_subset: bool = False + def assert_module( + module: str, + version: str, + expected: list[str] | dict[str, ExternalPkgInfo], + *, + check_subset: bool = False, ) -> None: - result = rule_runner.request( - ExternalModulePkgImportPaths, - [ExternalModulePkgImportPathsRequest(module_path, version, input_digest)], + module_info = rule_runner.request( + ExternalModuleInfo, [ExternalModuleInfoRequest(module, version, input_digest)] ) + # If `check_subset`, check that the expected import_paths are included. if check_subset: - assert set(expected).issubset(result) + assert isinstance(expected, list) + assert set(expected).issubset(module_info.keys()) else: - assert list(result) == expected - - assert_packages( + # If expected is a dict, check that the ExternalPkgInfo is correct for each package. + if isinstance(expected, dict): + assert dict(module_info) == expected + # Else, only check that the import paths are present. + else: + assert list(module_info.keys()) == expected + + # Check our subsetting logic. + for pkg_info in module_info.values(): + extracted_pkg = rule_runner.request( + ExternalPkgInfo, + [ExternalPkgInfoRequest(pkg_info.import_path, module, version, input_digest)], + ) + assert extracted_pkg == pkg_info + + assert_module( "cloud.google.com/go", "v0.26.0", ["cloud.google.com/go/bigquery", "cloud.google.com/go/firestore"], check_subset=True, ) - assert_packages("github.com/google/uuid", "v1.3.0", ["github.com/google/uuid"]) - assert_packages( + uuid_mod = "github.com/google/uuid" + uuid_version = "v1.3.0" + uuid_digest = rule_runner.request( + _DownloadedModule, [_DownloadedModuleRequest(uuid_mod, uuid_version, input_digest)] + ).digest + assert_module( + uuid_mod, + uuid_version, + { + uuid_mod: ExternalPkgInfo( + import_path=uuid_mod, + module_path=uuid_mod, + version=uuid_version, + digest=uuid_digest, + imports=( + "bytes", + "crypto/md5", + "crypto/rand", + "crypto/sha1", + "database/sql/driver", + "encoding/binary", + "encoding/hex", + "encoding/json", + "errors", + "fmt", + "hash", + "io", + "net", + "os", + "strings", + "sync", + "time", + ), + go_files=( + "dce.go", + "doc.go", + "hash.go", + "marshal.go", + "node.go", + "node_net.go", + "null.go", + "sql.go", + "time.go", + "util.go", + "uuid.go", + "version1.go", + "version4.go", + ), + s_files=(), + ) + }, + ) + + assert_module( "github.com/google/go-cmp", "v0.5.6", [ @@ -296,17 +295,36 @@ def assert_packages( "github.com/google/go-cmp/cmp/internal/value", ], ) - assert_packages( + assert_module( "golang.org/x/text", "v0.0.0-20170915032832-14c0d48ead0c", ["golang.org/x/text/cmd/gotext", "golang.org/x/text/collate"], check_subset=True, ) - assert_packages( + assert_module( "golang.org/x/xerrors", "v0.0.0-20191204190536-9bdfabe68543", ["golang.org/x/xerrors", "golang.org/x/xerrors/internal"], ) - assert_packages("rsc.io/quote", "v1.5.2", ["rsc.io/quote", "rsc.io/quote/buggy"]) - assert_packages("rsc.io/sampler", "v1.3.0", ["rsc.io/sampler"]) + assert_module("rsc.io/quote", "v1.5.2", ["rsc.io/quote", "rsc.io/quote/buggy"]) + assert_module("rsc.io/sampler", "v1.3.0", ["rsc.io/sampler"]) + + +def test_determine_pkg_info_missing(rule_runner: RuleRunner) -> None: + input_digest = rule_runner.make_snapshot({"go.mod": GO_MOD, "go.sum": GO_SUM}).digest + with engine_error( + AssertionError, + contains=( + "The package another_project.org/foo does not belong to the module " + "github.com/google/uuid@v1.3.0" + ), + ): + rule_runner.request( + ExternalPkgInfo, + [ + ExternalPkgInfoRequest( + "another_project.org/foo", "github.com/google/uuid", "v1.3.0", input_digest + ) + ], + ) diff --git a/src/python/pants/backend/go/util_rules/go_pkg.py b/src/python/pants/backend/go/util_rules/go_pkg.py index 46d06a192d6..695291d556b 100644 --- a/src/python/pants/backend/go/util_rules/go_pkg.py +++ b/src/python/pants/backend/go/util_rules/go_pkg.py @@ -42,10 +42,6 @@ class ResolvedGoPackage: # or higher level of the source tree. module_address: Address | None - # External module information - module_path: str | None - module_version: str | None - # Name of the package as given by `package` directives in the source files. Obtained from `Name` key in # package metadata. package_name: str @@ -103,8 +99,6 @@ def from_metadata( import_path: str | None = None, address: Address | None = None, module_address: Address | None = None, - module_path: str | None = None, - module_version: str | None = None, ) -> ResolvedGoPackage: # TODO: Raise an exception on errors. They are only emitted as warnings for now because the `go` tool is # flagging missing first-party code as a dependency error. But we want dependency inference and won't know @@ -127,23 +121,16 @@ def from_metadata( "SwigCXXFiles", ): files = metadata.get(key, []) - package_description = ( - f"go_package at address {address}" - if address - else f"external package at import path {import_path} in {module_path}@{module_version}" - ) if files: raise ValueError( - f"The {package_description} contains the following unsupported source files " - f"that were detected under the key '{key}': {', '.join(files)}." + f"The go_package at address {address} contains the following unsupported source " + f"files that were detected under the key '{key}': {', '.join(files)}." ) return cls( address=address, import_path=import_path if import_path is not None else metadata["ImportPath"], module_address=module_address, - module_path=module_path, - module_version=module_version, package_name=metadata["Name"], imports=tuple(metadata.get("Imports", [])), test_imports=tuple(metadata.get("TestImports", [])),