diff --git a/src/python/pants/backend/experimental/go/register.py b/src/python/pants/backend/experimental/go/register.py index 371bbccfb0f0..fb06b84288b8 100644 --- a/src/python/pants/backend/experimental/go/register.py +++ b/src/python/pants/backend/experimental/go/register.py @@ -17,7 +17,7 @@ assembly, build_go_pkg, compile, - external_module, + external_pkg, go_mod, go_pkg, import_analysis, @@ -36,7 +36,7 @@ def rules(): *assembly.rules(), *build_go_pkg.rules(), *compile.rules(), - *external_module.rules(), + *external_pkg.rules(), *golang.rules(), *import_analysis.rules(), *go_mod.rules(), diff --git a/src/python/pants/backend/go/goals/package_binary_integration_test.py b/src/python/pants/backend/go/goals/package_binary_integration_test.py index 9ed5cf871796..6a081255da75 100644 --- a/src/python/pants/backend/go/goals/package_binary_integration_test.py +++ b/src/python/pants/backend/go/goals/package_binary_integration_test.py @@ -17,7 +17,7 @@ assembly, build_go_pkg, compile, - external_module, + external_pkg, go_mod, go_pkg, import_analysis, @@ -46,7 +46,7 @@ def rule_runner() -> RuleRunner: *go_mod.rules(), *link.rules(), *target_type_rules.rules(), - *external_module.rules(), + *external_pkg.rules(), *sdk.rules(), QueryRule(BuiltPackage, (GoBinaryFieldSet,)), ], diff --git a/src/python/pants/backend/go/goals/tailor_test.py b/src/python/pants/backend/go/goals/tailor_test.py index 0b8cc7c917ab..07c717eb8db7 100644 --- a/src/python/pants/backend/go/goals/tailor_test.py +++ b/src/python/pants/backend/go/goals/tailor_test.py @@ -10,7 +10,7 @@ ) from pants.backend.go.goals.tailor import rules as go_tailor_rules from pants.backend.go.target_types import GoModTarget, GoPackage -from pants.backend.go.util_rules import external_module, go_mod, sdk +from pants.backend.go.util_rules import external_pkg, go_mod, sdk from pants.core.goals.tailor import ( AllOwnedSources, PutativeTarget, @@ -31,7 +31,7 @@ def rule_runner() -> RuleRunner: *go_tailor_rules(), *external_tool.rules(), *source_files.rules(), - *external_module.rules(), + *external_pkg.rules(), *go_mod.rules(), *sdk.rules(), *target_type_rules.rules(), diff --git a/src/python/pants/backend/go/target_type_rules.py b/src/python/pants/backend/go/target_type_rules.py index 04ff50d34625..a91dca470253 100644 --- a/src/python/pants/backend/go/target_type_rules.py +++ b/src/python/pants/backend/go/target_type_rules.py @@ -23,15 +23,15 @@ GoPackageSources, ) from pants.backend.go.util_rules import go_pkg, import_analysis -from pants.backend.go.util_rules.external_module import ( - ExternalModulePkgImportPaths, - ExternalModulePkgImportPathsRequest, - ResolveExternalGoPackageRequest, +from pants.backend.go.util_rules.external_pkg import ( + ExternalModuleInfo, + ExternalModuleInfoRequest, + ExternalPkgInfo, + ExternalPkgInfoRequest, ) from pants.backend.go.util_rules.go_mod import ( GoModInfo, GoModInfoRequest, - ModuleDescriptor, OwningGoMod, OwningGoModRequest, ) @@ -189,30 +189,30 @@ class InjectGoExternalPackageDependenciesRequest(InjectDependenciesRequest): inject_for = GoExternalPackageDependencies -# TODO(#12761): This duplicates first-party dependency inference but that other rule cannot operate -# on _go_external_package targets since there is no sources field in a _go_external_package. -# Consider how to merge the inference/injection rules into one. Maybe use a private Sources field? @rule async def inject_go_external_package_dependencies( request: InjectGoExternalPackageDependenciesRequest, std_lib_imports: GoStdLibImports, package_mapping: GoImportPathToPackageMapping, ) -> InjectedDependencies: - wrapped_target = await Get(WrappedTarget, Address, request.dependencies_field.address) + addr = request.dependencies_field.address + wrapped_target = await Get(WrappedTarget, Address, addr) tgt = wrapped_target.target - assert isinstance(tgt, GoExternalPackageTarget) - owning_go_mod = await Get(OwningGoMod, OwningGoModRequest(tgt.address)) + owning_go_mod = await Get(OwningGoMod, OwningGoModRequest(addr)) go_mod_info = await Get(GoModInfo, GoModInfoRequest(owning_go_mod.address)) - - this_go_package = await Get( - ResolvedGoPackage, ResolveExternalGoPackageRequest(tgt, go_mod_info.stripped_digest) + pkg_info = await Get( + ExternalPkgInfo, + ExternalPkgInfoRequest( + module_path=tgt[GoExternalModulePathField].value, + version=tgt[GoExternalModuleVersionField].value, + import_path=tgt[GoExternalPackageImportPathField].value, + go_mod_stripped_digest=go_mod_info.stripped_digest, + ), ) - # Loop through all of the imports of this package and add dependencies on other packages and - # external modules. inferred_dependencies = [] - for import_path in this_go_package.imports + this_go_package.test_imports: + for import_path in pkg_info.imports: if import_path in std_lib_imports: continue @@ -229,7 +229,7 @@ async def inject_go_external_package_dependencies( else: logger.debug( f"Unable to infer dependency for import path '{import_path}' " - f"in go_external_package at address '{this_go_package.address}'." + f"in go_external_package at address '{addr}'." ) return InjectedDependencies(inferred_dependencies) @@ -250,10 +250,10 @@ async def generate_go_external_package_targets( ) -> GeneratedTargets: generator_addr = request.generator.address go_mod_info = await Get(GoModInfo, GoModInfoRequest(generator_addr)) - all_pkg_import_paths = await MultiGet( + all_module_info = await MultiGet( Get( - ExternalModulePkgImportPaths, - ExternalModulePkgImportPathsRequest( + ExternalModuleInfo, + ExternalModuleInfoRequest( module_path=module_descriptor.path, version=module_descriptor.version, go_mod_stripped_digest=go_mod_info.stripped_digest, @@ -262,31 +262,23 @@ async def generate_go_external_package_targets( for module_descriptor in go_mod_info.modules ) - def create_tgt( - module_descriptor: ModuleDescriptor, pkg_import_path: str - ) -> GoExternalPackageTarget: + def create_tgt(pkg_info: ExternalPkgInfo) -> GoExternalPackageTarget: return GoExternalPackageTarget( { - GoExternalModulePathField.alias: module_descriptor.path, - GoExternalModuleVersionField.alias: module_descriptor.version, - GoExternalPackageImportPathField.alias: pkg_import_path, + GoExternalModulePathField.alias: pkg_info.module_path, + GoExternalModuleVersionField.alias: pkg_info.version, + GoExternalPackageImportPathField.alias: pkg_info.import_path, }, # E.g. `src/go:mod#github.com/google/uuid`. - Address( - generator_addr.spec_path, - target_name=generator_addr.target_name, - generated_name=pkg_import_path, - ), + generator_addr.create_generated(pkg_info.import_path), ) return GeneratedTargets( request.generator, ( - create_tgt(module_descriptor, pkg_import_path) - for module_descriptor, pkg_import_paths in zip( - go_mod_info.modules, all_pkg_import_paths - ) - for pkg_import_path in pkg_import_paths + create_tgt(pkg_info) + for module_info in all_module_info + for pkg_info in module_info.values() ), ) diff --git a/src/python/pants/backend/go/target_type_rules_test.py b/src/python/pants/backend/go/target_type_rules_test.py index a268ebe5dd0f..a70ae5822306 100644 --- a/src/python/pants/backend/go/target_type_rules_test.py +++ b/src/python/pants/backend/go/target_type_rules_test.py @@ -27,7 +27,7 @@ GoPackage, GoPackageSources, ) -from pants.backend.go.util_rules import external_module, go_mod, go_pkg, sdk +from pants.backend.go.util_rules import external_pkg, go_mod, go_pkg, sdk from pants.base.exceptions import ResolveError from pants.build_graph.address import Address from pants.core.target_types import GenericTarget @@ -53,7 +53,7 @@ def rule_runner() -> RuleRunner: rules=[ *go_mod.rules(), *go_pkg.rules(), - *external_module.rules(), + *external_pkg.rules(), *sdk.rules(), *target_type_rules.rules(), QueryRule(Addresses, [DependenciesRequest]), diff --git a/src/python/pants/backend/go/util_rules/assembly_integration_test.py b/src/python/pants/backend/go/util_rules/assembly_integration_test.py index d26ca8634eee..29d1d9d3eda8 100644 --- a/src/python/pants/backend/go/util_rules/assembly_integration_test.py +++ b/src/python/pants/backend/go/util_rules/assembly_integration_test.py @@ -17,7 +17,7 @@ assembly, build_go_pkg, compile, - external_module, + external_pkg, go_mod, go_pkg, import_analysis, @@ -46,7 +46,7 @@ def rule_runner() -> RuleRunner: *go_mod.rules(), *link.rules(), *target_type_rules.rules(), - *external_module.rules(), + *external_pkg.rules(), *sdk.rules(), QueryRule(BuiltPackage, (GoBinaryFieldSet,)), ], diff --git a/src/python/pants/backend/go/util_rules/build_go_pkg.py b/src/python/pants/backend/go/util_rules/build_go_pkg.py index 6dbc59e084e9..7eeebc617143 100644 --- a/src/python/pants/backend/go/util_rules/build_go_pkg.py +++ b/src/python/pants/backend/go/util_rules/build_go_pkg.py @@ -9,7 +9,7 @@ from pants.backend.go.target_types import ( GoExternalModulePathField, GoExternalModuleVersionField, - GoExternalPackageTarget, + GoExternalPackageImportPathField, GoPackageSources, ) from pants.backend.go.util_rules.assembly import ( @@ -19,11 +19,7 @@ AssemblyPreCompilationRequest, ) from pants.backend.go.util_rules.compile import CompiledGoSources, CompileGoSourcesRequest -from pants.backend.go.util_rules.external_module import ( - DownloadedModule, - DownloadedModuleRequest, - ResolveExternalGoPackageRequest, -) +from pants.backend.go.util_rules.external_pkg import ExternalPkgInfo, ExternalPkgInfoRequest from pants.backend.go.util_rules.go_mod import ( GoModInfo, GoModInfoRequest, @@ -75,40 +71,45 @@ async def build_go_package(request: BuildGoPackageRequest) -> BuiltGoPackage: target = wrapped_target.target if is_first_party_package_target(target): - source_files, resolved_package = await MultiGet( + _source_files, _resolved_package = await MultiGet( Get( SourceFiles, SourceFilesRequest((target[GoPackageSources],)), ), Get(ResolvedGoPackage, ResolveGoPackageRequest(address=target.address)), ) - source_files_digest = source_files.snapshot.digest + source_files_digest = _source_files.snapshot.digest source_files_subpath = target.address.spec_path + + original_import_path = _resolved_package.import_path + go_files = _resolved_package.go_files + s_files = _resolved_package.s_files + elif is_third_party_package_target(target): - assert isinstance(target, GoExternalPackageTarget) + original_import_path = target[GoExternalPackageImportPathField].value + _module_path = target[GoExternalModulePathField].value + source_files_subpath = original_import_path[len(_module_path) :] + _owning_go_mod = await Get(OwningGoMod, OwningGoModRequest(target.address)) _go_mod_info = await Get(GoModInfo, GoModInfoRequest(_owning_go_mod.address)) - module_path = target[GoExternalModulePathField].value - _downloaded_module, resolved_package = await MultiGet( - Get( - DownloadedModule, - DownloadedModuleRequest( - module_path, - target[GoExternalModuleVersionField].value, - _go_mod_info.stripped_digest, - ), - ), - Get( - ResolvedGoPackage, - ResolveExternalGoPackageRequest(target, _go_mod_info.stripped_digest), + _pkg_info = await Get( + ExternalPkgInfo, + ExternalPkgInfoRequest( + import_path=original_import_path, + module_path=_module_path, + version=target[GoExternalModuleVersionField].value, + go_mod_stripped_digest=_go_mod_info.stripped_digest, ), ) - source_files_digest = _downloaded_module.digest - source_files_subpath = resolved_package.import_path[len(module_path) :] + + source_files_digest = _pkg_info.digest + go_files = _pkg_info.go_files + s_files = _pkg_info.s_files + else: raise AssertionError(f"Unknown how to build target at address {request.address} with Go.") - import_path = "main" if request.is_main else resolved_package.import_path + import_path = "main" if request.is_main else original_import_path # TODO: If you use `Targets` here, then we replace the direct dep on the `go_mod` with all # of its generated targets...Figure this out. @@ -138,12 +139,10 @@ async def build_go_package(request: BuildGoPackageRequest) -> BuiltGoPackage: assembly_digests = None symabis_path = None - if resolved_package.s_files: + if s_files: assembly_setup = await Get( AssemblyPreCompilation, - AssemblyPreCompilationRequest( - input_digest, resolved_package.s_files, source_files_subpath - ), + AssemblyPreCompilationRequest(input_digest, s_files, source_files_subpath), ) input_digest = assembly_setup.merged_compilation_input_digest assembly_digests = assembly_setup.assembly_digests @@ -153,7 +152,7 @@ async def build_go_package(request: BuildGoPackageRequest) -> BuiltGoPackage: CompiledGoSources, CompileGoSourcesRequest( digest=input_digest, - sources=tuple(f"./{source_files_subpath}/{name}" for name in resolved_package.go_files), + sources=tuple(f"./{source_files_subpath}/{name}" for name in go_files), import_path=import_path, description=f"Compile Go package: {import_path}", import_config_path=import_config.CONFIG_PATH, @@ -167,7 +166,7 @@ async def build_go_package(request: BuildGoPackageRequest) -> BuiltGoPackage: AssemblyPostCompilationRequest( compilation_digest, assembly_digests, - resolved_package.s_files, + s_files, source_files_subpath, ), ) diff --git a/src/python/pants/backend/go/util_rules/build_go_pkg_test.py b/src/python/pants/backend/go/util_rules/build_go_pkg_test.py index c8a2d733b71e..8e8ec5391ad4 100644 --- a/src/python/pants/backend/go/util_rules/build_go_pkg_test.py +++ b/src/python/pants/backend/go/util_rules/build_go_pkg_test.py @@ -14,7 +14,7 @@ assembly, build_go_pkg, compile, - external_module, + external_pkg, go_mod, go_pkg, import_analysis, @@ -41,7 +41,7 @@ def rule_runner() -> RuleRunner: *import_analysis.rules(), *go_mod.rules(), *go_pkg.rules(), - *external_module.rules(), + *external_pkg.rules(), *target_type_rules.rules(), QueryRule(BuiltGoPackage, [BuildGoPackageRequest]), ], diff --git a/src/python/pants/backend/go/util_rules/external_module_test.py b/src/python/pants/backend/go/util_rules/external_module_test.py deleted file mode 100644 index 217d4ee07b03..000000000000 --- a/src/python/pants/backend/go/util_rules/external_module_test.py +++ /dev/null @@ -1,312 +0,0 @@ -# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). -# Licensed under the Apache License, Version 2.0 (see LICENSE). - -from __future__ import annotations - -from textwrap import dedent - -import pytest - -from pants.backend.go import target_type_rules -from pants.backend.go.target_types import GoExternalPackageTarget, GoModTarget -from pants.backend.go.util_rules import external_module, go_mod, go_pkg, sdk -from pants.backend.go.util_rules.external_module import ( - AllDownloadedModules, - AllDownloadedModulesRequest, - DownloadedModule, - DownloadedModuleRequest, - ExternalModulePkgImportPaths, - ExternalModulePkgImportPathsRequest, - ResolveExternalGoPackageRequest, -) -from pants.backend.go.util_rules.go_pkg import ResolvedGoPackage -from pants.engine.addresses import Address -from pants.engine.fs import Digest, PathGlobs, Snapshot -from pants.engine.process import ProcessExecutionFailure -from pants.engine.rules import QueryRule -from pants.testutil.rule_runner import RuleRunner, engine_error - - -@pytest.fixture -def rule_runner() -> RuleRunner: - rule_runner = RuleRunner( - rules=[ - *sdk.rules(), - *go_mod.rules(), - *go_pkg.rules(), - *external_module.rules(), - *target_type_rules.rules(), - QueryRule(AllDownloadedModules, [AllDownloadedModulesRequest]), - QueryRule(DownloadedModule, [DownloadedModuleRequest]), - QueryRule(ExternalModulePkgImportPaths, [ExternalModulePkgImportPathsRequest]), - QueryRule(ResolvedGoPackage, [ResolveExternalGoPackageRequest]), - ], - target_types=[GoModTarget], - ) - rule_runner.set_options([], env_inherit={"PATH"}) - return rule_runner - - -GO_MOD = dedent( - """\ - module example.com/external-module - go 1.17 - - // No dependencies, already has `go.mod`. - require github.com/google/uuid v1.3.0 - - // No dependencies, but missing `go.mod`. - require cloud.google.com/go v0.26.0 - - // Has dependencies, already has a `go.sum`. - require ( - github.com/google/go-cmp v0.5.6 - golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 // indirect - ) - - // Has dependencies, missing `go.sum`. This causes `go list` to fail in that directory unless - // we add `go.sum`. - require ( - rsc.io/quote v1.5.2 - golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c // indirect - rsc.io/sampler v1.3.0 // indirect - ) - """ -) - -GO_SUM = dedent( - """\ - cloud.google.com/go v0.26.0 h1:e0WKqKTd5BnrG8aKH3J3h+QvEIQtSUcf2n5UZ5ZgLtQ= - cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= - github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= - github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= - github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= - github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= - golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c h1:qgOY6WgZOaTkIIMiVjBQcw93ERBE4m30iBm00nkL0i8= - golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= - golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= - golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= - rsc.io/quote v1.5.2 h1:w5fcysjrx7yqtD/aO+QwRjYZOKnaM9Uh2b40tElTs3Y= - rsc.io/quote v1.5.2/go.mod h1:LzX7hefJvL54yjefDEDHNONDjII0t9xZLPXsUe+TKr0= - rsc.io/sampler v1.3.0 h1:7uVkIFmeBqHfdjD+gZwtXXI+RODJ2Wc4O7MPEh/QiW4= - rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= - """ -) - - -def test_download_modules(rule_runner: RuleRunner) -> None: - input_digest = rule_runner.make_snapshot({"go.mod": GO_MOD, "go.sum": GO_SUM}).digest - downloaded_modules = rule_runner.request( - AllDownloadedModules, [AllDownloadedModulesRequest(input_digest)] - ) - assert len(downloaded_modules) == 7 - - def assert_module(module: str, version: str, sample_file: str) -> None: - assert (module, version) in downloaded_modules - digest = downloaded_modules[(module, version)] - snapshot = rule_runner.request(Snapshot, [digest]) - assert "go.mod" in snapshot.files - assert "go.sum" in snapshot.files - assert sample_file in snapshot.files - - extracted_module = rule_runner.request( - DownloadedModule, [DownloadedModuleRequest(module, version, input_digest)] - ) - extracted_snapshot = rule_runner.request(Snapshot, [extracted_module.digest]) - assert extracted_snapshot == snapshot - - assert_module("cloud.google.com/go", "v0.26.0", "bigtable/filter.go") - assert_module("github.com/google/uuid", "v1.3.0", "uuid.go") - assert_module("github.com/google/go-cmp", "v0.5.6", "cmp/cmpopts/errors_go113.go") - assert_module("golang.org/x/text", "v0.0.0-20170915032832-14c0d48ead0c", "width/transform.go") - assert_module("golang.org/x/xerrors", "v0.0.0-20191204190536-9bdfabe68543", "wrap.go") - assert_module("rsc.io/quote", "v1.5.2", "quote.go") - assert_module("rsc.io/sampler", "v1.3.0", "sampler.go") - - -def test_download_modules_missing_module(rule_runner: RuleRunner) -> None: - input_digest = rule_runner.make_snapshot({"go.mod": GO_MOD, "go.sum": GO_SUM}).digest - with engine_error( - AssertionError, contains="The module some_project.org/project@v1.1 was not downloaded" - ): - rule_runner.request( - DownloadedModule, - [DownloadedModuleRequest("some_project.org/project", "v1.1", input_digest)], - ) - - -def test_download_modules_invalid_go_sum(rule_runner: RuleRunner) -> None: - input_digest = rule_runner.make_snapshot( - { - "go.mod": dedent( - """\ - module example.com/external-module - go 1.17 - require github.com/google/uuid v1.3.0 - """ - ), - "go.sum": dedent( - """\ - github.com/google/uuid v1.3.0 h1:00000gmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= - github.com/google/uuid v1.3.0/go.mod h1:00000e4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= - """ - ), - } - ).digest - with engine_error(ProcessExecutionFailure, contains="SECURITY ERROR"): - rule_runner.request(AllDownloadedModules, [AllDownloadedModulesRequest(input_digest)]) - - -def test_download_modules_missing_go_sum(rule_runner: RuleRunner) -> None: - input_digest = rule_runner.make_snapshot( - { - "go.mod": dedent( - """\ - module example.com/external-module - go 1.17 - require github.com/google/uuid v1.3.0 - """ - ), - # `go.sum` is for a different module. - "go.sum": dedent( - """\ - cloud.google.com/go v0.26.0 h1:e0WKqKTd5BnrG8aKH3J3h+QvEIQtSUcf2n5UZ5ZgLtQ= - cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= - """ - ), - } - ).digest - with engine_error(contains="`go.mod` and/or `go.sum` changed!"): - rule_runner.request(AllDownloadedModules, [AllDownloadedModulesRequest(input_digest)]) - - -def test_determine_external_package_info(rule_runner: RuleRunner) -> None: - rule_runner.write_files({"go.mod": GO_MOD, "go.sum": GO_SUM, "BUILD": "go_mod(name='mod')"}) - input_digest = rule_runner.request(Digest, [PathGlobs(["go.mod", "go.sum"])]) - - def get_pkg_info(import_path: str) -> ResolvedGoPackage: - pkg_addr = Address("", target_name="mod", generated_name=import_path) - tgt = rule_runner.get_target(pkg_addr) - assert isinstance(tgt, GoExternalPackageTarget) - result = rule_runner.request( - ResolvedGoPackage, [ResolveExternalGoPackageRequest(tgt, input_digest)] - ) - assert result.address == pkg_addr - assert result.module_address is None - assert result.import_path == import_path - return result - - cmp_info = get_pkg_info("github.com/google/go-cmp/cmp/cmpopts") - assert cmp_info.module_path == "github.com/google/go-cmp" - assert cmp_info.module_version == "v0.5.6" - assert cmp_info.package_name == "cmpopts" - assert cmp_info.imports == ( - "errors", - "fmt", - "github.com/google/go-cmp/cmp", - "github.com/google/go-cmp/cmp/internal/function", - "math", - "reflect", - "sort", - "strings", - "time", - "unicode", - "unicode/utf8", - ) - assert cmp_info.test_imports == ( - "bytes", - "errors", - "fmt", - "github.com/google/go-cmp/cmp", - "golang.org/x/xerrors", - "io", - "math", - "reflect", - "strings", - "sync", - "testing", - "time", - ) - assert cmp_info.go_files == ( - "equate.go", - "errors_go113.go", - "ignore.go", - "sort.go", - "struct_filter.go", - "xform.go", - ) - assert cmp_info.test_go_files == ("util_test.go",) - assert cmp_info.xtest_go_files == ("example_test.go",) - assert not cmp_info.c_files - assert not cmp_info.cgo_files - assert not cmp_info.cxx_files - assert not cmp_info.m_files - assert not cmp_info.h_files - assert not cmp_info.s_files - assert not cmp_info.syso_files - - # Spot check that the other modules can be analyzed. - for pkg in ( - "cloud.google.com/go/bigquery", - "github.com/google/uuid", - "golang.org/x/text/collate", - "golang.org/x/xerrors", - "rsc.io/quote", - "rsc.io/sampler", - ): - get_pkg_info(pkg) - - -def test_determine_external_module_package_import_paths(rule_runner: RuleRunner) -> None: - input_digest = rule_runner.make_snapshot({"go.mod": GO_MOD, "go.sum": GO_SUM}).digest - - def assert_packages( - module_path: str, version: str, expected: list[str], *, check_subset: bool = False - ) -> None: - result = rule_runner.request( - ExternalModulePkgImportPaths, - [ExternalModulePkgImportPathsRequest(module_path, version, input_digest)], - ) - if check_subset: - assert set(expected).issubset(result) - else: - assert list(result) == expected - - assert_packages( - "cloud.google.com/go", - "v0.26.0", - ["cloud.google.com/go/bigquery", "cloud.google.com/go/firestore"], - check_subset=True, - ) - assert_packages("github.com/google/uuid", "v1.3.0", ["github.com/google/uuid"]) - - assert_packages( - "github.com/google/go-cmp", - "v0.5.6", - [ - "github.com/google/go-cmp/cmp", - "github.com/google/go-cmp/cmp/cmpopts", - "github.com/google/go-cmp/cmp/internal/diff", - "github.com/google/go-cmp/cmp/internal/flags", - "github.com/google/go-cmp/cmp/internal/function", - "github.com/google/go-cmp/cmp/internal/testprotos", - "github.com/google/go-cmp/cmp/internal/teststructs", - "github.com/google/go-cmp/cmp/internal/teststructs/foo1", - "github.com/google/go-cmp/cmp/internal/teststructs/foo2", - "github.com/google/go-cmp/cmp/internal/value", - ], - ) - assert_packages( - "golang.org/x/text", - "v0.0.0-20170915032832-14c0d48ead0c", - ["golang.org/x/text/cmd/gotext", "golang.org/x/text/collate"], - check_subset=True, - ) - assert_packages( - "golang.org/x/xerrors", - "v0.0.0-20191204190536-9bdfabe68543", - ["golang.org/x/xerrors", "golang.org/x/xerrors/internal"], - ) - - assert_packages("rsc.io/quote", "v1.5.2", ["rsc.io/quote", "rsc.io/quote/buggy"]) - assert_packages("rsc.io/sampler", "v1.3.0", ["rsc.io/sampler"]) diff --git a/src/python/pants/backend/go/util_rules/external_module.py b/src/python/pants/backend/go/util_rules/external_pkg.py similarity index 65% rename from src/python/pants/backend/go/util_rules/external_module.py rename to src/python/pants/backend/go/util_rules/external_pkg.py index 213088c87cc5..98d36b00177d 100644 --- a/src/python/pants/backend/go/util_rules/external_module.py +++ b/src/python/pants/backend/go/util_rules/external_pkg.py @@ -1,22 +1,13 @@ # Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). -import json import os from dataclasses import dataclass from typing import Tuple import ijson -from pants.backend.go.target_types import ( - GoExternalModulePathField, - GoExternalModuleVersionField, - GoExternalPackageImportPathField, - GoExternalPackageTarget, -) -from pants.backend.go.util_rules.go_pkg import ResolvedGoPackage from pants.backend.go.util_rules.sdk import GoSdkProcess -from pants.engine.collection import DeduplicatedCollection from pants.engine.engine_aware import EngineAwareParameter from pants.engine.fs import ( CreateDigest, @@ -36,8 +27,12 @@ from pants.util.frozendict import FrozenDict from pants.util.strutil import strip_v2_chroot_path +# ----------------------------------------------------------------------------------------------- +# Download modules +# ----------------------------------------------------------------------------------------------- + -class AllDownloadedModules(FrozenDict[Tuple[str, str], Digest]): +class _AllDownloadedModules(FrozenDict[Tuple[str, str], Digest]): """A mapping of each downloaded (module, version) to its digest. Each digest is stripped of the `gopath` prefix and also guaranteed to have a `go.mod` and @@ -47,7 +42,7 @@ class AllDownloadedModules(FrozenDict[Tuple[str, str], Digest]): @dataclass(frozen=True) -class AllDownloadedModulesRequest: +class _AllDownloadedModulesRequest: """Download all modules from the `go.mod`. The `go.mod` and `go.sum` must already be up-to-date. @@ -56,10 +51,32 @@ class AllDownloadedModulesRequest: go_mod_stripped_digest: Digest +@dataclass(frozen=True) +class _DownloadedModule: + """A downloaded module's directory. + + The digest is stripped of the `gopath` prefix and also guaranteed to have a `go.mod` and + `go.sum` for the particular module. This means that you can operate on the module (e.g. `go + list`) directly, without needing to set the working_dir etc. + """ + + digest: Digest + + +@dataclass(frozen=True) +class _DownloadedModuleRequest(EngineAwareParameter): + module_path: str + version: str + go_mod_stripped_digest: Digest + + def debug_hint(self) -> str: + return f"{self.module_path}@{self.version}" + + @rule async def download_external_modules( - request: AllDownloadedModulesRequest, -) -> AllDownloadedModules: + request: _AllDownloadedModulesRequest, +) -> _AllDownloadedModules: # TODO: Clean this up. input_digest_entries = await Get(DigestEntries, Digest, request.go_mod_stripped_digest) assert len(input_digest_entries) == 2 @@ -154,95 +171,75 @@ async def download_external_modules( module_paths_and_versions_to_dirs.keys(), stripped_subsets ) } - return AllDownloadedModules(module_paths_and_versions_to_digests) - - -@dataclass(frozen=True) -class DownloadedModule: - """A downloaded module's directory. - - The digest is stripped of the `gopath` prefix and also guaranteed to have a `go.mod` and - `go.sum` for the particular module. This means that you can operate on the module (e.g. `go - list`) directly, without needing to set the working_dir etc. - """ - - digest: Digest - - -@dataclass(frozen=True) -class DownloadedModuleRequest: - module_path: str - version: str - go_mod_stripped_digest: Digest + return _AllDownloadedModules(module_paths_and_versions_to_digests) @rule async def extract_module_from_downloaded_modules( - request: DownloadedModuleRequest, -) -> DownloadedModule: + request: _DownloadedModuleRequest, +) -> _DownloadedModule: all_modules = await Get( - AllDownloadedModules, AllDownloadedModulesRequest(request.go_mod_stripped_digest) + _AllDownloadedModules, _AllDownloadedModulesRequest(request.go_mod_stripped_digest) ) digest = all_modules.get((request.module_path, request.version)) if digest is None: raise AssertionError( f"The module {request.module_path}@{request.version} was not downloaded. Unless " - "you explicitly created an `_go_external_package`, this should not happen." + "you explicitly created an `_go_external_package`, this should not happen. " "Please open an issue at https://github.com/pantsbuild/pants/issues/new/choose with " "this error message." ) - return DownloadedModule(digest) + return _DownloadedModule(digest) + + +# ----------------------------------------------------------------------------------------------- +# Determine package info +# ----------------------------------------------------------------------------------------------- @dataclass(frozen=True) -class ResolveExternalGoPackageRequest(EngineAwareParameter): - tgt: GoExternalPackageTarget - go_mod_stripped_digest: Digest +class ExternalPkgInfo: + """All the info needed to build an external package. - def debug_hint(self) -> str: - return self.tgt[GoExternalPackageImportPathField].value + The digest is stripped of the `gopath` prefix. + """ + import_path: str + module_path: str + version: str -@rule -async def compute_external_go_package_info( - request: ResolveExternalGoPackageRequest, -) -> ResolvedGoPackage: - module_path = request.tgt[GoExternalModulePathField].value - module_version = request.tgt[GoExternalModuleVersionField].value + digest: Digest - downloaded_module = await Get( - DownloadedModule, - DownloadedModuleRequest(module_path, module_version, request.go_mod_stripped_digest), - ) + # Note that we don't care about test-related metadata like `TestImports`, as we'll never run + # tests directly on an external package. + imports: tuple[str, ...] + go_files: tuple[str, ...] + s_files: tuple[str, ...] - import_path = request.tgt[GoExternalPackageImportPathField].value - assert import_path.startswith(module_path) - subpath = import_path[len(module_path) :] - json_result = await Get( - ProcessResult, - GoSdkProcess( - command=("list", "-mod=readonly", "-json", f"./{subpath}"), - env={"GOPROXY": "off"}, - input_digest=downloaded_module.digest, - description=f"Determine metadata for Go external package {import_path}", - ), - ) +@dataclass(frozen=True) +class ExternalPkgInfoRequest(EngineAwareParameter): + """Request the info and digest needed to build an external package. - metadata = json.loads(json_result.stdout) - return ResolvedGoPackage.from_metadata( - metadata, - import_path=import_path, - address=request.tgt.address, - module_address=None, - module_path=module_path, - module_version=module_version, - ) + The package's module must be included in the input `go.mod`/`go.sum`. + """ + + import_path: str + module_path: str + version: str + go_mod_stripped_digest: Digest + + def debug_hint(self) -> str: + return self.import_path + + +class ExternalModuleInfo(FrozenDict[str, ExternalPkgInfo]): + """A mapping of the import path for each package in the module to its `ExternalPackageInfo`.""" @dataclass(frozen=True) -class ExternalModulePkgImportPathsRequest: - """Request the import paths for all packages belonging to an external Go module. +class ExternalModuleInfoRequest(EngineAwareParameter): + """Request info for every package contained in an external module. The module must be included in the input `go.mod`/`go.sum`. """ @@ -251,20 +248,17 @@ class ExternalModulePkgImportPathsRequest: version: str go_mod_stripped_digest: Digest - -class ExternalModulePkgImportPaths(DeduplicatedCollection[str]): - """The import paths for all packages belonging to an external Go module.""" - - sort_input = True + def debug_hint(self) -> str: + return f"{self.module_path}@{self.version}" @rule -async def compute_package_import_paths_from_external_module( - request: ExternalModulePkgImportPathsRequest, -) -> ExternalModulePkgImportPaths: +async def compute_external_module_metadata( + request: ExternalModuleInfoRequest, +) -> ExternalModuleInfo: downloaded_module = await Get( - DownloadedModule, - DownloadedModuleRequest( + _DownloadedModule, + _DownloadedModuleRequest( request.module_path, request.version, request.go_mod_stripped_digest ), ) @@ -272,19 +266,47 @@ async def compute_package_import_paths_from_external_module( ProcessResult, GoSdkProcess( input_digest=downloaded_module.digest, - # "-find" skips determining dependencies and imports for each package. - command=("list", "-find", "-mod=readonly", "-json", "./..."), + command=("list", "-mod=readonly", "-json", "./..."), env={"GOPROXY": "off"}, description=( - "Determine packages belonging to Go external module " - f"{request.module_path}@{request.version}" + f"Determine metadata for Go external module {request.module_path}@{request.version}" ), ), ) - return ExternalModulePkgImportPaths( - metadata["ImportPath"] - for metadata in ijson.items(json_result.stdout, "", multiple_values=True) + + import_path_to_info = {} + for metadata in ijson.items(json_result.stdout, "", multiple_values=True): + import_path = metadata["ImportPath"] + pkg_info = ExternalPkgInfo( + import_path=import_path, + module_path=request.module_path, + version=request.version, + digest=downloaded_module.digest, + imports=tuple(metadata.get("Imports", ())), + go_files=tuple(metadata.get("GoFiles", ())), + s_files=tuple(metadata.get("SFiles", ())), + ) + import_path_to_info[import_path] = pkg_info + return ExternalModuleInfo(import_path_to_info) + + +@rule +async def extract_package_info_from_module_info(request: ExternalPkgInfoRequest) -> ExternalPkgInfo: + module_info = await Get( + ExternalModuleInfo, + ExternalModuleInfoRequest( + request.module_path, request.version, request.go_mod_stripped_digest + ), ) + pkg_info = module_info.get(request.import_path) + if pkg_info is None: + raise AssertionError( + f"The package {request.import_path} does not belong to the module " + f"{request.module_path}@{request.version}. Unless you explicitly created an " + "`_go_external_package`, this should not happen. Please open an issue at " + "https://github.com/pantsbuild/pants/issues/new/choose with this error message." + ) + return pkg_info def rules(): diff --git a/src/python/pants/backend/go/util_rules/external_pkg_test.py b/src/python/pants/backend/go/util_rules/external_pkg_test.py new file mode 100644 index 000000000000..75624ec31895 --- /dev/null +++ b/src/python/pants/backend/go/util_rules/external_pkg_test.py @@ -0,0 +1,404 @@ +# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import annotations + +from textwrap import dedent + +import pytest + +from pants.backend.go.target_types import GoModTarget +from pants.backend.go.util_rules import external_pkg, sdk +from pants.backend.go.util_rules.external_pkg import ( + ExternalModuleInfo, + ExternalModuleInfoRequest, + ExternalPkgInfo, + ExternalPkgInfoRequest, + _AllDownloadedModules, + _AllDownloadedModulesRequest, + _DownloadedModule, + _DownloadedModuleRequest, +) +from pants.engine.fs import Digest, PathGlobs, Snapshot +from pants.engine.process import ProcessExecutionFailure +from pants.engine.rules import QueryRule +from pants.testutil.rule_runner import RuleRunner, engine_error + + +@pytest.fixture +def rule_runner() -> RuleRunner: + rule_runner = RuleRunner( + rules=[ + *sdk.rules(), + *external_pkg.rules(), + QueryRule(_AllDownloadedModules, [_AllDownloadedModulesRequest]), + QueryRule(_DownloadedModule, [_DownloadedModuleRequest]), + QueryRule(ExternalModuleInfo, [ExternalModuleInfoRequest]), + QueryRule(ExternalPkgInfo, [ExternalPkgInfoRequest]), + ], + target_types=[GoModTarget], + ) + rule_runner.set_options([], env_inherit={"PATH"}) + return rule_runner + + +GO_MOD = dedent( + """\ + module example.com/external-module + go 1.17 + + // No dependencies, already has `go.mod`. + require github.com/google/uuid v1.3.0 + + // No dependencies, but missing `go.mod`. + require cloud.google.com/go v0.26.0 + + // Has dependencies, already has a `go.sum`. + require ( + github.com/google/go-cmp v0.5.6 + golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 // indirect + ) + + // Has dependencies, missing `go.sum`. This causes `go list` to fail in that directory unless + // we add `go.sum`. + require ( + rsc.io/quote v1.5.2 + golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c // indirect + rsc.io/sampler v1.3.0 // indirect + ) + """ +) + +GO_SUM = dedent( + """\ + cloud.google.com/go v0.26.0 h1:e0WKqKTd5BnrG8aKH3J3h+QvEIQtSUcf2n5UZ5ZgLtQ= + cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= + github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= + github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= + github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= + github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= + golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c h1:qgOY6WgZOaTkIIMiVjBQcw93ERBE4m30iBm00nkL0i8= + golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= + golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= + golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= + rsc.io/quote v1.5.2 h1:w5fcysjrx7yqtD/aO+QwRjYZOKnaM9Uh2b40tElTs3Y= + rsc.io/quote v1.5.2/go.mod h1:LzX7hefJvL54yjefDEDHNONDjII0t9xZLPXsUe+TKr0= + rsc.io/sampler v1.3.0 h1:7uVkIFmeBqHfdjD+gZwtXXI+RODJ2Wc4O7MPEh/QiW4= + rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= + """ +) + + +# ----------------------------------------------------------------------------------------------- +# Download modules +# ----------------------------------------------------------------------------------------------- + + +def test_download_modules(rule_runner: RuleRunner) -> None: + input_digest = rule_runner.make_snapshot({"go.mod": GO_MOD, "go.sum": GO_SUM}).digest + downloaded_modules = rule_runner.request( + _AllDownloadedModules, [_AllDownloadedModulesRequest(input_digest)] + ) + assert len(downloaded_modules) == 7 + + def assert_module(module: str, version: str, sample_file: str) -> None: + assert (module, version) in downloaded_modules + digest = downloaded_modules[(module, version)] + snapshot = rule_runner.request(Snapshot, [digest]) + assert "go.mod" in snapshot.files + assert "go.sum" in snapshot.files + assert sample_file in snapshot.files + + extracted_module = rule_runner.request( + _DownloadedModule, [_DownloadedModuleRequest(module, version, input_digest)] + ) + extracted_snapshot = rule_runner.request(Snapshot, [extracted_module.digest]) + assert extracted_snapshot == snapshot + + assert_module("cloud.google.com/go", "v0.26.0", "bigtable/filter.go") + assert_module("github.com/google/uuid", "v1.3.0", "uuid.go") + assert_module("github.com/google/go-cmp", "v0.5.6", "cmp/cmpopts/errors_go113.go") + assert_module("golang.org/x/text", "v0.0.0-20170915032832-14c0d48ead0c", "width/transform.go") + assert_module("golang.org/x/xerrors", "v0.0.0-20191204190536-9bdfabe68543", "wrap.go") + assert_module("rsc.io/quote", "v1.5.2", "quote.go") + assert_module("rsc.io/sampler", "v1.3.0", "sampler.go") + + +def test_download_modules_missing_module(rule_runner: RuleRunner) -> None: + input_digest = rule_runner.make_snapshot({"go.mod": GO_MOD, "go.sum": GO_SUM}).digest + with engine_error( + AssertionError, contains="The module some_project.org/project@v1.1 was not downloaded" + ): + rule_runner.request( + _DownloadedModule, + [_DownloadedModuleRequest("some_project.org/project", "v1.1", input_digest)], + ) + + +def test_download_modules_invalid_go_sum(rule_runner: RuleRunner) -> None: + input_digest = rule_runner.make_snapshot( + { + "go.mod": dedent( + """\ + module example.com/external-module + go 1.17 + require github.com/google/uuid v1.3.0 + """ + ), + "go.sum": dedent( + """\ + github.com/google/uuid v1.3.0 h1:00000gmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= + github.com/google/uuid v1.3.0/go.mod h1:00000e4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= + """ + ), + } + ).digest + with engine_error(ProcessExecutionFailure, contains="SECURITY ERROR"): + rule_runner.request(_AllDownloadedModules, [_AllDownloadedModulesRequest(input_digest)]) + + +def test_download_modules_missing_go_sum(rule_runner: RuleRunner) -> None: + input_digest = rule_runner.make_snapshot( + { + "go.mod": dedent( + """\ + module example.com/external-module + go 1.17 + require github.com/google/uuid v1.3.0 + """ + ), + # `go.sum` is for a different module. + "go.sum": dedent( + """\ + cloud.google.com/go v0.26.0 h1:e0WKqKTd5BnrG8aKH3J3h+QvEIQtSUcf2n5UZ5ZgLtQ= + cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= + """ + ), + } + ).digest + with engine_error(contains="`go.mod` and/or `go.sum` changed!"): + rule_runner.request(_AllDownloadedModules, [_AllDownloadedModulesRequest(input_digest)]) + + +# ----------------------------------------------------------------------------------------------- +# Determine package info +# ----------------------------------------------------------------------------------------------- + + +def test_determine_pkg_info(rule_runner: RuleRunner) -> None: + rule_runner.write_files({"go.mod": GO_MOD, "go.sum": GO_SUM, "BUILD": "go_mod(name='mod')"}) + input_digest = rule_runner.request(Digest, [PathGlobs(["go.mod", "go.sum"])]) + + def assert_module( + module: str, + version: str, + expected: list[str] | dict[str, ExternalPkgInfo], + *, + check_subset: bool = False, + ) -> None: + module_info = rule_runner.request( + ExternalModuleInfo, [ExternalModuleInfoRequest(module, version, input_digest)] + ) + # If `check_subset`, check that the expected import_paths are included. + if check_subset: + assert isinstance(expected, list) + assert set(expected).issubset(module_info.keys()) + else: + # If expected is a dict, check that the ExternalPkgInfo is correct for each package. + if isinstance(expected, dict): + assert dict(module_info) == expected + # Else, only check that the import paths are present. + else: + assert list(module_info.keys()) == expected + + # Check our subsetting logic. + for pkg_info in module_info.values(): + extracted_pkg = rule_runner.request( + ExternalPkgInfo, + [ExternalPkgInfoRequest(pkg_info.import_path, module, version, input_digest)], + ) + assert extracted_pkg == pkg_info + + assert_module( + "cloud.google.com/go", + "v0.26.0", + ["cloud.google.com/go/bigquery", "cloud.google.com/go/firestore"], + check_subset=True, + ) + + uuid_mod = "github.com/google/uuid" + uuid_version = "v1.3.0" + uuid_digest = rule_runner.request( + _DownloadedModule, [_DownloadedModuleRequest(uuid_mod, uuid_version, input_digest)] + ).digest + assert_module( + uuid_mod, + uuid_version, + { + uuid_mod: ExternalPkgInfo( + import_path=uuid_mod, + module_path=uuid_mod, + version=uuid_version, + digest=uuid_digest, + imports=( + "bytes", + "crypto/md5", + "crypto/rand", + "crypto/sha1", + "database/sql/driver", + "encoding/binary", + "encoding/hex", + "encoding/json", + "errors", + "fmt", + "hash", + "io", + "net", + "os", + "strings", + "sync", + "time", + ), + go_files=( + "dce.go", + "doc.go", + "hash.go", + "marshal.go", + "node.go", + "node_net.go", + "null.go", + "sql.go", + "time.go", + "util.go", + "uuid.go", + "version1.go", + "version4.go", + ), + s_files=(), + ) + }, + ) + + assert_module( + "github.com/google/go-cmp", + "v0.5.6", + [ + "github.com/google/go-cmp/cmp", + "github.com/google/go-cmp/cmp/cmpopts", + "github.com/google/go-cmp/cmp/internal/diff", + "github.com/google/go-cmp/cmp/internal/flags", + "github.com/google/go-cmp/cmp/internal/function", + "github.com/google/go-cmp/cmp/internal/testprotos", + "github.com/google/go-cmp/cmp/internal/teststructs", + "github.com/google/go-cmp/cmp/internal/teststructs/foo1", + "github.com/google/go-cmp/cmp/internal/teststructs/foo2", + "github.com/google/go-cmp/cmp/internal/value", + ], + ) + assert_module( + "golang.org/x/text", + "v0.0.0-20170915032832-14c0d48ead0c", + ["golang.org/x/text/cmd/gotext", "golang.org/x/text/collate"], + check_subset=True, + ) + assert_module( + "golang.org/x/xerrors", + "v0.0.0-20191204190536-9bdfabe68543", + ["golang.org/x/xerrors", "golang.org/x/xerrors/internal"], + ) + + assert_module("rsc.io/quote", "v1.5.2", ["rsc.io/quote", "rsc.io/quote/buggy"]) + assert_module("rsc.io/sampler", "v1.3.0", ["rsc.io/sampler"]) + + # + # def get_pkg_info(import_path: str) -> ResolvedGoPackage: + # pkg_addr = Address("", target_name="mod", generated_name=import_path) + # tgt = rule_runner.get_target(pkg_addr) + # assert isinstance(tgt, GoExternalPackageTarget) + # result = rule_runner.request( + # ResolvedGoPackage, [ResolveExternalGoPackageRequest(tgt, input_digest)] + # ) + # assert result.address == pkg_addr + # assert result.module_address is None + # assert result.import_path == import_path + # return result + # + # cmp_info = get_pkg_info("github.com/google/go-cmp/cmp/cmpopts") + # assert cmp_info.module_path == "github.com/google/go-cmp" + # assert cmp_info.module_version == "v0.5.6" + # assert cmp_info.package_name == "cmpopts" + # assert cmp_info.imports == ( + # "errors", + # "fmt", + # "github.com/google/go-cmp/cmp", + # "github.com/google/go-cmp/cmp/internal/function", + # "math", + # "reflect", + # "sort", + # "strings", + # "time", + # "unicode", + # "unicode/utf8", + # ) + # assert cmp_info.test_imports == ( + # "bytes", + # "errors", + # "fmt", + # "github.com/google/go-cmp/cmp", + # "golang.org/x/xerrors", + # "io", + # "math", + # "reflect", + # "strings", + # "sync", + # "testing", + # "time", + # ) + # assert cmp_info.go_files == ( + # "equate.go", + # "errors_go113.go", + # "ignore.go", + # "sort.go", + # "struct_filter.go", + # "xform.go", + # ) + # assert cmp_info.test_go_files == ("util_test.go",) + # assert cmp_info.xtest_go_files == ("example_test.go",) + # assert not cmp_info.c_files + # assert not cmp_info.cgo_files + # assert not cmp_info.cxx_files + # assert not cmp_info.m_files + # assert not cmp_info.h_files + # assert not cmp_info.s_files + # assert not cmp_info.syso_files + # + # # Spot check that the other modules can be analyzed. + # for pkg in ( + # "cloud.google.com/go/bigquery", + # "github.com/google/uuid", + # "golang.org/x/text/collate", + # "golang.org/x/xerrors", + # "rsc.io/quote", + # "rsc.io/sampler", + # ): + # get_pkg_info(pkg) + # + + +def test_determine_pkg_info_missing(rule_runner: RuleRunner) -> None: + input_digest = rule_runner.make_snapshot({"go.mod": GO_MOD, "go.sum": GO_SUM}).digest + with engine_error( + AssertionError, + contains=( + "The package another_project.org/foo does not belong to the module " + "github.com/google/uuid@v1.3.0" + ), + ): + rule_runner.request( + ExternalPkgInfo, + [ + ExternalPkgInfoRequest( + "another_project.org/foo", "github.com/google/uuid", "v1.3.0", input_digest + ) + ], + ) diff --git a/src/python/pants/backend/go/util_rules/go_pkg.py b/src/python/pants/backend/go/util_rules/go_pkg.py index 46d06a192d69..695291d556b5 100644 --- a/src/python/pants/backend/go/util_rules/go_pkg.py +++ b/src/python/pants/backend/go/util_rules/go_pkg.py @@ -42,10 +42,6 @@ class ResolvedGoPackage: # or higher level of the source tree. module_address: Address | None - # External module information - module_path: str | None - module_version: str | None - # Name of the package as given by `package` directives in the source files. Obtained from `Name` key in # package metadata. package_name: str @@ -103,8 +99,6 @@ def from_metadata( import_path: str | None = None, address: Address | None = None, module_address: Address | None = None, - module_path: str | None = None, - module_version: str | None = None, ) -> ResolvedGoPackage: # TODO: Raise an exception on errors. They are only emitted as warnings for now because the `go` tool is # flagging missing first-party code as a dependency error. But we want dependency inference and won't know @@ -127,23 +121,16 @@ def from_metadata( "SwigCXXFiles", ): files = metadata.get(key, []) - package_description = ( - f"go_package at address {address}" - if address - else f"external package at import path {import_path} in {module_path}@{module_version}" - ) if files: raise ValueError( - f"The {package_description} contains the following unsupported source files " - f"that were detected under the key '{key}': {', '.join(files)}." + f"The go_package at address {address} contains the following unsupported source " + f"files that were detected under the key '{key}': {', '.join(files)}." ) return cls( address=address, import_path=import_path if import_path is not None else metadata["ImportPath"], module_address=module_address, - module_path=module_path, - module_version=module_version, package_name=metadata["Name"], imports=tuple(metadata.get("Imports", [])), test_imports=tuple(metadata.get("TestImports", [])),