From ae545fc6c67cdf6823f31ed0cf4ce4df52f4963e Mon Sep 17 00:00:00 2001 From: Andreas Stenius Date: Mon, 26 Jul 2021 17:29:13 +0200 Subject: [PATCH 1/8] Add hadolint to validate Dockerfiles. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Example lint results: $ ./pants lint docker:helloworld 18:54:08.20 [INFO] Completed: lint - hadolint succeeded. ✓ hadolint succeeded. $ ./pants lint docker:helloworld 18:48:00.57 [WARN] Completed: lint - hadolint failed (exit code 1). docker/Dockerfile:2 DL3008 warning: Pin versions in apt get install. Instead of `apt-get install ` use `apt-get install =` docker/Dockerfile:2 DL3009 info: Delete the apt-get lists after installing something docker/Dockerfile:2 DL3015 info: Avoid additional packages by specifying `--no-install-recommends` 𐄂 hadolint failed. [ci skip-rust] [ci skip-build-wheels] --- src/python/pants/backend/docker/lint.py | 100 ++++++++++++++++++ .../backend/experimental/docker/register.py | 6 +- 2 files changed, 105 insertions(+), 1 deletion(-) create mode 100644 src/python/pants/backend/docker/lint.py diff --git a/src/python/pants/backend/docker/lint.py b/src/python/pants/backend/docker/lint.py new file mode 100644 index 00000000000..190561407a7 --- /dev/null +++ b/src/python/pants/backend/docker/lint.py @@ -0,0 +1,100 @@ +from dataclasses import dataclass + +from pants.backend.docker.target_types import DockerImageSources +from pants.core.goals.lint import LintRequest, LintResult, LintResults +from pants.core.util_rules.external_tool import ( + DownloadedExternalTool, + ExternalToolRequest, + TemplatedExternalTool, +) +from pants.core.util_rules.source_files import SourceFiles, SourceFilesRequest +from pants.engine.fs import Digest, MergeDigests +from pants.engine.platform import Platform +from pants.engine.process import FallibleProcessResult, Process +from pants.engine.rules import Get, MultiGet, collect_rules, rule +from pants.engine.target import BoolField, FieldSet, Target +from pants.engine.unions import UnionRule +from pants.util.logging import LogLevel +from pants.util.strutil import pluralize + + +class HadolintTool(TemplatedExternalTool): + options_scope = "download-hadolint" + name = "hadolint" + help = "Hadolint Dockerfile linter (https://github.com/hadolint/hadolint)" + + default_version = "v2.6.0" + default_url_template = ( + "https://github.com/hadolint/hadolint/releases/download/{version}/hadolint-{platform}" + ) + default_url_platform_mapping = { + "darwin": "Darwin-x86_64", + "linux": "Linux-x86_64", + "windows": "Windows-x86_64.exe", + } + default_known_versions = [ + "v2.6.0|darwin |7d41496bf591f2b9c7daa76d4aa1db04ea97b9e11b44a24a4e404a10aab33686|2392080", + "v2.6.0|linux |152e3c3375f26711650d4e11f9e382cf1bdf3f912d7379823e8fac4b1bce88d6|5812840", + "v2.6.0|windows|f74ed11f5b24c0065bed273625c5a1cef858738393d1cf7fa0c2e163fa3dad9d|5170176", + ] + + +class SkipHadolintField(BoolField): + alias = "skip_hadolint" + default = False + help = "If true, don't run hadolint on this target's Dockerfile." + + +@dataclass(frozen=True) +class HadolintFieldSet(FieldSet): + required_fields = (DockerImageSources,) + + sources: DockerImageSources + + @classmethod + def opt_out(cls, tgt: Target) -> bool: + return tgt.get(SkipHadolintField).value + + +class HadolintRequest(LintRequest): + field_set_type = HadolintFieldSet + + +@rule(desc="Lint using Hadolint") +async def hadolint_lint(request: HadolintRequest, hadolint: HadolintTool) -> LintResults: + tool, sources = await MultiGet( + Get(DownloadedExternalTool, ExternalToolRequest, hadolint.get_request(Platform.current)), + Get( + SourceFiles, SourceFilesRequest([field_set.sources for field_set in request.field_sets]) + ), + ) + + input_digest = await Get(Digest, MergeDigests((sources.snapshot.digest, tool.digest))) + + result = await Get( + FallibleProcessResult, + Process( + argv=[tool.exe, *sources.snapshot.files], + input_digest=input_digest, + description=f"Run `hadolint` on {pluralize(len(sources.snapshot.files), 'Dockerfile')}.", + level=LogLevel.DEBUG, + ), + ) + + return LintResults( + [ + LintResult( + exit_code=result.exit_code, + stdout=result.stdout.decode(), + stderr=result.stderr.decode(), + ) + ], + linter_name="hadolint", + ) + + +def rules(): + return [ + *collect_rules(), + UnionRule(LintRequest, HadolintRequest), + ] diff --git a/src/python/pants/backend/experimental/docker/register.py b/src/python/pants/backend/experimental/docker/register.py index bdfaba43d02..c5c8f6060be 100644 --- a/src/python/pants/backend/experimental/docker/register.py +++ b/src/python/pants/backend/experimental/docker/register.py @@ -1,12 +1,16 @@ # Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). +from pants.backend.docker import lint from pants.backend.docker import tailor from pants.backend.docker.target_types import DockerImage def rules(): - return (*tailor.rules(),) + return ( + *lint.rules(), + *tailor.rules(), + ) def target_types(): From 3e51f136fb1f514ff5d7250cdf2663ce44d3c378 Mon Sep 17 00:00:00 2001 From: Andreas Stenius Date: Thu, 29 Jul 2021 11:36:59 +0200 Subject: [PATCH 2/8] Refactor hadolint tool integration. # Rust tests and lints will be skipped. Delete if not intended. [ci skip-rust] # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] --- src/python/pants/backend/docker/lint.py | 100 ------------ src/python/pants/backend/docker/lint/BUILD | 4 + .../pants/backend/docker/lint/__init__.py | 0 .../pants/backend/docker/lint/hadolint/BUILD | 5 + .../backend/docker/lint/hadolint/__init__.py | 0 .../backend/docker/lint/hadolint/register.py | 12 ++ .../backend/docker/lint/hadolint/rules.py | 117 ++++++++++++++ .../lint/hadolint/rules_integration_test.py | 147 ++++++++++++++++++ .../docker/lint/hadolint/skip_field.py | 15 ++ .../backend/docker/lint/hadolint/subsystem.py | 79 ++++++++++ .../backend/experimental/docker/register.py | 6 +- src/python/pants/init/BUILD | 1 + 12 files changed, 381 insertions(+), 105 deletions(-) delete mode 100644 src/python/pants/backend/docker/lint.py create mode 100644 src/python/pants/backend/docker/lint/BUILD create mode 100644 src/python/pants/backend/docker/lint/__init__.py create mode 100644 src/python/pants/backend/docker/lint/hadolint/BUILD create mode 100644 src/python/pants/backend/docker/lint/hadolint/__init__.py create mode 100644 src/python/pants/backend/docker/lint/hadolint/register.py create mode 100644 src/python/pants/backend/docker/lint/hadolint/rules.py create mode 100644 src/python/pants/backend/docker/lint/hadolint/rules_integration_test.py create mode 100644 src/python/pants/backend/docker/lint/hadolint/skip_field.py create mode 100644 src/python/pants/backend/docker/lint/hadolint/subsystem.py diff --git a/src/python/pants/backend/docker/lint.py b/src/python/pants/backend/docker/lint.py deleted file mode 100644 index 190561407a7..00000000000 --- a/src/python/pants/backend/docker/lint.py +++ /dev/null @@ -1,100 +0,0 @@ -from dataclasses import dataclass - -from pants.backend.docker.target_types import DockerImageSources -from pants.core.goals.lint import LintRequest, LintResult, LintResults -from pants.core.util_rules.external_tool import ( - DownloadedExternalTool, - ExternalToolRequest, - TemplatedExternalTool, -) -from pants.core.util_rules.source_files import SourceFiles, SourceFilesRequest -from pants.engine.fs import Digest, MergeDigests -from pants.engine.platform import Platform -from pants.engine.process import FallibleProcessResult, Process -from pants.engine.rules import Get, MultiGet, collect_rules, rule -from pants.engine.target import BoolField, FieldSet, Target -from pants.engine.unions import UnionRule -from pants.util.logging import LogLevel -from pants.util.strutil import pluralize - - -class HadolintTool(TemplatedExternalTool): - options_scope = "download-hadolint" - name = "hadolint" - help = "Hadolint Dockerfile linter (https://github.com/hadolint/hadolint)" - - default_version = "v2.6.0" - default_url_template = ( - "https://github.com/hadolint/hadolint/releases/download/{version}/hadolint-{platform}" - ) - default_url_platform_mapping = { - "darwin": "Darwin-x86_64", - "linux": "Linux-x86_64", - "windows": "Windows-x86_64.exe", - } - default_known_versions = [ - "v2.6.0|darwin |7d41496bf591f2b9c7daa76d4aa1db04ea97b9e11b44a24a4e404a10aab33686|2392080", - "v2.6.0|linux |152e3c3375f26711650d4e11f9e382cf1bdf3f912d7379823e8fac4b1bce88d6|5812840", - "v2.6.0|windows|f74ed11f5b24c0065bed273625c5a1cef858738393d1cf7fa0c2e163fa3dad9d|5170176", - ] - - -class SkipHadolintField(BoolField): - alias = "skip_hadolint" - default = False - help = "If true, don't run hadolint on this target's Dockerfile." - - -@dataclass(frozen=True) -class HadolintFieldSet(FieldSet): - required_fields = (DockerImageSources,) - - sources: DockerImageSources - - @classmethod - def opt_out(cls, tgt: Target) -> bool: - return tgt.get(SkipHadolintField).value - - -class HadolintRequest(LintRequest): - field_set_type = HadolintFieldSet - - -@rule(desc="Lint using Hadolint") -async def hadolint_lint(request: HadolintRequest, hadolint: HadolintTool) -> LintResults: - tool, sources = await MultiGet( - Get(DownloadedExternalTool, ExternalToolRequest, hadolint.get_request(Platform.current)), - Get( - SourceFiles, SourceFilesRequest([field_set.sources for field_set in request.field_sets]) - ), - ) - - input_digest = await Get(Digest, MergeDigests((sources.snapshot.digest, tool.digest))) - - result = await Get( - FallibleProcessResult, - Process( - argv=[tool.exe, *sources.snapshot.files], - input_digest=input_digest, - description=f"Run `hadolint` on {pluralize(len(sources.snapshot.files), 'Dockerfile')}.", - level=LogLevel.DEBUG, - ), - ) - - return LintResults( - [ - LintResult( - exit_code=result.exit_code, - stdout=result.stdout.decode(), - stderr=result.stderr.decode(), - ) - ], - linter_name="hadolint", - ) - - -def rules(): - return [ - *collect_rules(), - UnionRule(LintRequest, HadolintRequest), - ] diff --git a/src/python/pants/backend/docker/lint/BUILD b/src/python/pants/backend/docker/lint/BUILD new file mode 100644 index 00000000000..76e8c47837f --- /dev/null +++ b/src/python/pants/backend/docker/lint/BUILD @@ -0,0 +1,4 @@ +# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +python_library() diff --git a/src/python/pants/backend/docker/lint/__init__.py b/src/python/pants/backend/docker/lint/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/python/pants/backend/docker/lint/hadolint/BUILD b/src/python/pants/backend/docker/lint/hadolint/BUILD new file mode 100644 index 00000000000..7ea1f3b46f2 --- /dev/null +++ b/src/python/pants/backend/docker/lint/hadolint/BUILD @@ -0,0 +1,5 @@ +# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +python_library() +python_tests(name="tests") diff --git a/src/python/pants/backend/docker/lint/hadolint/__init__.py b/src/python/pants/backend/docker/lint/hadolint/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/python/pants/backend/docker/lint/hadolint/register.py b/src/python/pants/backend/docker/lint/hadolint/register.py new file mode 100644 index 00000000000..b6ea9362861 --- /dev/null +++ b/src/python/pants/backend/docker/lint/hadolint/register.py @@ -0,0 +1,12 @@ +# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from pants.backend.docker.lint.hadolint import skip_field +from pants.backend.docker.lint.hadolint.rules import rules as hadolint_rules + + +def rules(): + return ( + *hadolint_rules(), + *skip_field.rules(), + ) diff --git a/src/python/pants/backend/docker/lint/hadolint/rules.py b/src/python/pants/backend/docker/lint/hadolint/rules.py new file mode 100644 index 00000000000..54898589b0d --- /dev/null +++ b/src/python/pants/backend/docker/lint/hadolint/rules.py @@ -0,0 +1,117 @@ +# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). +from __future__ import annotations + +import os.path +from dataclasses import dataclass + +from pants.backend.docker.lint.hadolint.skip_field import SkipHadolintField +from pants.backend.docker.lint.hadolint.subsystem import Hadolint +from pants.backend.docker.target_types import DockerImageSources +from pants.core.goals.lint import LintRequest, LintResult, LintResults +from pants.core.util_rules.config_files import ConfigFiles, ConfigFilesRequest +from pants.core.util_rules.external_tool import DownloadedExternalTool, ExternalToolRequest +from pants.core.util_rules.source_files import SourceFiles, SourceFilesRequest +from pants.engine.fs import Digest, MergeDigests +from pants.engine.platform import Platform +from pants.engine.process import FallibleProcessResult, Process +from pants.engine.rules import Get, MultiGet, collect_rules, rule +from pants.engine.target import FieldSet, Target +from pants.engine.unions import UnionRule +from pants.util.logging import LogLevel +from pants.util.strutil import pluralize + + +@dataclass(frozen=True) +class HadolintFieldSet(FieldSet): + required_fields = (DockerImageSources,) + + sources: DockerImageSources + + @classmethod + def opt_out(cls, tgt: Target) -> bool: + return tgt.get(SkipHadolintField).value + + +class HadolintRequest(LintRequest): + field_set_type = HadolintFieldSet + + +@rule(desc="Lint with Hadolint", level=LogLevel.DEBUG) +async def run_hadolint(request: HadolintRequest, hadolint: Hadolint) -> LintResults: + if hadolint.skip: + return LintResults([], linter_name="Hadolint") + + downloaded_hadolint, sources = await MultiGet( + Get(DownloadedExternalTool, ExternalToolRequest, hadolint.get_request(Platform.current)), + Get( + SourceFiles, + SourceFilesRequest( + [field_set.sources for field_set in request.field_sets], + for_sources_types=(DockerImageSources,), + enable_codegen=True, + ), + ), + ) + config_files = await Get( + ConfigFiles, ConfigFilesRequest, hadolint.config_request(sources.snapshot.dirs) + ) + input_digest = await Get( + Digest, + MergeDigests( + ( + sources.snapshot.digest, + downloaded_hadolint.digest, + config_files.snapshot.digest, + ) + ), + ) + + # As hadolint uses a single config file, we need to partition our runs per config file + # discovered. + files_with_config = _group_files_with_config( + sources.snapshot.files, config_files.snapshot.files + ) + processes = [ + Process( + argv=[downloaded_hadolint.exe, *config, *hadolint.args, *files], + input_digest=input_digest, + description=f"Run `hadolint` on {pluralize(len(files), 'Dockerfile')}.", + level=LogLevel.DEBUG, + ) + for files, config in files_with_config + ] + process_results = await MultiGet(Get(FallibleProcessResult, Process, p) for p in processes) + results = [ + LintResult.from_fallible_process_result(process_result) + for process_result in process_results + ] + return LintResults(results, linter_name="hadolint") + + +def _group_files_with_config( + source_files: tuple[str, ...], config_files: tuple[str, ...] +) -> list[tuple[tuple[str, ...], list[str]]]: + """Group all source files that is in the same directory or below a config file.""" + groups = [] + consumed_files: list[str] = [] + + for config_file in config_files: + path = os.path.dirname(config_file) + files = {source_file for source_file in source_files if source_file.startswith(path)} + if files: + groups.append((tuple(files), ["--config", config_file])) + consumed_files.extend(files) + + if len(consumed_files) < len(source_files): + files = set(source_files) - set(consumed_files) + groups.append((tuple(files), [])) + + return groups + + +def rules(): + return [ + *collect_rules(), + UnionRule(LintRequest, HadolintRequest), + ] diff --git a/src/python/pants/backend/docker/lint/hadolint/rules_integration_test.py b/src/python/pants/backend/docker/lint/hadolint/rules_integration_test.py new file mode 100644 index 00000000000..b8a93d643c4 --- /dev/null +++ b/src/python/pants/backend/docker/lint/hadolint/rules_integration_test.py @@ -0,0 +1,147 @@ +# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import annotations + +from textwrap import dedent + +import pytest + +from pants.backend.docker.lint.hadolint.rules import HadolintFieldSet, HadolintRequest +from pants.backend.docker.lint.hadolint.rules import rules as hadolint_rules +from pants.backend.docker.target_types import DockerImage +from pants.core.goals.lint import LintResult, LintResults +from pants.core.util_rules import config_files, external_tool, source_files +from pants.engine.addresses import Address +from pants.engine.target import Target +from pants.testutil.rule_runner import QueryRule, RuleRunner + + +@pytest.fixture +def rule_runner() -> RuleRunner: + return RuleRunner( + rules=[ + *hadolint_rules(), + *config_files.rules(), + *external_tool.rules(), + *source_files.rules(), + QueryRule(LintResults, [HadolintRequest]), + ], + target_types=[DockerImage], + ) + + +GOOD_FILE = dedent( + """ + FROM python:3.8 + """ +) + +BAD_FILE = dedent( + """ + FROM python + """ +) + + +def run_hadolint( + rule_runner: RuleRunner, targets: list[Target], *, extra_args: list[str] | None = None +) -> tuple[LintResult, ...]: + rule_runner.set_options( + ["--backend-packages=pants.backend.docker.lint.hadolint", *(extra_args or ())], + env_inherit={"PATH"}, + ) + results = rule_runner.request( + LintResults, + [HadolintRequest(HadolintFieldSet.create(tgt) for tgt in targets)], + ) + return results.results + + +def assert_success( + rule_runner: RuleRunner, target: Target, *, extra_args: list[str] | None = None +) -> None: + result = run_hadolint(rule_runner, [target], extra_args=extra_args) + assert len(result) == 1 + assert result[0].exit_code == 0 + assert not result[0].stdout + assert not result[0].stderr + + +def test_passing(rule_runner: RuleRunner) -> None: + rule_runner.write_files({"Dockerfile": GOOD_FILE, "BUILD": "docker_image(name='t')"}) + tgt = rule_runner.get_target(Address("", target_name="t")) + assert_success(rule_runner, tgt) + + +def test_failing(rule_runner: RuleRunner) -> None: + rule_runner.write_files({"Dockerfile": BAD_FILE, "BUILD": "docker_image(name='t')"}) + tgt = rule_runner.get_target(Address("", target_name="t")) + result = run_hadolint(rule_runner, [tgt]) + assert len(result) == 1 + assert result[0].exit_code == 1 + assert "Dockerfile:2 " in result[0].stdout + + +def test_multiple_targets(rule_runner: RuleRunner) -> None: + rule_runner.write_files( + { + "Dockerfile.good": GOOD_FILE, + "Dockerfile.bad": BAD_FILE, + "BUILD": dedent( + """ + docker_image(name="good", sources=("Dockerfile.good",)) + docker_image(name="bad", sources=("Dockerfile.bad",)) + """ + ), + } + ) + tgts = [ + rule_runner.get_target( + Address("", target_name="good", relative_file_path="Dockerfile.good") + ), + rule_runner.get_target(Address("", target_name="bad", relative_file_path="Dockerfile.bad")), + ] + result = run_hadolint(rule_runner, tgts) + assert len(result) == 1 + assert result[0].exit_code == 1 + assert "Dockerfile.good" not in result[0].stdout + assert "Dockerfile.bad:2 " in result[0].stdout + + +def test_config_files(rule_runner: RuleRunner) -> None: + rule_runner.write_files( + { + "a/Dockerfile": BAD_FILE, + "a/BUILD": "docker_image()", + "a/.hadolint.yaml": "ignored: [DL3006]", + "b/Dockerfile": BAD_FILE, + "b/BUILD": "docker_image()", + } + ) + tgts = [ + rule_runner.get_target(Address("a")), + rule_runner.get_target(Address("b")), + ] + result = run_hadolint(rule_runner, tgts) + # We get two runs of hadolint, for the `a` and `b` directories respectively. + assert len(result) == 2 + assert result[0].exit_code == 0 + assert "a/Dockerfile" not in result[0].stdout + assert "b/Dockerfile:2 " not in result[0].stdout + assert result[1].exit_code == 1 + assert "a/Dockerfile" not in result[1].stdout + assert "b/Dockerfile:2 " in result[1].stdout + + +def test_passthrough_args(rule_runner: RuleRunner) -> None: + rule_runner.write_files({"Dockerfile": BAD_FILE, "BUILD": "docker_image(name='t')"}) + tgt = rule_runner.get_target(Address("", target_name="t")) + assert_success(rule_runner, tgt, extra_args=["--hadolint-args=--ignore DL3006"]) + + +def test_skip(rule_runner: RuleRunner) -> None: + rule_runner.write_files({"Dockerfile": BAD_FILE, "BUILD": "docker_image(name='t')"}) + tgt = rule_runner.get_target(Address("", target_name="t")) + result = run_hadolint(rule_runner, [tgt], extra_args=["--hadolint-skip"]) + assert not result diff --git a/src/python/pants/backend/docker/lint/hadolint/skip_field.py b/src/python/pants/backend/docker/lint/hadolint/skip_field.py new file mode 100644 index 00000000000..7dba9645100 --- /dev/null +++ b/src/python/pants/backend/docker/lint/hadolint/skip_field.py @@ -0,0 +1,15 @@ +# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from pants.backend.docker.target_types import DockerImage +from pants.engine.target import BoolField + + +class SkipHadolintField(BoolField): + alias = "skip_hadolint" + default = False + help = "If true, don't run hadolint on this target's Dockerfile." + + +def rules(): + return [DockerImage.register_plugin_field(SkipHadolintField)] diff --git a/src/python/pants/backend/docker/lint/hadolint/subsystem.py b/src/python/pants/backend/docker/lint/hadolint/subsystem.py new file mode 100644 index 00000000000..353ba68e63e --- /dev/null +++ b/src/python/pants/backend/docker/lint/hadolint/subsystem.py @@ -0,0 +1,79 @@ +# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import annotations + +import os +from typing import Iterable, cast + +from pants.core.util_rules.config_files import ConfigFilesRequest +from pants.core.util_rules.external_tool import TemplatedExternalTool +from pants.option.custom_types import shell_str + + +class Hadolint(TemplatedExternalTool): + options_scope = "hadolint" + name = "hadolint" + help = "A linter for Dockerfiles." + + default_version = "v2.6.0" + default_known_versions = [ + f"{default_version}|macos_arm64 |7d41496bf591f2b9c7daa76d4aa1db04ea97b9e11b44a24a4e404a10aab33686|2392080", + f"{default_version}|macos_x86_64|7d41496bf591f2b9c7daa76d4aa1db04ea97b9e11b44a24a4e404a10aab33686|2392080", + f"{default_version}|linux_x86_64|152e3c3375f26711650d4e11f9e382cf1bdf3f912d7379823e8fac4b1bce88d6|5812840", + ] + default_url_template = ( + "https://github.com/hadolint/hadolint/releases/download/{version}/hadolint-{platform}" + ) + default_url_platform_mapping = { + "macos_arm64": "Darwin-x86_64", + "macos_x86_64": "Darwin-x86_64", + "linux_x86_64": "Linux-x86_64", + } + + @classmethod + def register_options(cls, register): + super().register_options(register) + register( + "--skip", + type=bool, + default=False, + help="Don't use Hadolint when running `./pants lint`.", + ) + register( + "--args", + type=list, + member_type=shell_str, + help=( + "Arguments to pass directly to Hadolint, e.g. `--hadolint-args='--format json'`.'" + ), + ) + register( + "--config-discovery", + type=bool, + default=True, + advanced=True, + help=( + "If true, Pants will include all relevant `.hadolint.yaml` and `.hadolint.yml` files " + "during runs." + ), + ) + + @property + def skip(self) -> bool: + return cast(bool, self.options.skip) + + @property + def args(self) -> tuple[str, ...]: + return tuple(self.options.args) + + def config_request(self, dirs: Iterable[str]) -> ConfigFilesRequest: + # Refer to https://github.com/hadolint/hadolint#configure for how config files are + # discovered. + candidates = [] + for d in ("", *dirs): + candidates.append(os.path.join(d, ".hadolint.yaml")) + candidates.append(os.path.join(d, ".hadolint.yml")) + return ConfigFilesRequest( + discovery=cast(bool, self.options.config_discovery), check_existence=candidates + ) diff --git a/src/python/pants/backend/experimental/docker/register.py b/src/python/pants/backend/experimental/docker/register.py index c5c8f6060be..bdfaba43d02 100644 --- a/src/python/pants/backend/experimental/docker/register.py +++ b/src/python/pants/backend/experimental/docker/register.py @@ -1,16 +1,12 @@ # Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). -from pants.backend.docker import lint from pants.backend.docker import tailor from pants.backend.docker.target_types import DockerImage def rules(): - return ( - *lint.rules(), - *tailor.rules(), - ) + return (*tailor.rules(),) def target_types(): diff --git a/src/python/pants/init/BUILD b/src/python/pants/init/BUILD index 47ba22bf7cc..b4dbaf68387 100644 --- a/src/python/pants/init/BUILD +++ b/src/python/pants/init/BUILD @@ -8,6 +8,7 @@ target( dependencies=[ 'src/python/pants/backend/awslambda/python', 'src/python/pants/backend/codegen/protobuf/python', + 'src/python/pants/backend/docker/lint/hadolint', 'src/python/pants/backend/experimental/docker', 'src/python/pants/backend/experimental/go', 'src/python/pants/backend/experimental/go/lint/gofmt', From ffbda3f3537dc1e153cddda753e860b8fcdb3833 Mon Sep 17 00:00:00 2001 From: Andreas Stenius Date: Thu, 29 Jul 2021 12:24:00 +0200 Subject: [PATCH 3/8] Add --hadolint-config option. # Rust tests and lints will be skipped. Delete if not intended. [ci skip-rust] # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] --- .../backend/docker/lint/hadolint/rules.py | 23 ++++++++++----- .../lint/hadolint/rules_integration_test.py | 3 ++ .../backend/docker/lint/hadolint/subsystem.py | 29 ++++++++++++++++--- 3 files changed, 43 insertions(+), 12 deletions(-) diff --git a/src/python/pants/backend/docker/lint/hadolint/rules.py b/src/python/pants/backend/docker/lint/hadolint/rules.py index 54898589b0d..a54262041c3 100644 --- a/src/python/pants/backend/docker/lint/hadolint/rules.py +++ b/src/python/pants/backend/docker/lint/hadolint/rules.py @@ -70,7 +70,9 @@ async def run_hadolint(request: HadolintRequest, hadolint: Hadolint) -> LintResu # As hadolint uses a single config file, we need to partition our runs per config file # discovered. files_with_config = _group_files_with_config( - sources.snapshot.files, config_files.snapshot.files + set(sources.snapshot.files), + config_files.snapshot.files, + not hadolint.config, ) processes = [ Process( @@ -90,21 +92,26 @@ async def run_hadolint(request: HadolintRequest, hadolint: Hadolint) -> LintResu def _group_files_with_config( - source_files: tuple[str, ...], config_files: tuple[str, ...] + source_files: set[str], config_files: tuple[str, ...], config_files_discovered: bool ) -> list[tuple[tuple[str, ...], list[str]]]: - """Group all source files that is in the same directory or below a config file.""" + """If config_files_discovered, group all source files that is in the same directory or below a + config file, otherwise, all files will be kept in one group per config file that was provided as + option.""" groups = [] - consumed_files: list[str] = [] + consumed_files: set[str] = set() for config_file in config_files: - path = os.path.dirname(config_file) - files = {source_file for source_file in source_files if source_file.startswith(path)} + if not config_files_discovered: + files = source_files + else: + path = os.path.dirname(config_file) + files = {source_file for source_file in source_files if source_file.startswith(path)} if files: groups.append((tuple(files), ["--config", config_file])) - consumed_files.extend(files) + consumed_files.update(files) if len(consumed_files) < len(source_files): - files = set(source_files) - set(consumed_files) + files = set(source_files) - consumed_files groups.append((tuple(files), [])) return groups diff --git a/src/python/pants/backend/docker/lint/hadolint/rules_integration_test.py b/src/python/pants/backend/docker/lint/hadolint/rules_integration_test.py index b8a93d643c4..ce24f3e74ed 100644 --- a/src/python/pants/backend/docker/lint/hadolint/rules_integration_test.py +++ b/src/python/pants/backend/docker/lint/hadolint/rules_integration_test.py @@ -133,6 +133,9 @@ def test_config_files(rule_runner: RuleRunner) -> None: assert "a/Dockerfile" not in result[1].stdout assert "b/Dockerfile:2 " in result[1].stdout + tgt = rule_runner.get_target(Address("b")) + assert_success(rule_runner, tgt, extra_args=["--hadolint-config=a/.hadolint.yaml"]) + def test_passthrough_args(rule_runner: RuleRunner) -> None: rule_runner.write_files({"Dockerfile": BAD_FILE, "BUILD": "docker_image(name='t')"}) diff --git a/src/python/pants/backend/docker/lint/hadolint/subsystem.py b/src/python/pants/backend/docker/lint/hadolint/subsystem.py index 353ba68e63e..975709a9410 100644 --- a/src/python/pants/backend/docker/lint/hadolint/subsystem.py +++ b/src/python/pants/backend/docker/lint/hadolint/subsystem.py @@ -8,7 +8,7 @@ from pants.core.util_rules.config_files import ConfigFilesRequest from pants.core.util_rules.external_tool import TemplatedExternalTool -from pants.option.custom_types import shell_str +from pants.option.custom_types import file_option, shell_str class Hadolint(TemplatedExternalTool): @@ -48,14 +48,28 @@ def register_options(cls, register): "Arguments to pass directly to Hadolint, e.g. `--hadolint-args='--format json'`.'" ), ) + register( + "--config", + type=file_option, + default=None, + advanced=True, + help=( + "Path to an YAML config file understood by Hadolint " + "(https://github.com/hadolint/hadolint#configure).\n\n" + f"Setting this option will disable `[{cls.options_scope}].config_discovery`. Use " + "this option if the config is located in a non-standard location." + ), + ) register( "--config-discovery", type=bool, default=True, advanced=True, help=( - "If true, Pants will include all relevant `.hadolint.yaml` and `.hadolint.yml` files " - "during runs." + "If true, Pants will include all relevant config files during runs " + "(`.hadolint.yaml` and `.hadolint.yml`).\n\n" + f"Use `[{cls.options_scope}].config` instead if your config is in a " + "non-standard location." ), ) @@ -67,6 +81,10 @@ def skip(self) -> bool: def args(self) -> tuple[str, ...]: return tuple(self.options.args) + @property + def config(self) -> str | None: + return cast("str | None", self.options.config) + def config_request(self, dirs: Iterable[str]) -> ConfigFilesRequest: # Refer to https://github.com/hadolint/hadolint#configure for how config files are # discovered. @@ -75,5 +93,8 @@ def config_request(self, dirs: Iterable[str]) -> ConfigFilesRequest: candidates.append(os.path.join(d, ".hadolint.yaml")) candidates.append(os.path.join(d, ".hadolint.yml")) return ConfigFilesRequest( - discovery=cast(bool, self.options.config_discovery), check_existence=candidates + specified=self.config, + specified_option_name=f"[{self.options_scope}].config", + discovery=cast(bool, self.options.config_discovery), + check_existence=candidates, ) From f52796c09659a03a75e652144b439aa04435cd79 Mon Sep 17 00:00:00 2001 From: Andreas Stenius Date: Thu, 29 Jul 2021 13:46:57 +0200 Subject: [PATCH 4/8] Replace os.path for PurePath for improved file relation tests. # Rust tests and lints will be skipped. Delete if not intended. [ci skip-rust] # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] --- .../backend/docker/lint/hadolint/rules.py | 16 +++-- .../docker/lint/hadolint/rules_test.py | 64 +++++++++++++++++++ 2 files changed, 73 insertions(+), 7 deletions(-) create mode 100644 src/python/pants/backend/docker/lint/hadolint/rules_test.py diff --git a/src/python/pants/backend/docker/lint/hadolint/rules.py b/src/python/pants/backend/docker/lint/hadolint/rules.py index a54262041c3..bf8d45389f0 100644 --- a/src/python/pants/backend/docker/lint/hadolint/rules.py +++ b/src/python/pants/backend/docker/lint/hadolint/rules.py @@ -2,8 +2,8 @@ # Licensed under the Apache License, Version 2.0 (see LICENSE). from __future__ import annotations -import os.path from dataclasses import dataclass +from pathlib import PurePath from pants.backend.docker.lint.hadolint.skip_field import SkipHadolintField from pants.backend.docker.lint.hadolint.subsystem import Hadolint @@ -70,7 +70,7 @@ async def run_hadolint(request: HadolintRequest, hadolint: Hadolint) -> LintResu # As hadolint uses a single config file, we need to partition our runs per config file # discovered. files_with_config = _group_files_with_config( - set(sources.snapshot.files), + sources.snapshot.files, config_files.snapshot.files, not hadolint.config, ) @@ -92,22 +92,24 @@ async def run_hadolint(request: HadolintRequest, hadolint: Hadolint) -> LintResu def _group_files_with_config( - source_files: set[str], config_files: tuple[str, ...], config_files_discovered: bool + source_files: tuple[str, ...], config_files: tuple[str, ...], config_files_discovered: bool ) -> list[tuple[tuple[str, ...], list[str]]]: """If config_files_discovered, group all source files that is in the same directory or below a config file, otherwise, all files will be kept in one group per config file that was provided as option.""" groups = [] consumed_files: set[str] = set() + source_paths = {PurePath(source_file) for source_file in source_files} for config_file in config_files: if not config_files_discovered: - files = source_files + files = set(source_files) else: - path = os.path.dirname(config_file) - files = {source_file for source_file in source_files if source_file.startswith(path)} + config_path = PurePath(config_file).parent + files = {str(source) for source in source_paths if config_path in source.parents} if files: - groups.append((tuple(files), ["--config", config_file])) + # Sort files, to make the order predictable for the tests. + groups.append((tuple(sorted(files)), ["--config", config_file])) consumed_files.update(files) if len(consumed_files) < len(source_files): diff --git a/src/python/pants/backend/docker/lint/hadolint/rules_test.py b/src/python/pants/backend/docker/lint/hadolint/rules_test.py new file mode 100644 index 00000000000..ca0c539ba66 --- /dev/null +++ b/src/python/pants/backend/docker/lint/hadolint/rules_test.py @@ -0,0 +1,64 @@ +# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +import pytest + +from pants.backend.docker.lint.hadolint.rules import _group_files_with_config + + +@pytest.mark.parametrize( + "source_files, config_files, config_files_discovered, expected", + [ + ( + {"a/Dockerfile", "b/Dockerfile"}, + ("a/.hadolint.yaml",), + True, + [ + (("a/Dockerfile",), ["--config", "a/.hadolint.yaml"]), + (("b/Dockerfile",), []), + ], + ), + ( + {"a/Dockerfile", "b/Dockerfile"}, + ("hadolint.yaml",), + False, + [ + (("a/Dockerfile", "b/Dockerfile"), ["--config", "hadolint.yaml"]), + ], + ), + ( + {"a/Dockerfile", "aa/Dockerfile"}, + ("a/.hadolint.yaml",), + True, + [ + (("a/Dockerfile",), ["--config", "a/.hadolint.yaml"]), + (("aa/Dockerfile",), []), + ], + ), + ( + {"a/Dockerfile", "b/Dockerfile", "c/Dockerfile", "d/Dockerfile", "c/e/Dockerfile"}, + ("a/.hadolint.yaml", "c/.hadolint.yaml"), + True, + [ + (("a/Dockerfile",), ["--config", "a/.hadolint.yaml"]), + ( + ( + "c/Dockerfile", + "c/e/Dockerfile", + ), + ["--config", "c/.hadolint.yaml"], + ), + ( + ( + "b/Dockerfile", + "d/Dockerfile", + ), + [], + ), + ], + ), + ], +) +def test_group_files_with_config(source_files, config_files, config_files_discovered, expected): + actual = _group_files_with_config(source_files, config_files, config_files_discovered) + assert actual == expected From 6118eeeecc289967400da9abd0209a6d24b0d3b6 Mon Sep 17 00:00:00 2001 From: Andreas Stenius Date: Thu, 29 Jul 2021 23:13:16 +0200 Subject: [PATCH 5/8] drop multiple config file support. # Rust tests and lints will be skipped. Delete if not intended. [ci skip-rust] # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] --- .../backend/docker/lint/hadolint/rules.py | 64 +++++-------------- .../lint/hadolint/rules_integration_test.py | 35 ++++------ .../docker/lint/hadolint/rules_test.py | 64 ------------------- 3 files changed, 28 insertions(+), 135 deletions(-) delete mode 100644 src/python/pants/backend/docker/lint/hadolint/rules_test.py diff --git a/src/python/pants/backend/docker/lint/hadolint/rules.py b/src/python/pants/backend/docker/lint/hadolint/rules.py index bf8d45389f0..76eaf2b4653 100644 --- a/src/python/pants/backend/docker/lint/hadolint/rules.py +++ b/src/python/pants/backend/docker/lint/hadolint/rules.py @@ -3,7 +3,6 @@ from __future__ import annotations from dataclasses import dataclass -from pathlib import PurePath from pants.backend.docker.lint.hadolint.skip_field import SkipHadolintField from pants.backend.docker.lint.hadolint.subsystem import Hadolint @@ -37,6 +36,15 @@ class HadolintRequest(LintRequest): field_set_type = HadolintFieldSet +def generate_argv(source_files: SourceFiles, hadolint: Hadolint) -> tuple[str, ...]: + args = [] + if hadolint.config: + args.append(f"--config={hadolint.config}") + args.extend(hadolint.args) + args.extend(source_files.files) + return tuple(args) + + @rule(desc="Lint with Hadolint", level=LogLevel.DEBUG) async def run_hadolint(request: HadolintRequest, hadolint: Hadolint) -> LintResults: if hadolint.skip: @@ -66,59 +74,19 @@ async def run_hadolint(request: HadolintRequest, hadolint: Hadolint) -> LintResu ) ), ) - - # As hadolint uses a single config file, we need to partition our runs per config file - # discovered. - files_with_config = _group_files_with_config( - sources.snapshot.files, - config_files.snapshot.files, - not hadolint.config, - ) - processes = [ + process_result = await Get( + FallibleProcessResult, Process( - argv=[downloaded_hadolint.exe, *config, *hadolint.args, *files], + argv=[downloaded_hadolint.exe, *generate_argv(sources, hadolint)], input_digest=input_digest, - description=f"Run `hadolint` on {pluralize(len(files), 'Dockerfile')}.", + description=f"Run `hadolint` on {pluralize(len(sources.files), 'Dockerfile')}.", level=LogLevel.DEBUG, - ) - for files, config in files_with_config - ] - process_results = await MultiGet(Get(FallibleProcessResult, Process, p) for p in processes) - results = [ - LintResult.from_fallible_process_result(process_result) - for process_result in process_results - ] + ), + ) + results = [LintResult.from_fallible_process_result(process_result)] return LintResults(results, linter_name="hadolint") -def _group_files_with_config( - source_files: tuple[str, ...], config_files: tuple[str, ...], config_files_discovered: bool -) -> list[tuple[tuple[str, ...], list[str]]]: - """If config_files_discovered, group all source files that is in the same directory or below a - config file, otherwise, all files will be kept in one group per config file that was provided as - option.""" - groups = [] - consumed_files: set[str] = set() - source_paths = {PurePath(source_file) for source_file in source_files} - - for config_file in config_files: - if not config_files_discovered: - files = set(source_files) - else: - config_path = PurePath(config_file).parent - files = {str(source) for source in source_paths if config_path in source.parents} - if files: - # Sort files, to make the order predictable for the tests. - groups.append((tuple(sorted(files)), ["--config", config_file])) - consumed_files.update(files) - - if len(consumed_files) < len(source_files): - files = set(source_files) - consumed_files - groups.append((tuple(files), [])) - - return groups - - def rules(): return [ *collect_rules(), diff --git a/src/python/pants/backend/docker/lint/hadolint/rules_integration_test.py b/src/python/pants/backend/docker/lint/hadolint/rules_integration_test.py index ce24f3e74ed..984086a2f7d 100644 --- a/src/python/pants/backend/docker/lint/hadolint/rules_integration_test.py +++ b/src/python/pants/backend/docker/lint/hadolint/rules_integration_test.py @@ -31,24 +31,15 @@ def rule_runner() -> RuleRunner: ) -GOOD_FILE = dedent( - """ - FROM python:3.8 - """ -) - -BAD_FILE = dedent( - """ - FROM python - """ -) +GOOD_FILE = "FROM python:3.8" +BAD_FILE = "FROM python" def run_hadolint( rule_runner: RuleRunner, targets: list[Target], *, extra_args: list[str] | None = None ) -> tuple[LintResult, ...]: rule_runner.set_options( - ["--backend-packages=pants.backend.docker.lint.hadolint", *(extra_args or ())], + extra_args or (), env_inherit={"PATH"}, ) results = rule_runner.request( @@ -80,7 +71,7 @@ def test_failing(rule_runner: RuleRunner) -> None: result = run_hadolint(rule_runner, [tgt]) assert len(result) == 1 assert result[0].exit_code == 1 - assert "Dockerfile:2 " in result[0].stdout + assert "Dockerfile:1 " in result[0].stdout def test_multiple_targets(rule_runner: RuleRunner) -> None: @@ -106,17 +97,19 @@ def test_multiple_targets(rule_runner: RuleRunner) -> None: assert len(result) == 1 assert result[0].exit_code == 1 assert "Dockerfile.good" not in result[0].stdout - assert "Dockerfile.bad:2 " in result[0].stdout + assert "Dockerfile.bad:1 " in result[0].stdout def test_config_files(rule_runner: RuleRunner) -> None: rule_runner.write_files( { + ".hadolint.yaml": "ignored: [DL3006, DL3011]", "a/Dockerfile": BAD_FILE, "a/BUILD": "docker_image()", - "a/.hadolint.yaml": "ignored: [DL3006]", "b/Dockerfile": BAD_FILE, "b/BUILD": "docker_image()", + "c/BUILD": "docker_image()", + "c/Dockerfile": "EXPOSE 123456", } ) tgts = [ @@ -124,17 +117,13 @@ def test_config_files(rule_runner: RuleRunner) -> None: rule_runner.get_target(Address("b")), ] result = run_hadolint(rule_runner, tgts) - # We get two runs of hadolint, for the `a` and `b` directories respectively. - assert len(result) == 2 + assert len(result) == 1 assert result[0].exit_code == 0 assert "a/Dockerfile" not in result[0].stdout - assert "b/Dockerfile:2 " not in result[0].stdout - assert result[1].exit_code == 1 - assert "a/Dockerfile" not in result[1].stdout - assert "b/Dockerfile:2 " in result[1].stdout + assert "b/Dockerfile " not in result[0].stdout - tgt = rule_runner.get_target(Address("b")) - assert_success(rule_runner, tgt, extra_args=["--hadolint-config=a/.hadolint.yaml"]) + tgt = rule_runner.get_target(Address("c")) + assert_success(rule_runner, tgt, extra_args=["--hadolint-config=.hadolint.yaml"]) def test_passthrough_args(rule_runner: RuleRunner) -> None: diff --git a/src/python/pants/backend/docker/lint/hadolint/rules_test.py b/src/python/pants/backend/docker/lint/hadolint/rules_test.py deleted file mode 100644 index ca0c539ba66..00000000000 --- a/src/python/pants/backend/docker/lint/hadolint/rules_test.py +++ /dev/null @@ -1,64 +0,0 @@ -# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). -# Licensed under the Apache License, Version 2.0 (see LICENSE). - -import pytest - -from pants.backend.docker.lint.hadolint.rules import _group_files_with_config - - -@pytest.mark.parametrize( - "source_files, config_files, config_files_discovered, expected", - [ - ( - {"a/Dockerfile", "b/Dockerfile"}, - ("a/.hadolint.yaml",), - True, - [ - (("a/Dockerfile",), ["--config", "a/.hadolint.yaml"]), - (("b/Dockerfile",), []), - ], - ), - ( - {"a/Dockerfile", "b/Dockerfile"}, - ("hadolint.yaml",), - False, - [ - (("a/Dockerfile", "b/Dockerfile"), ["--config", "hadolint.yaml"]), - ], - ), - ( - {"a/Dockerfile", "aa/Dockerfile"}, - ("a/.hadolint.yaml",), - True, - [ - (("a/Dockerfile",), ["--config", "a/.hadolint.yaml"]), - (("aa/Dockerfile",), []), - ], - ), - ( - {"a/Dockerfile", "b/Dockerfile", "c/Dockerfile", "d/Dockerfile", "c/e/Dockerfile"}, - ("a/.hadolint.yaml", "c/.hadolint.yaml"), - True, - [ - (("a/Dockerfile",), ["--config", "a/.hadolint.yaml"]), - ( - ( - "c/Dockerfile", - "c/e/Dockerfile", - ), - ["--config", "c/.hadolint.yaml"], - ), - ( - ( - "b/Dockerfile", - "d/Dockerfile", - ), - [], - ), - ], - ), - ], -) -def test_group_files_with_config(source_files, config_files, config_files_discovered, expected): - actual = _group_files_with_config(source_files, config_files, config_files_discovered) - assert actual == expected From bef4cb72ff6609a35acec74f6907522c41d6fcd1 Mon Sep 17 00:00:00 2001 From: Andreas Stenius Date: Thu, 29 Jul 2021 23:18:35 +0200 Subject: [PATCH 6/8] only discover hadolint config files in project root. # Rust tests and lints will be skipped. Delete if not intended. [ci skip-rust] # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] --- .../pants/backend/docker/lint/hadolint/rules.py | 4 +--- .../pants/backend/docker/lint/hadolint/subsystem.py | 11 +++-------- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/src/python/pants/backend/docker/lint/hadolint/rules.py b/src/python/pants/backend/docker/lint/hadolint/rules.py index 76eaf2b4653..75fe9284dc5 100644 --- a/src/python/pants/backend/docker/lint/hadolint/rules.py +++ b/src/python/pants/backend/docker/lint/hadolint/rules.py @@ -61,9 +61,7 @@ async def run_hadolint(request: HadolintRequest, hadolint: Hadolint) -> LintResu ), ), ) - config_files = await Get( - ConfigFiles, ConfigFilesRequest, hadolint.config_request(sources.snapshot.dirs) - ) + config_files = await Get(ConfigFiles, ConfigFilesRequest, hadolint.config_request()) input_digest = await Get( Digest, MergeDigests( diff --git a/src/python/pants/backend/docker/lint/hadolint/subsystem.py b/src/python/pants/backend/docker/lint/hadolint/subsystem.py index 975709a9410..2b36708e84a 100644 --- a/src/python/pants/backend/docker/lint/hadolint/subsystem.py +++ b/src/python/pants/backend/docker/lint/hadolint/subsystem.py @@ -3,8 +3,7 @@ from __future__ import annotations -import os -from typing import Iterable, cast +from typing import cast from pants.core.util_rules.config_files import ConfigFilesRequest from pants.core.util_rules.external_tool import TemplatedExternalTool @@ -85,16 +84,12 @@ def args(self) -> tuple[str, ...]: def config(self) -> str | None: return cast("str | None", self.options.config) - def config_request(self, dirs: Iterable[str]) -> ConfigFilesRequest: + def config_request(self) -> ConfigFilesRequest: # Refer to https://github.com/hadolint/hadolint#configure for how config files are # discovered. - candidates = [] - for d in ("", *dirs): - candidates.append(os.path.join(d, ".hadolint.yaml")) - candidates.append(os.path.join(d, ".hadolint.yml")) return ConfigFilesRequest( specified=self.config, specified_option_name=f"[{self.options_scope}].config", discovery=cast(bool, self.options.config_discovery), - check_existence=candidates, + check_existence=[".hadolint.yaml", ".hadolint.yml"], ) From 6893cc3708bd0b9c3c2f2cbe3c69d5d58903cb70 Mon Sep 17 00:00:00 2001 From: Andreas Stenius Date: Fri, 30 Jul 2021 07:20:42 +0200 Subject: [PATCH 7/8] final tweaks. # Rust tests and lints will be skipped. Delete if not intended. [ci skip-rust] # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] --- src/python/pants/backend/docker/lint/hadolint/rules.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/python/pants/backend/docker/lint/hadolint/rules.py b/src/python/pants/backend/docker/lint/hadolint/rules.py index 75fe9284dc5..cacba09bbff 100644 --- a/src/python/pants/backend/docker/lint/hadolint/rules.py +++ b/src/python/pants/backend/docker/lint/hadolint/rules.py @@ -50,7 +50,7 @@ async def run_hadolint(request: HadolintRequest, hadolint: Hadolint) -> LintResu if hadolint.skip: return LintResults([], linter_name="Hadolint") - downloaded_hadolint, sources = await MultiGet( + downloaded_hadolint, sources, config_files = await MultiGet( Get(DownloadedExternalTool, ExternalToolRequest, hadolint.get_request(Platform.current)), Get( SourceFiles, @@ -60,8 +60,8 @@ async def run_hadolint(request: HadolintRequest, hadolint: Hadolint) -> LintResu enable_codegen=True, ), ), + Get(ConfigFiles, ConfigFilesRequest, hadolint.config_request()), ) - config_files = await Get(ConfigFiles, ConfigFilesRequest, hadolint.config_request()) input_digest = await Get( Digest, MergeDigests( @@ -81,8 +81,9 @@ async def run_hadolint(request: HadolintRequest, hadolint: Hadolint) -> LintResu level=LogLevel.DEBUG, ), ) - results = [LintResult.from_fallible_process_result(process_result)] - return LintResults(results, linter_name="hadolint") + return LintResults( + [LintResult.from_fallible_process_result(process_result)], linter_name="hadolint" + ) def rules(): From 8588126946595ca52a31dc1cc9c02764fb70ffa3 Mon Sep 17 00:00:00 2001 From: Andreas Stenius Date: Fri, 30 Jul 2021 07:28:23 +0200 Subject: [PATCH 8/8] move hadolint pluging register module to experimental. Signed-off-by: Andreas Stenius # Rust tests and lints will be skipped. Delete if not intended. [ci skip-rust] # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] --- .../pants/backend/experimental/docker/lint/hadolint/BUILD | 4 ++++ .../backend/experimental/docker/lint/hadolint/__init__.py | 0 .../{ => experimental}/docker/lint/hadolint/register.py | 0 src/python/pants/init/BUILD | 2 +- 4 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 src/python/pants/backend/experimental/docker/lint/hadolint/BUILD create mode 100644 src/python/pants/backend/experimental/docker/lint/hadolint/__init__.py rename src/python/pants/backend/{ => experimental}/docker/lint/hadolint/register.py (100%) diff --git a/src/python/pants/backend/experimental/docker/lint/hadolint/BUILD b/src/python/pants/backend/experimental/docker/lint/hadolint/BUILD new file mode 100644 index 00000000000..76e8c47837f --- /dev/null +++ b/src/python/pants/backend/experimental/docker/lint/hadolint/BUILD @@ -0,0 +1,4 @@ +# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +python_library() diff --git a/src/python/pants/backend/experimental/docker/lint/hadolint/__init__.py b/src/python/pants/backend/experimental/docker/lint/hadolint/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/python/pants/backend/docker/lint/hadolint/register.py b/src/python/pants/backend/experimental/docker/lint/hadolint/register.py similarity index 100% rename from src/python/pants/backend/docker/lint/hadolint/register.py rename to src/python/pants/backend/experimental/docker/lint/hadolint/register.py diff --git a/src/python/pants/init/BUILD b/src/python/pants/init/BUILD index b4dbaf68387..cf4b8f792be 100644 --- a/src/python/pants/init/BUILD +++ b/src/python/pants/init/BUILD @@ -8,8 +8,8 @@ target( dependencies=[ 'src/python/pants/backend/awslambda/python', 'src/python/pants/backend/codegen/protobuf/python', - 'src/python/pants/backend/docker/lint/hadolint', 'src/python/pants/backend/experimental/docker', + 'src/python/pants/backend/experimental/docker/lint/hadolint', 'src/python/pants/backend/experimental/go', 'src/python/pants/backend/experimental/go/lint/gofmt', 'src/python/pants/backend/experimental/python',