Docker lint

src/python/pants/backend/docker/lint/BUILD

@@ -0,0 +1,4 @@
+# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+python_library()

src/python/pants/backend/docker/lint/hadolint/BUILD

@@ -0,0 +1,5 @@
+# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+python_library()
+python_tests(name="tests")

src/python/pants/backend/docker/lint/hadolint/register.py

@@ -0,0 +1,12 @@
+# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from pants.backend.docker.lint.hadolint import skip_field
+from pants.backend.docker.lint.hadolint.rules import rules as hadolint_rules
+
+
+def rules():
+    return (
+        *hadolint_rules(),
+        *skip_field.rules(),
+    )

src/python/pants/backend/docker/lint/hadolint/rules.py

@@ -0,0 +1,126 @@
+# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import PurePath
+
+from pants.backend.docker.lint.hadolint.skip_field import SkipHadolintField
+from pants.backend.docker.lint.hadolint.subsystem import Hadolint
+from pants.backend.docker.target_types import DockerImageSources
+from pants.core.goals.lint import LintRequest, LintResult, LintResults
+from pants.core.util_rules.config_files import ConfigFiles, ConfigFilesRequest
+from pants.core.util_rules.external_tool import DownloadedExternalTool, ExternalToolRequest
+from pants.core.util_rules.source_files import SourceFiles, SourceFilesRequest
+from pants.engine.fs import Digest, MergeDigests
+from pants.engine.platform import Platform
+from pants.engine.process import FallibleProcessResult, Process
+from pants.engine.rules import Get, MultiGet, collect_rules, rule
+from pants.engine.target import FieldSet, Target
+from pants.engine.unions import UnionRule
+from pants.util.logging import LogLevel
+from pants.util.strutil import pluralize
+
+
+@dataclass(frozen=True)
+class HadolintFieldSet(FieldSet):
+    required_fields = (DockerImageSources,)
+
+    sources: DockerImageSources
+
+    @classmethod
+    def opt_out(cls, tgt: Target) -> bool:
+        return tgt.get(SkipHadolintField).value
+
+
+class HadolintRequest(LintRequest):
+    field_set_type = HadolintFieldSet
+
+
+@rule(desc="Lint with Hadolint", level=LogLevel.DEBUG)
+async def run_hadolint(request: HadolintRequest, hadolint: Hadolint) -> LintResults:
+    if hadolint.skip:
+        return LintResults([], linter_name="Hadolint")
+
+    downloaded_hadolint, sources = await MultiGet(
+        Get(DownloadedExternalTool, ExternalToolRequest, hadolint.get_request(Platform.current)),
+        Get(
+            SourceFiles,
+            SourceFilesRequest(
+                [field_set.sources for field_set in request.field_sets],
+                for_sources_types=(DockerImageSources,),
+                enable_codegen=True,
+            ),
+        ),
+    )
+    config_files = await Get(
+        ConfigFiles, ConfigFilesRequest, hadolint.config_request(sources.snapshot.dirs)
+    )
+    input_digest = await Get(
+        Digest,
+        MergeDigests(
+            (
+                sources.snapshot.digest,
+                downloaded_hadolint.digest,
+                config_files.snapshot.digest,
+            )
+        ),
+    )
+
+    # As hadolint uses a single config file, we need to partition our runs per config file
+    # discovered.
+    files_with_config = _group_files_with_config(
+        sources.snapshot.files,
+        config_files.snapshot.files,
+        not hadolint.config,
+    )
+    processes = [
+        Process(
+            argv=[downloaded_hadolint.exe, *config, *hadolint.args, *files],
+            input_digest=input_digest,
+            description=f"Run `hadolint` on {pluralize(len(files), 'Dockerfile')}.",
+            level=LogLevel.DEBUG,
+        )
+        for files, config in files_with_config
+    ]
+    process_results = await MultiGet(Get(FallibleProcessResult, Process, p) for p in processes)
+    results = [
+        LintResult.from_fallible_process_result(process_result)
+        for process_result in process_results
+    ]
+    return LintResults(results, linter_name="hadolint")
+
+
+def _group_files_with_config(
+    source_files: tuple[str, ...], config_files: tuple[str, ...], config_files_discovered: bool
+) -> list[tuple[tuple[str, ...], list[str]]]:
+    """If config_files_discovered, group all source files that is in the same directory or below a
+    config file, otherwise, all files will be kept in one group per config file that was provided as
+    option."""
+    groups = []
+    consumed_files: set[str] = set()
+    source_paths = {PurePath(source_file) for source_file in source_files}
+
+    for config_file in config_files:
+        if not config_files_discovered:
+            files = set(source_files)
+        else:
+            config_path = PurePath(config_file).parent
+            files = {str(source) for source in source_paths if config_path in source.parents}
+        if files:
+            # Sort files, to make the order predictable for the tests.
+            groups.append((tuple(sorted(files)), ["--config", config_file]))
+            consumed_files.update(files)
+
+    if len(consumed_files) < len(source_files):
+        files = set(source_files) - consumed_files
+        groups.append((tuple(files), []))
+
+    return groups
+
+
+def rules():
+    return [
+        *collect_rules(),
+        UnionRule(LintRequest, HadolintRequest),
+    ]

src/python/pants/backend/docker/lint/hadolint/rules_integration_test.py

@@ -0,0 +1,150 @@
+# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import annotations
+
+from textwrap import dedent
+
+import pytest
+
+from pants.backend.docker.lint.hadolint.rules import HadolintFieldSet, HadolintRequest
+from pants.backend.docker.lint.hadolint.rules import rules as hadolint_rules
+from pants.backend.docker.target_types import DockerImage
+from pants.core.goals.lint import LintResult, LintResults
+from pants.core.util_rules import config_files, external_tool, source_files
+from pants.engine.addresses import Address
+from pants.engine.target import Target
+from pants.testutil.rule_runner import QueryRule, RuleRunner
+
+
+@pytest.fixture
+def rule_runner() -> RuleRunner:
+    return RuleRunner(
+        rules=[
+            *hadolint_rules(),
+            *config_files.rules(),
+            *external_tool.rules(),
+            *source_files.rules(),
+            QueryRule(LintResults, [HadolintRequest]),
+        ],
+        target_types=[DockerImage],
+    )
+
+
+GOOD_FILE = dedent(
+    """
+    FROM python:3.8
+    """
+)
+
+BAD_FILE = dedent(
+    """
+    FROM python
+    """
+)
+
+
+def run_hadolint(
+    rule_runner: RuleRunner, targets: list[Target], *, extra_args: list[str] | None = None
+) -> tuple[LintResult, ...]:
+    rule_runner.set_options(
+        ["--backend-packages=pants.backend.docker.lint.hadolint", *(extra_args or ())],
+        env_inherit={"PATH"},
+    )
+    results = rule_runner.request(
+        LintResults,
+        [HadolintRequest(HadolintFieldSet.create(tgt) for tgt in targets)],
+    )
+    return results.results
+
+
+def assert_success(
+    rule_runner: RuleRunner, target: Target, *, extra_args: list[str] | None = None
+) -> None:
+    result = run_hadolint(rule_runner, [target], extra_args=extra_args)
+    assert len(result) == 1
+    assert result[0].exit_code == 0
+    assert not result[0].stdout
+    assert not result[0].stderr
+
+
+def test_passing(rule_runner: RuleRunner) -> None:
+    rule_runner.write_files({"Dockerfile": GOOD_FILE, "BUILD": "docker_image(name='t')"})
+    tgt = rule_runner.get_target(Address("", target_name="t"))
+    assert_success(rule_runner, tgt)
+
+
+def test_failing(rule_runner: RuleRunner) -> None:
+    rule_runner.write_files({"Dockerfile": BAD_FILE, "BUILD": "docker_image(name='t')"})
+    tgt = rule_runner.get_target(Address("", target_name="t"))
+    result = run_hadolint(rule_runner, [tgt])
+    assert len(result) == 1
+    assert result[0].exit_code == 1
+    assert "Dockerfile:2 " in result[0].stdout
+
+
+def test_multiple_targets(rule_runner: RuleRunner) -> None:
+    rule_runner.write_files(
+        {
+            "Dockerfile.good": GOOD_FILE,
+            "Dockerfile.bad": BAD_FILE,
+            "BUILD": dedent(
+                """
+                docker_image(name="good", sources=("Dockerfile.good",))
+                docker_image(name="bad", sources=("Dockerfile.bad",))
+                """
+            ),
+        }
+    )
+    tgts = [
+        rule_runner.get_target(
+            Address("", target_name="good", relative_file_path="Dockerfile.good")
+        ),
+        rule_runner.get_target(Address("", target_name="bad", relative_file_path="Dockerfile.bad")),
+    ]
+    result = run_hadolint(rule_runner, tgts)
+    assert len(result) == 1
+    assert result[0].exit_code == 1
+    assert "Dockerfile.good" not in result[0].stdout
+    assert "Dockerfile.bad:2 " in result[0].stdout
+
+
+def test_config_files(rule_runner: RuleRunner) -> None:
+    rule_runner.write_files(
+        {
+            "a/Dockerfile": BAD_FILE,
+            "a/BUILD": "docker_image()",
+            "a/.hadolint.yaml": "ignored: [DL3006]",
+            "b/Dockerfile": BAD_FILE,
+            "b/BUILD": "docker_image()",
+        }
+    )
+    tgts = [
+        rule_runner.get_target(Address("a")),
+        rule_runner.get_target(Address("b")),
+    ]
+    result = run_hadolint(rule_runner, tgts)
+    # We get two runs of hadolint, for the `a` and `b` directories respectively.
+    assert len(result) == 2
+    assert result[0].exit_code == 0
+    assert "a/Dockerfile" not in result[0].stdout
+    assert "b/Dockerfile:2 " not in result[0].stdout
+    assert result[1].exit_code == 1
+    assert "a/Dockerfile" not in result[1].stdout
+    assert "b/Dockerfile:2 " in result[1].stdout
+
+    tgt = rule_runner.get_target(Address("b"))
+    assert_success(rule_runner, tgt, extra_args=["--hadolint-config=a/.hadolint.yaml"])
+
+
+def test_passthrough_args(rule_runner: RuleRunner) -> None:
+    rule_runner.write_files({"Dockerfile": BAD_FILE, "BUILD": "docker_image(name='t')"})
+    tgt = rule_runner.get_target(Address("", target_name="t"))
+    assert_success(rule_runner, tgt, extra_args=["--hadolint-args=--ignore DL3006"])
+
+
+def test_skip(rule_runner: RuleRunner) -> None:
+    rule_runner.write_files({"Dockerfile": BAD_FILE, "BUILD": "docker_image(name='t')"})
+    tgt = rule_runner.get_target(Address("", target_name="t"))
+    result = run_hadolint(rule_runner, [tgt], extra_args=["--hadolint-skip"])
+    assert not result

src/python/pants/backend/docker/lint/hadolint/rules_test.py

@@ -0,0 +1,64 @@
+# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+import pytest
+
+from pants.backend.docker.lint.hadolint.rules import _group_files_with_config
+
+
+@pytest.mark.parametrize(
+    "source_files, config_files, config_files_discovered, expected",
+    [
+        (
+            {"a/Dockerfile", "b/Dockerfile"},
+            ("a/.hadolint.yaml",),
+            True,
+            [
+                (("a/Dockerfile",), ["--config", "a/.hadolint.yaml"]),
+                (("b/Dockerfile",), []),
+            ],
+        ),
+        (
+            {"a/Dockerfile", "b/Dockerfile"},
+            ("hadolint.yaml",),
+            False,
+            [
+                (("a/Dockerfile", "b/Dockerfile"), ["--config", "hadolint.yaml"]),
+            ],
+        ),
+        (
+            {"a/Dockerfile", "aa/Dockerfile"},
+            ("a/.hadolint.yaml",),
+            True,
+            [
+                (("a/Dockerfile",), ["--config", "a/.hadolint.yaml"]),
+                (("aa/Dockerfile",), []),
+            ],
+        ),
+        (
+            {"a/Dockerfile", "b/Dockerfile", "c/Dockerfile", "d/Dockerfile", "c/e/Dockerfile"},
+            ("a/.hadolint.yaml", "c/.hadolint.yaml"),
+            True,
+            [
+                (("a/Dockerfile",), ["--config", "a/.hadolint.yaml"]),
+                (
+                    (
+                        "c/Dockerfile",
+                        "c/e/Dockerfile",
+                    ),
+                    ["--config", "c/.hadolint.yaml"],
+                ),
+                (
+                    (
+                        "b/Dockerfile",
+                        "d/Dockerfile",
+                    ),
+                    [],
+                ),
+            ],
+        ),
+    ],
+)
+def test_group_files_with_config(source_files, config_files, config_files_discovered, expected):
+    actual = _group_files_with_config(source_files, config_files, config_files_discovered)
+    assert actual == expected

src/python/pants/backend/docker/lint/hadolint/skip_field.py

@@ -0,0 +1,15 @@
+# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from pants.backend.docker.target_types import DockerImage
+from pants.engine.target import BoolField
+
+
+class SkipHadolintField(BoolField):
+    alias = "skip_hadolint"
+    default = False
+    help = "If true, don't run hadolint on this target's Dockerfile."
+
+
+def rules():
+    return [DockerImage.register_plugin_field(SkipHadolintField)]