Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add source linting for recipe v2 #2028

Open
wants to merge 8 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 12 additions & 6 deletions conda_smithy/lint_recipe.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
from inspect import cleandoc
from pathlib import Path
from textwrap import indent
from typing import List, Tuple
from typing import Any, List, Tuple

import github
import jsonschema
Expand Down Expand Up @@ -76,7 +76,7 @@
NEEDED_FAMILIES = ["gpl", "bsd", "mit", "apache", "psf"]


def lintify_forge_yaml(recipe_dir=None) -> (list, list):
def lintify_forge_yaml(recipe_dir: str | None = None) -> (list, list):
if recipe_dir:
forge_yaml_filename = (
glob(os.path.join(recipe_dir, "..", "conda-forge.yml"))
Expand All @@ -100,9 +100,9 @@ def lintify_forge_yaml(recipe_dir=None) -> (list, list):


def lintify_meta_yaml(
meta,
recipe_dir=None,
conda_forge=False,
meta: Any,
recipe_dir: str = None,
conda_forge: bool = False,
recipe_version: int = 1,
) -> Tuple[List[str], List[str]]:
lints = []
Expand Down Expand Up @@ -186,7 +186,9 @@ def lintify_meta_yaml(
lint_build_section_should_be_before_run(requirements_section, lints)

# 9: Files downloaded should have a hash.
lint_sources_should_have_hash(sources_section, lints)
lint_sources_should_have_hash(
sources_section, lints, recipe_version=recipe_version
)

# 10: License should not include the word 'license'.
lint_license_should_not_have_license(about_section, lints)
Expand Down Expand Up @@ -632,6 +634,9 @@ def main(
forge_config = _read_forge_config(feedstock_dir)
if forge_config.get("conda_build_tool", "") == RATTLER_BUILD_TOOL:
build_tool = RATTLER_BUILD_TOOL
else:
if os.path.exists(os.path.join(recipe_dir, "recipe.yaml")):
build_tool = RATTLER_BUILD_TOOL

if build_tool == RATTLER_BUILD_TOOL:
recipe_file = os.path.join(recipe_dir, "recipe.yaml")
Expand All @@ -651,6 +656,7 @@ def main(
meta = get_yaml().load(Path(recipe_file))

recipe_version = 2 if build_tool == RATTLER_BUILD_TOOL else 1

results, hints = lintify_meta_yaml(
meta,
recipe_dir,
Expand Down
53 changes: 28 additions & 25 deletions conda_smithy/linter/conda_recipe_v2_linter.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,32 +50,35 @@ def lint_recipe_tests(
tests_lints = []
tests_hints = []

if not any(key in TEST_KEYS for key in test_section):
a_test_file_exists = recipe_dir is not None and any(
os.path.exists(os.path.join(recipe_dir, test_file))
for test_file in TEST_FILES
)
if a_test_file_exists:
return

if not outputs_section:
lints.append("The recipe must have some tests.")
else:
has_outputs_test = False
no_test_hints = []
for section in outputs_section:
test_section = section.get("tests", {})
if any(key in TEST_KEYS for key in test_section):
has_outputs_test = True
else:
no_test_hints.append(
"It looks like the '{}' output doesn't "
"have any tests.".format(section.get("name", "???"))
)
if has_outputs_test:
hints.extend(no_test_hints)
else:
for test in test_section:
if not any(key in TEST_KEYS for key in test.keys()):
a_test_file_exists = recipe_dir is not None and any(
os.path.exists(os.path.join(recipe_dir, test_file))
for test_file in TEST_FILES
)
if a_test_file_exists:
return

if not outputs_section:
lints.append("The recipe must have some tests.")
else:
has_outputs_test = False
no_test_hints = []
for section in outputs_section:
test_section = section.get("tests", {})
if any(key in TEST_KEYS for key in test_section):
has_outputs_test = True
else:
no_test_hints.append(
"It looks like the '{}' output doesn't "
"have any tests.".format(
section.get("name", "???")
)
)
if has_outputs_test:
hints.extend(no_test_hints)
else:
lints.append("The recipe must have some tests.")

lints.extend(tests_lints)
hints.extend(tests_hints)
Expand Down
20 changes: 19 additions & 1 deletion conda_smithy/linter/lints.py
Original file line number Diff line number Diff line change
Expand Up @@ -197,7 +197,11 @@ def lint_build_section_should_be_before_run(requirements_section, lints):
)


def lint_sources_should_have_hash(sources_section, lints):
def lint_sources_should_have_hash(
sources_section: list[dict[str, Any]],
lints: list[str],
recipe_version: int = 1,
):
for source_section in sources_section:
if "url" in source_section and not (
{"sha1", "sha256", "md5"} & set(source_section.keys())
Expand All @@ -207,6 +211,20 @@ def lint_sources_should_have_hash(sources_section, lints):
"or md5 checksum (sha256 preferably)."
)

# make sure there is no templating involved for the hash and they look alright
if recipe_version == 2:
for source in sources_section:
if source.get("sha256"):
if not re.match(r"[0-9a-f]{64}", source["sha256"]):
lints.append(
"sha256 checksum must be 64 characters long. Templates are not allowed in the sha256 checksum."
)
if source.get("md5"):
if not re.match(r"[0-9a-f]{32}", source["md5"]):
lints.append(
"md5 checksum must be 32 characters long. Templates are not allowed in the md5 checksum."
)


def lint_license_should_not_have_license(about_section, lints):
license = about_section.get("license", "").lower()
Expand Down
4 changes: 2 additions & 2 deletions conda_smithy/linter/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
FIELDS as _CONDA_BUILD_FIELDS,
)
from rattler_build_conda_compat import loader as rattler_loader
from rattler_build_conda_compat.recipe_sources import get_all_url_sources
from rattler_build_conda_compat.recipe_sources import get_all_sources

FIELDS = copy.deepcopy(_CONDA_BUILD_FIELDS)

Expand Down Expand Up @@ -78,7 +78,7 @@ def get_recipe_v2_section(meta, name) -> Union[Dict, List[Dict]]:
elif name == "tests":
return rattler_loader.load_all_tests(meta)
elif name == "source":
sources = get_all_url_sources(meta)
sources = get_all_sources(meta)
return list(sources)

return meta.get(name, {})
Expand Down
35 changes: 0 additions & 35 deletions tests/recipes/rattler_recipes/boltons_arch/recipe.yaml

This file was deleted.

17 changes: 0 additions & 17 deletions tests/recipes/rattler_recipes/boltons_arch/variants.yaml

This file was deleted.

This file was deleted.

This file was deleted.

This file was deleted.

48 changes: 48 additions & 0 deletions tests/test_lint_recipe.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,25 @@ def is_gh_token_set():
return "GH_TOKEN" in os.environ


@contextmanager
def get_recipe_in_dir(recipe_name: str) -> Path:
base_dir = Path(__file__).parent
recipe_path = base_dir / "recipes" / recipe_name
assert recipe_path.exists(), f"Recipe {recipe_name} does not exist"

# create a temporary directory to copy the recipe into
with tmp_directory() as tmp_dir:
# copy the file into the temporary directory
recipe_folder = Path(tmp_dir) / "recipe"
recipe_folder.mkdir()
shutil.copy(recipe_path, recipe_folder / "recipe.yaml")

try:
yield recipe_folder
finally:
pass


@contextmanager
def tmp_directory():
tmp_dir = tempfile.mkdtemp("recipe_")
Expand Down Expand Up @@ -2656,5 +2675,34 @@ def test_pin_compatible_in_run_exports_output(recipe_version: int):
assert any(lint.startswith(expected) for lint in lints)


def test_v1_recipes():
with get_recipe_in_dir("v1_recipes/recipe-no-lint.yaml") as recipe_dir:
lints, hints = linter.main(str(recipe_dir), return_hints=True)
assert not lints


def test_source_section_v1_template():
with get_recipe_in_dir(
"v1_recipes/recipe-sha-template.yaml"
) as recipe_dir:
lints, hints = linter.main(str(recipe_dir), return_hints=True)
assert any(
"sha256 checksum must be 64 characters long. Templates are not allowed in the sha256 checksum."
in lint
for lint in lints
)


def test_v1_package_name_version():
with get_recipe_in_dir(
"v1_recipes/recipe-lint-name-version.yaml"
) as recipe_dir:
lints, hints = linter.main(str(recipe_dir), return_hints=True)
lint_1 = "Recipe name has invalid characters. only lowercase alpha, numeric, underscores, hyphens and dots allowed"
lint_2 = "Package version $!@# doesn't match conda spec: Invalid version '$!@#': invalid character(s)"
assert lint_1 in lints
assert lint_2 in lints


if __name__ == "__main__":
unittest.main()
Loading