Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: add support for pyproject.toml and env.yml/env.yaml files #550

Open
wants to merge 5 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
[build-system]
requires = ["setuptools>=42"]
requires = ["setuptools>=42", "insecure-package"

dylanpulver marked this conversation as resolved.
Show resolved Hide resolved
]
build-backend = "setuptools.build_meta"

3 changes: 2 additions & 1 deletion safety/scan/command.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,6 @@ class ScannableEcosystems(Enum):
"""Enum representing scannable ecosystems."""
PYTHON = Ecosystem.PYTHON.value


def process_report(
obj: Any, console: Console, report: ReportModel, output: str,
save_as: Optional[Tuple[str, Path]], **kwargs
Expand Down Expand Up @@ -314,6 +313,8 @@ def scan(ctx: typer.Context,
with console.status(wait_msg, spinner=DEFAULT_SPINNER) as status:
for path, analyzed_file in process_files(paths=file_paths,
config=config):
print("now here", analyzed_file.dependency_results.dependencies)
print("now here", analyzed_file.file_type)
dylanpulver marked this conversation as resolved.
Show resolved Hide resolved
count += len(analyzed_file.dependency_results.dependencies)

# Update exit code if vulnerabilities are found
Expand Down
41 changes: 40 additions & 1 deletion safety/scan/ecosystems/python/dependencies.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
from pathlib import Path
import sys
from typing import Generator, List, Optional

import toml
from safety_schemas.models import FileType, PythonDependency
from safety_schemas.models.package import PythonSpecification
from ..base import InspectableFile
Expand Down Expand Up @@ -270,6 +270,42 @@ def read_virtual_environment_dependencies(f: InspectableFile) -> Generator[Pytho
latest_version_without_known_vulnerabilities=None,
more_info_url=None)

def read_pyproject_toml_dependencies(file: Path) -> Generator[PythonDependency, None, None]:
with open(file, 'r') as f:
data = toml.load(f)
dependencies = []

# Handle 'build-system.requires'
if 'build-system' in data and 'requires' in data['build-system']:
dependencies.extend(data['build-system']['requires'])

# Handle 'project.dependencies'
if 'project' in data and 'dependencies' in data['project']:
dependencies.extend(data['project']['dependencies'])

# Handle 'tool.poetry.dependencies'
if 'tool' in data and 'poetry' in data['tool'] and 'dependencies' in data['tool']['poetry']:
for dep, version in data['tool']['poetry']['dependencies'].items():
if isinstance(version, str):
dependencies.append(f"{dep}=={version}")
else:
dependencies.append(dep)

for dep in dependencies:
dep_name, dep_version = (dep.split("==") + [None])[:2]
yield PythonDependency(
name=dep_name,
version=dep_version,
specifications=[
PythonSpecification(f"{dep_name}=={dep_version}" if dep_version else dep_name, found=file)
],
found=file,
insecure_versions=[],
secure_versions=[],
latest_version=None,
latest_version_without_known_vulnerabilities=None,
more_info_url=None
)

def get_dependencies(f: InspectableFile) -> List[PythonDependency]:
"""
Expand All @@ -291,4 +327,7 @@ def get_dependencies(f: InspectableFile) -> List[PythonDependency]:
if f.file_type == FileType.VIRTUAL_ENVIRONMENT:
return list(read_virtual_environment_dependencies(f))

if f.file_type == FileType.PYPROJECT_TOML:
return list(read_pyproject_toml_dependencies(f.file))

return []
12 changes: 11 additions & 1 deletion safety/scan/finder/file_finder.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@

from safety.errors import SafetyException

from .handlers import FileHandler, ECOSYSTEM_HANDLER_MAPPING
from .handlers import FileHandler, ECOSYSTEM_HANDLER_MAPPING, PyProjectTomlHandler

LOG = logging.getLogger(__name__)

Expand Down Expand Up @@ -75,6 +75,7 @@ def __init__(
self.target = target
self.include_files = include_files

print("ecosystems", ecosystems)
dylanpulver marked this conversation as resolved.
Show resolved Hide resolved
# If no handlers are provided, initialize them from the ecosystem mapping
if not handlers:
handlers = set(ECOSYSTEM_HANDLER_MAPPING[ecosystem]()
Expand Down Expand Up @@ -149,8 +150,17 @@ def process_directory(self, dir_path: str, max_deep: Optional[int] = None) -> Tu
files[file_type.value] = set()
files[file_type.value].add(inspectable_file)
break

special_files = {'pyproject.toml', 'env.yml', 'env.yaml'}
if file_name in special_files:
file_type = FileType(file_name)
inspectable_file = Path(root, file_name)
if file_type.value not in files or not files[file_type.value]:
files[file_type.value] = set()
files[file_type.value].add(inspectable_file)
level += 1


return dir_path, files

def search(self) -> Tuple[str, Dict[str, Set[Path]]]:
Expand Down
62 changes: 55 additions & 7 deletions safety/scan/finder/handlers.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@
import os
from pathlib import Path
from types import MappingProxyType
from typing import Dict, List, Optional, Optional, Tuple

from typing import Dict, List, Optional, Set
import toml
from safety_schemas.models import Ecosystem, FileType


Expand Down Expand Up @@ -52,7 +52,7 @@ def can_handle(self, root: str, file_name: str, include_files: Dict[FileType, Li
return None

@abstractmethod
def download_required_assets(self, session) -> Dict[str, str]:
def download_required_assets(self, session):
"""
Abstract method to download required assets for handling files. Should be implemented
by subclasses.
Expand Down Expand Up @@ -109,14 +109,62 @@ def __init__(self) -> None:
super().__init__()
self.ecosystem = Ecosystem.SAFETY_PROJECT

def download_required_assets(self, session) -> None:
"""
No required assets to download for Safety project files.
"""
def download_required_assets(self, session):
pass


class PyProjectTomlHandler(FileHandler):
def __init__(self) -> None:
super().__init__()
self.ecosystem = Ecosystem.PYTHON

def download_required_assets(self, session):
from safety.safety import fetch_database

SAFETY_DB_DIR = os.getenv("SAFETY_DB_DIR")

db = False if SAFETY_DB_DIR is None else SAFETY_DB_DIR


fetch_database(session=session, full=False, db=db, cached=True,
telemetry=True, ecosystem=Ecosystem.PYTHON,
from_cache=False)

fetch_database(session=session, full=True, db=db, cached=True,
telemetry=True, ecosystem=Ecosystem.PYTHON,
from_cache=False)

def can_handle(self, root: str, file_name: str, include_files: Dict[FileType, List[Path]]) -> Optional[FileType]:
if file_name == 'pyproject.toml':
print("recognized")
return FileType.PYPROJECT_TOML
return None

def handle(self, file_path: Path) -> Set[str]:
with open(file_path, 'r') as file:
data = toml.load(file)
print("printing data", data)
dependencies = set()

# Handle 'build-system.requires'
if 'build-system' in data and 'requires' in data['build-system']:
dependencies.update(data['build-system']['requires'])

# Handle 'project.dependencies'
if 'project' in data and 'dependencies' in data['project']:
dependencies.update(data['project']['dependencies'])

# Handle 'tool.poetry.dependencies'
if 'tool' in data and 'poetry' in data['tool'] and 'dependencies' in data['tool']['poetry']:
for dep, version in data['tool']['poetry']['dependencies'].items():
dependencies.add(f"{dep}=={version}" if isinstance(version, str) else dep)

return dependencies


# Mapping of ecosystems to their corresponding file handlers
ECOSYSTEM_HANDLER_MAPPING = MappingProxyType({
Ecosystem.PYTHON: PythonFileHandler,
Ecosystem.SAFETY_PROJECT: SafetyProjectFileHandler,
# Ecosystem.PYPROJECT_TOML: PyProjectTomlHandler,
})
Loading