From c7ce1279946edce242371cebca1bbe5f0561c300 Mon Sep 17 00:00:00 2001 From: Antti Kaihola <13725+akaihola@users.noreply.github.com> Date: Thu, 16 Feb 2023 17:55:12 +0200 Subject: [PATCH 01/30] Move code from `command_line.py` to Darkgraylib Move parts which are also needed by Graylint. --- src/darker/command_line.py | 101 ++----------------------------------- 1 file changed, 4 insertions(+), 97 deletions(-) diff --git a/src/darker/command_line.py b/src/darker/command_line.py index 32ca97b04..9267582b3 100644 --- a/src/darker/command_line.py +++ b/src/darker/command_line.py @@ -1,26 +1,12 @@ """Command line parsing for the ``darker`` binary""" -from argparse import SUPPRESS, ArgumentParser, Namespace -from typing import Any, List, Optional, Tuple +from argparse import ArgumentParser +from typing import Any, Optional +import darkgraylib.command_line from black import TargetVersion from darker import help as hlp -from darker.argparse_helpers import ( - LogLevelAction, - NewlinePreservingFormatter, - OptionsForReadmeAction, -) -from darker.config import ( - DarkerConfig, - OutputMode, - get_effective_config, - get_modified_config, - load_config, - override_color_with_environment, - validate_stdin_src, -) -from darker.version import __version__ def make_argument_parser(require_src: bool) -> ArgumentParser: @@ -30,37 +16,18 @@ def make_argument_parser(require_src: bool) -> ArgumentParser: on the command line. ``False`` to not require on. """ - parser = ArgumentParser( - description=hlp.DESCRIPTION, formatter_class=NewlinePreservingFormatter - ) - parser.register("action", "log_level", LogLevelAction) + parser = darkgraylib.command_line.make_argument_parser(require_src, hlp.DESCRIPTION) def add_arg(help_text: Optional[str], *name_or_flags: str, **kwargs: Any) -> None: kwargs["help"] = help_text parser.add_argument(*name_or_flags, **kwargs) - add_arg(hlp.SRC, "src", nargs="+" if require_src else "*", metavar="PATH") - add_arg(hlp.REVISION, "-r", "--revision", default="HEAD", metavar="REV") add_arg(hlp.DIFF, "--diff", action="store_true") add_arg(hlp.STDOUT, "-d", "--stdout", action="store_true") - add_arg(hlp.STDIN_FILENAME, "--stdin-filename", metavar="PATH") add_arg(hlp.CHECK, "--check", action="store_true") add_arg(hlp.FLYNT, "-f", "--flynt", action="store_true") add_arg(hlp.ISORT, "-i", "--isort", action="store_true") add_arg(hlp.LINT, "-L", "--lint", action="append", metavar="CMD", default=[]) - add_arg(hlp.CONFIG, "-c", "--config", metavar="PATH") - add_arg( - hlp.VERBOSE, - "-v", - "--verbose", - action="log_level", - dest="log_level", - const=-10, - ) - add_arg(hlp.QUIET, "-q", "--quiet", action="log_level", dest="log_level", const=10) - add_arg(hlp.COLOR, "--color", action="store_const", dest="color", const=True) - add_arg(hlp.NO_COLOR, "--no-color", action="store_const", dest="color", const=False) - add_arg(hlp.VERSION, "--version", action="version", version=__version__) add_arg( hlp.SKIP_STRING_NORMALIZATION, "-S", @@ -99,64 +66,4 @@ def add_arg(help_text: Optional[str], *name_or_flags: str, **kwargs: Any) -> Non metavar="VERSION", choices=[v.name.lower() for v in TargetVersion], ) - add_arg(hlp.WORKERS, "-W", "--workers", type=int, dest="workers", default=1) - # A hidden option for printing command lines option in a format suitable for - # `README.rst`: - add_arg(SUPPRESS, "--options-for-readme", action=OptionsForReadmeAction) return parser - - -def parse_command_line(argv: List[str]) -> Tuple[Namespace, DarkerConfig, DarkerConfig]: - """Return the parsed command line, using defaults from a configuration file - - Also return the effective configuration which combines defaults, the configuration - read from ``pyproject.toml`` (or the path given in ``--config``), environment - variables, and command line arguments. - - Finally, also return the set of configuration options which differ from defaults. - - """ - # 1. Parse the paths of files/directories to process into `args.src`, and the config - # file path into `args.config`. - parser_for_srcs = make_argument_parser(require_src=False) - args = parser_for_srcs.parse_args(argv) - - # 2. Locate `pyproject.toml` based on the `-c`/`--config` command line option, or - # if it's not provided, based on the paths to process, or in the current - # directory if no paths were given. Load Darker configuration from it. - pyproject_config = load_config(args.config, args.src) - - # 3. The PY_COLORS, NO_COLOR and FORCE_COLOR environment variables override the - # `--color` command line option. - config = override_color_with_environment(pyproject_config) - - # 4. Re-run the parser with configuration defaults. This way we get combined values - # based on the configuration file and the command line options for all options - # except `src` (the list of files to process). - parser_for_srcs.set_defaults(**config) - args = parser_for_srcs.parse_args(argv) - - # 5. Make sure an error for missing file/directory paths is thrown if we're not - # running in stdin mode and no file/directory is configured in `pyproject.toml`. - if args.stdin_filename is None and not config.get("src"): - parser = make_argument_parser(require_src=True) - parser.set_defaults(**config) - args = parser.parse_args(argv) - - # 6. Make sure there aren't invalid option combinations after merging configuration - # and command line options. - OutputMode.validate_diff_stdout(args.diff, args.stdout) - OutputMode.validate_stdout_src(args.stdout, args.src, args.stdin_filename) - validate_stdin_src(args.stdin_filename, args.src) - - # 7. Also create a parser which uses the original default configuration values. - # This is used to find out differences between the effective configuration and - # default configuration values, and print them out in verbose mode. - parser_with_original_defaults = make_argument_parser( - require_src=args.stdin_filename is None - ) - return ( - args, - get_effective_config(args), - get_modified_config(parser_with_original_defaults, args), - ) From 6e0dbc51118d3a9f48e0ab4db5145a50ff2694c3 Mon Sep 17 00:00:00 2001 From: Antti Kaihola <13725+akaihola@users.noreply.github.com> Date: Thu, 16 Feb 2023 17:55:16 +0200 Subject: [PATCH 02/30] Move code from `__main__.py` to Darkgraylib Move parts which are also needed by Graylint. --- src/darker/__main__.py | 45 +++++++++++++++++++----------------------- 1 file changed, 20 insertions(+), 25 deletions(-) diff --git a/src/darker/__main__.py b/src/darker/__main__.py index fd7852b19..64377775e 100644 --- a/src/darker/__main__.py +++ b/src/darker/__main__.py @@ -18,9 +18,9 @@ run_black, ) from darker.chooser import choose_lines -from darker.command_line import parse_command_line +from darker.command_line import make_argument_parser from darker.concurrency import get_executor -from darker.config import Exclusions, OutputMode, dump_config +from darker.config import Exclusions, OutputMode, dump_config, validate_config_output_mode from darker.diff import diff_chunks from darker.exceptions import DependencyError, MissingPackageError from darker.fstring import apply_flynt, flynt @@ -39,7 +39,6 @@ from darker.help import get_extra_instruction from darker.highlighting import colorize, should_use_color from darker.import_sorting import apply_isort, isort -from darker.linting import run_linters from darker.utils import ( GIT_DATEFORMAT, DiffChunk, @@ -49,6 +48,11 @@ glob_any, ) from darker.verification import ASTVerifier, BinarySearch, NotEquivalentError +from darkgraylib.command_line import parse_command_line +from darkgraylib.config import show_config_if_debug +from darkgraylib.log import setup_logging +from darkgraylib.main import resolve_paths +from graylint.linting import run_linters logger = logging.getLogger(__name__) @@ -480,23 +484,19 @@ def main( # pylint: disable=too-many-locals,too-many-branches,too-many-statemen should be) reformatted; 0 otherwise. """ - if argv is None: - argv = sys.argv[1:] - args, config, config_nondefault = parse_command_line(argv) - logging.basicConfig(level=args.log_level) - if args.log_level == logging.INFO: - formatter = logging.Formatter("%(levelname)s: %(message)s") - logging.getLogger().handlers[0].setFormatter(formatter) + args, config, config_nondefault = parse_command_line(make_argument_parser, argv) + # Make sure there aren't invalid option combinations after merging configuration and + # command line options. + OutputMode.validate_diff_stdout(args.diff, args.stdout) + OutputMode.validate_stdout_src(args.stdout, args.src, args.stdin_filename) + validate_config_output_mode(config) + + setup_logging(args.log_level) # Make sure we don't get excessive debug log output from Black logging.getLogger("blib2to3.pgen2.driver").setLevel(logging.WARNING) - if args.log_level <= logging.DEBUG: - print("\n# Effective configuration:\n") - print(dump_config(config)) - print("\n# Configuration options which differ from defaults:\n") - print(dump_config(config_nondefault)) - print("\n") + show_config_if_debug(config, config_nondefault, args.log_level) if args.isort and not isort: raise MissingPackageError( @@ -520,16 +520,11 @@ def main( # pylint: disable=too-many-locals,too-many-branches,too-many-statemen if args.skip_magic_trailing_comma is not None: black_config["skip_magic_trailing_comma"] = args.skip_magic_trailing_comma - stdin_mode = args.stdin_filename is not None - if stdin_mode: - paths = {Path(args.stdin_filename)} - # `parse_command_line` guarantees that `args.src` is empty - else: - paths = {Path(p) for p in args.src} - # `parse_command_line` guarantees that `args.stdin_filename` is `None` - root = get_common_root(paths) + paths, root = resolve_paths(args.stdin_filename, args.src) - revrange = RevisionRange.parse_with_common_ancestor(args.revision, root, stdin_mode) + revrange = RevisionRange.parse_with_common_ancestor( + args.revision, root, args.stdin_filename is not None + ) output_mode = OutputMode.from_args(args) write_modified_files = not args.check and output_mode == OutputMode.NOTHING if write_modified_files: From a4e8e4ac563fecaddf92523e812f633872e1cf5e Mon Sep 17 00:00:00 2001 From: Antti Kaihola <13725+akaihola@users.noreply.github.com> Date: Sun, 19 Feb 2023 18:27:07 +0200 Subject: [PATCH 03/30] Command line now mainly parsed in `darkgraylib` --- src/darker/command_line.py | 21 ++++++++++++++------- src/darker/help.py | 5 ----- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/src/darker/command_line.py b/src/darker/command_line.py index 9267582b3..709f84949 100644 --- a/src/darker/command_line.py +++ b/src/darker/command_line.py @@ -1,12 +1,14 @@ """Command line parsing for the ``darker`` binary""" from argparse import ArgumentParser -from typing import Any, Optional +from functools import partial -import darkgraylib.command_line from black import TargetVersion +import darkgraylib.command_line from darker import help as hlp +from darkgraylib.command_line import add_parser_argument +from graylint.command_line import add_lint_arg def make_argument_parser(require_src: bool) -> ArgumentParser: @@ -16,18 +18,23 @@ def make_argument_parser(require_src: bool) -> ArgumentParser: on the command line. ``False`` to not require on. """ - parser = darkgraylib.command_line.make_argument_parser(require_src, hlp.DESCRIPTION) + parser = darkgraylib.command_line.make_argument_parser( + require_src, + "Darker", + hlp.DESCRIPTION, + "Make `darker`, `black` and `isort` read configuration from `PATH`. Note that" + " other tools like `flynt`, `mypy`, `pylint` or `flake8` won't use this" + " configuration file.", + ) - def add_arg(help_text: Optional[str], *name_or_flags: str, **kwargs: Any) -> None: - kwargs["help"] = help_text - parser.add_argument(*name_or_flags, **kwargs) + add_arg = partial(add_parser_argument, parser) add_arg(hlp.DIFF, "--diff", action="store_true") add_arg(hlp.STDOUT, "-d", "--stdout", action="store_true") add_arg(hlp.CHECK, "--check", action="store_true") add_arg(hlp.FLYNT, "-f", "--flynt", action="store_true") add_arg(hlp.ISORT, "-i", "--isort", action="store_true") - add_arg(hlp.LINT, "-L", "--lint", action="append", metavar="CMD", default=[]) + add_lint_arg(parser) add_arg( hlp.SKIP_STRING_NORMALIZATION, "-S", diff --git a/src/darker/help.py b/src/darker/help.py index 7df4ee166..8772a65eb 100644 --- a/src/darker/help.py +++ b/src/darker/help.py @@ -108,11 +108,6 @@ def get_extra_instruction(dependency: str) -> str: " a terminal and or enabled by explicitly (see `--color`)." ) -CONFIG = ( - "Ask `black` and `isort` to read configuration from `PATH`. Note that other tools" - " like flynt, Mypy, Pylint and Flake8 won't use this configuration file." -) - VERBOSE = "Show steps taken and summarize modifications" QUIET = "Reduce amount of output" COLOR = ( From debcbe133fe3ce53829f62f8e9c7ba10939d1803 Mon Sep 17 00:00:00 2001 From: Antti Kaihola <13725+akaihola@users.noreply.github.com> Date: Thu, 23 Feb 2023 23:21:39 +0200 Subject: [PATCH 04/30] Move linting support implementation to `graylint` --- src/darker/linting.py | 560 ---------------------------- src/darker/tests/test_linting.py | 603 ------------------------------- src/darker/tests/test_main.py | 1 - 3 files changed, 1164 deletions(-) delete mode 100644 src/darker/linting.py delete mode 100644 src/darker/tests/test_linting.py diff --git a/src/darker/linting.py b/src/darker/linting.py deleted file mode 100644 index fcd514266..000000000 --- a/src/darker/linting.py +++ /dev/null @@ -1,560 +0,0 @@ -"""Run arbitrary linters on given files in a Git repository - -This supports any linter which reports on standard output and has a fairly standard -correct output format:: - - :: - ::: - -For example, Mypy outputs:: - - module.py:57: error: Function is missing a type annotation for one or more arguments - -Pylint, on the other hand:: - - module.py:44:8: R1720: Unnecessary "elif" after "raise" (no-else-raise) - -All such output from the linter will be printed on the standard output -provided that the ```` falls on a changed line. - -""" - -import logging -import os -import re -import shlex -from collections import defaultdict -from contextlib import contextmanager -from dataclasses import dataclass -from pathlib import Path -from subprocess import PIPE, Popen # nosec -from tempfile import TemporaryDirectory -from typing import ( - IO, - Callable, - Collection, - Dict, - Generator, - Iterable, - List, - Set, - Tuple, - Union, -) - -from darker.diff import map_unmodified_lines -from darker.git import ( - STDIN, - WORKTREE, - RevisionRange, - git_clone_local, - git_get_content_at_revision, - git_get_root, - git_rev_parse, -) -from darker.highlighting import colorize -from darker.utils import WINDOWS - -logger = logging.getLogger(__name__) - - -@dataclass(eq=True, frozen=True, order=True) -class MessageLocation: - """A file path, line number and column number for a linter message - - Line and column numbers a 0-based, and zero is used for an unspecified column, and - for the non-specified location. - - """ - - path: Path - line: int - column: int = 0 - - def __str__(self) -> str: - """Convert file path, line and column into a linter line prefix string - - :return: Either ``"path:line:column"`` or ``"path:line"`` (if column is zero) - - """ - if self.column: - return f"{self.path}:{self.line}:{self.column}" - return f"{self.path}:{self.line}" - - -NO_MESSAGE_LOCATION = MessageLocation(Path(""), 0, 0) - - -@dataclass -class LinterMessage: - """Information about a linter message""" - - linter: str - description: str - - -class DiffLineMapping: - """A mapping from unmodified lines in new and old versions of files""" - - def __init__(self) -> None: - self._mapping: Dict[Tuple[Path, int], Tuple[Path, int]] = {} - - def __setitem__( - self, new_location: MessageLocation, old_location: MessageLocation - ) -> None: - """Add a pointer from new to old line to the mapping - - :param new_location: The file path and linenum of the message in the new version - :param old_location: The file path and linenum of the message in the old version - - """ - self._mapping[new_location.path, new_location.line] = ( - old_location.path, - old_location.line, - ) - - def get(self, new_location: MessageLocation) -> MessageLocation: - """Get the old location of the message based on the mapping - - The mapping is between line numbers, so the column number of the message in the - new file is injected into the corresponding location in the old version of the - file. - - :param new_location: The path, line and column number of a linter message in the - new version of a file - :return: The path, line and column number of the same message in the old version - of the file - - """ - key = (new_location.path, new_location.line) - if key in self._mapping: - (old_path, old_line) = self._mapping[key] - return MessageLocation(old_path, old_line, new_location.column) - return NO_MESSAGE_LOCATION - - -def normalize_whitespace(message: LinterMessage) -> LinterMessage: - """Given a line of linter output, shortens runs of whitespace to a single space - - Also removes any leading or trailing whitespace. - - This is done to support comparison of different ``cov_to_lint.py`` runs. To make the - output more readable and compact, the tool adjusts whitespace. This is done to both - align runs of lines and to remove blocks of extra indentation. For differing sets of - coverage messages from ``pytest-cov`` runs of different versions of the code, these - whitespace adjustments can differ, so we need to normalize them to compare and match - them. - - :param message: The linter message to normalize - :return: The normalized linter message with leading and trailing whitespace stripped - and runs of whitespace characters collapsed into single spaces - - """ - return LinterMessage( - message.linter, re.sub(r"\s\s+", " ", message.description).strip() - ) - - -def make_linter_env(root: Path, revision: str) -> Dict[str, str]: - """Populate environment variables for running linters - - :param root: The path to the root of the Git repository - :param revision: The commit hash of the Git revision being linted, or ``"WORKTREE"`` - if the working tree is being linted - :return: The environment variables dictionary to pass to the linter - - """ - return { - **os.environ, - "DARKER_LINT_ORIG_REPO": str(root), - "DARKER_LINT_REV_COMMIT": ( - "WORKTREE" if revision == "WORKTREE" else revision[:7] - ), - } - - -def _strict_nonneg_int(text: str) -> int: - """Strict parsing of strings to non-negative integers - - Allow no leading or trailing whitespace, nor plus or minus signs. - - :param text: The string to convert - :raises ValueError: Raises if the string has any non-numeric characters - :return: [description] - :rtype: [type] - """ - if text.strip("+-\t ") != text: - raise ValueError(r"invalid literal for int() with base 10: {text}") - return int(text) - - -def _parse_linter_line( - linter: str, line: str, cwd: Path -) -> Tuple[MessageLocation, LinterMessage]: - """Parse one line of linter output - - Only parses lines with - - a relative or absolute file path (without leading-trailing whitespace), - - a non-negative line number (without leading/trailing whitespace), - - optionally a column number (without leading/trailing whitespace), and - - a description. - - Examples of successfully parsed lines:: - - path/to/file.py:42: Description - /absolute/path/to/file.py:42:5: Description - - Given ``cwd=Path("/absolute")``, these would be parsed into:: - - (Path("path/to/file.py"), 42, "path/to/file.py:42:", "Description") - (Path("path/to/file.py"), 42, "path/to/file.py:42:5:", "Description") - - For all other lines, a dummy entry is returned: an empty path, zero as the line - number, an empty location string and an empty description. Such lines should be - simply ignored, since many linters display supplementary information insterspersed - with the actual linting notifications. - - :param linter: The name of the linter - :param line: The linter output line to parse. May have a trailing newline. - :param cwd: The directory in which the linter was run, and relative to which paths - are returned - :return: A 2-tuple of - - the file path, line and column numbers of the linter message, and - - the linter name and message description. - - """ - try: - location, description = line.rstrip().split(": ", 1) - if location[1:3] == ":\\": - # Absolute Windows paths need special handling. Separate out the ``C:`` (or - # similar), then split by colons, and finally re-insert the ``C:``. - path_in_drive, linenum_str, *rest = location[2:].split(":") - path_str = f"{location[:2]}{path_in_drive}" - else: - path_str, linenum_str, *rest = location.split(":") - if path_str.strip() != path_str: - raise ValueError(r"Filename {path_str!r} has leading/trailing whitespace") - linenum = _strict_nonneg_int(linenum_str) - if len(rest) > 1: - raise ValueError("Too many colon-separated tokens in {location!r}") - if len(rest) == 1: - # Make sure the column looks like an int in "::" - column = _strict_nonneg_int(rest[0]) # noqa: F841 - else: - column = 0 - except ValueError: - # Encountered a non-parsable line which doesn't express a linting error. - # For example, on Mypy: - # "Found XX errors in YY files (checked ZZ source files)" - # "Success: no issues found in 1 source file" - logger.debug("Unparsable linter output: %s", line[:-1]) - return (NO_MESSAGE_LOCATION, LinterMessage(linter, "")) - path = Path(path_str) - if path.is_absolute(): - try: - path = path.relative_to(cwd) - except ValueError: - logger.warning( - "Linter message for a file %s outside root directory %s", - path_str, - cwd, - ) - return (NO_MESSAGE_LOCATION, LinterMessage(linter, "")) - return (MessageLocation(path, linenum, column), LinterMessage(linter, description)) - - -def _require_rev2_worktree(rev2: str) -> None: - """Exit with an error message if ``rev2`` is not ``WORKTREE`` - - This is used when running linters since linting arbitrary commits is not supported. - - :param rev2: The ``rev2`` revision to lint. - - """ - if rev2 != WORKTREE: - raise NotImplementedError( - "Linting arbitrary commits is not supported. " - "Please use -r {|..|...} instead." - ) - - -@contextmanager -def _check_linter_output( - cmdline: Union[str, List[str]], - root: Path, - paths: Collection[Path], - env: Dict[str, str], -) -> Generator[IO[str], None, None]: - """Run a linter as a subprocess and return its standard output stream - - :param cmdline: The command line for running the linter - :param root: The common root of all files to lint - :param paths: Paths of files to check, relative to ``root`` - :param env: Environment variables to pass to the linter - :return: The standard output stream of the linter subprocess - - """ - if isinstance(cmdline, str): - cmdline_parts = shlex.split(cmdline, posix=not WINDOWS) - else: - cmdline_parts = cmdline - cmdline_and_paths = cmdline_parts + [str(path) for path in sorted(paths)] - logger.debug("[%s]$ %s", root, shlex.join(cmdline_and_paths)) - with Popen( # nosec - cmdline_and_paths, - stdout=PIPE, - encoding="utf-8", - cwd=root, - env=env, - ) as linter_process: - # condition needed for MyPy (see https://stackoverflow.com/q/57350490/15770) - if linter_process.stdout is None: - raise RuntimeError("Stdout piping failed") - yield linter_process.stdout - - -def run_linter( # pylint: disable=too-many-locals - cmdline: Union[str, List[str]], - root: Path, - paths: Collection[Path], - env: Dict[str, str], -) -> Dict[MessageLocation, LinterMessage]: - """Run the given linter and return linting errors falling on changed lines - - :param cmdline: The command line for running the linter - :param root: The common root of all files to lint - :param paths: Paths of files to check, relative to ``root`` - :param env: Environment variables to pass to the linter - :return: The number of modified lines with linting errors from this linter - - """ - missing_files = set() - result = {} - if isinstance(cmdline, str): - linter = shlex.split(cmdline, posix=not WINDOWS)[0] - cmdline_str = cmdline - else: - linter = cmdline[0] - cmdline_str = shlex.join(cmdline) - # 10. run a linter subprocess for files mentioned on the command line which may be - # modified or unmodified, to get current linting status in the working tree - # (steps 10.-12. are optional) - with _check_linter_output(cmdline, root, paths, env) as linter_stdout: - for line in linter_stdout: - (location, message) = _parse_linter_line(linter, line, root) - if location is NO_MESSAGE_LOCATION or location.path in missing_files: - continue - if location.path.suffix != ".py": - logger.warning( - "Linter message for a non-Python file: %s: %s", - location, - message.description, - ) - continue - if not location.path.is_file() and not location.path.is_symlink(): - logger.warning("Missing file %s from %s", location.path, cmdline_str) - missing_files.add(location.path) - continue - result[location] = message - return result - - -def run_linters( - linter_cmdlines: List[Union[str, List[str]]], - root: Path, - paths: Set[Path], - revrange: RevisionRange, - use_color: bool, -) -> int: - """Run the given linters on a set of files in the repository, filter messages - - Linter message filtering works by - - - running linters once in ``rev1`` to establish a baseline, - - running them again in ``rev2`` to get linter messages after user changes, and - - printing out only new messages which were not present in the baseline. - - If the source tree is not a Git repository, a baseline is not used, and all linter - messages are printed - - :param linter_cmdlines: The command lines for linter tools to run on the files - :param root: The root of the relative paths - :param paths: The files and directories to check, relative to ``root`` - :param revrange: The Git revisions to compare - :param use_color: ``True`` to use syntax highlighting for linter output - :raises NotImplementedError: if ``--stdin-filename`` is used - :return: Total number of linting errors found on modified lines - - """ - if not linter_cmdlines: - return 0 - if revrange.rev2 == STDIN: - raise NotImplementedError( - "The -l/--lint option isn't yet available with --stdin-filename" - ) - _require_rev2_worktree(revrange.rev2) - git_root = git_get_root(root) - if not git_root: - # In a non-Git root, don't use a baseline - messages = _get_messages_from_linters( - linter_cmdlines, - root, - paths, - make_linter_env(root, "WORKTREE"), - ) - return _print_new_linter_messages( - baseline={}, - new_messages=messages, - diff_line_mapping=DiffLineMapping(), - use_color=use_color, - ) - git_paths = {(root / path).relative_to(git_root) for path in paths} - # 10. first do a temporary checkout at `rev1` and run linter subprocesses once for - # all files which are mentioned on the command line to establish a baseline - # (steps 10.-12. are optional) - baseline = _get_messages_from_linters_for_baseline( - linter_cmdlines, - git_root, - git_paths, - revrange.rev1, - ) - messages = _get_messages_from_linters( - linter_cmdlines, - git_root, - git_paths, - make_linter_env(git_root, "WORKTREE"), - ) - files_with_messages = {location.path for location in messages} - # 11. create a mapping from line numbers of unmodified lines in the current versions - # to corresponding line numbers in ``rev1`` - diff_line_mapping = _create_line_mapping(git_root, files_with_messages, revrange) - # 12. hide linter messages which appear in the current versions and identically on - # corresponding lines in ``rev1``, and show all other linter messages - return _print_new_linter_messages(baseline, messages, diff_line_mapping, use_color) - - -def _identity_line_processor(message: LinterMessage) -> LinterMessage: - """Return message unmodified in the default line processor - - :param message: The original message - :return: The unmodified message - - """ - return message - - -def _get_messages_from_linters( - linter_cmdlines: Iterable[Union[str, List[str]]], - root: Path, - paths: Collection[Path], - env: Dict[str, str], - line_processor: Callable[[LinterMessage], LinterMessage] = _identity_line_processor, -) -> Dict[MessageLocation, List[LinterMessage]]: - """Run given linters for the given directory and return linting errors - - :param linter_cmdlines: The command lines for running the linters - :param root: The common root of all files to lint - :param paths: Paths of files to check, relative to ``root`` - :param env: The environment variables to pass to the linter - :param line_processor: Pre-processing callback for linter output lines - :return: Linter messages - - """ - result = defaultdict(list) - for cmdline in linter_cmdlines: - for message_location, message in run_linter(cmdline, root, paths, env).items(): - result[message_location].append(line_processor(message)) - return result - - -def _print_new_linter_messages( - baseline: Dict[MessageLocation, List[LinterMessage]], - new_messages: Dict[MessageLocation, List[LinterMessage]], - diff_line_mapping: DiffLineMapping, - use_color: bool, -) -> int: - """Print all linter messages except those same as before on unmodified lines - - :param baseline: Linter messages and their locations for a previous version - :param new_messages: New linter messages in a new version of the source file - :param diff_line_mapping: Mapping between unmodified lines in old and new versions - :param use_color: ``True`` to highlight linter messages in the output - :return: The number of linter errors displayed - - """ - error_count = 0 - prev_location = NO_MESSAGE_LOCATION - for message_location, messages in sorted(new_messages.items()): - old_location = diff_line_mapping.get(message_location) - is_modified_line = old_location == NO_MESSAGE_LOCATION - old_messages: List[LinterMessage] = baseline.get(old_location, []) - for message in messages: - if not is_modified_line and normalize_whitespace(message) in old_messages: - # Only hide messages when - # - they occurred previously on the corresponding line - # - the line hasn't been modified - continue - if ( - message_location.path != prev_location.path - or message_location.line > prev_location.line + 1 - ): - print() - prev_location = message_location - print(colorize(f"{message_location}:", "lint_location", use_color), end=" ") - print(colorize(message.description, "lint_description", use_color), end=" ") - print(f"[{message.linter}]") - error_count += 1 - return error_count - - -def _get_messages_from_linters_for_baseline( - linter_cmdlines: List[Union[str, List[str]]], - root: Path, - paths: Collection[Path], - revision: str, -) -> Dict[MessageLocation, List[LinterMessage]]: - """Clone the Git repository at a given revision and run linters against it - - :param linter_cmdlines: The command lines for linter tools to run on the files - :param root: The root of the Git repository - :param paths: The files and directories to check, relative to ``root`` - :param revision: The revision to check out - :return: Linter messages - - """ - with TemporaryDirectory() as tmpdir: - tmp_path = Path(tmpdir) / "baseline-revision" / root.name - with git_clone_local(root, revision, tmp_path) as clone_root: - rev1_commit = git_rev_parse(revision, root) - result = _get_messages_from_linters( - linter_cmdlines, - clone_root, - paths, - make_linter_env(root, rev1_commit), - normalize_whitespace, - ) - return result - - -def _create_line_mapping( - root: Path, files_with_messages: Iterable[Path], revrange: RevisionRange -) -> DiffLineMapping: - """Create a mapping from unmodified lines in new files to same lines in old versions - - :param root: The root of the repository - :param files_with_messages: Paths to files which have linter messages - :param revrange: The revisions to compare - :return: A dict which maps the line number of each unmodified line in the new - versions of files to corresponding line numbers in old versions of the same - files - - """ - diff_line_mapping = DiffLineMapping() - for path in files_with_messages: - doc1 = git_get_content_at_revision(path, revrange.rev1, root) - doc2 = git_get_content_at_revision(path, revrange.rev2, root) - for linenum2, linenum1 in map_unmodified_lines(doc1, doc2).items(): - location1 = MessageLocation(path, linenum1) - location2 = MessageLocation(path, linenum2) - diff_line_mapping[location2] = location1 - return diff_line_mapping diff --git a/src/darker/tests/test_linting.py b/src/darker/tests/test_linting.py deleted file mode 100644 index 7849401fd..000000000 --- a/src/darker/tests/test_linting.py +++ /dev/null @@ -1,603 +0,0 @@ -# pylint: disable=protected-access,too-many-arguments,use-dict-literal - -"""Unit tests for :mod:`darker.linting`""" - -import os -from pathlib import Path -from subprocess import PIPE, Popen # nosec -from textwrap import dedent -from typing import Any, Dict, Iterable, List, Tuple, Union -from unittest.mock import patch - -import pytest - -from darker import linting -from darker.git import WORKTREE, RevisionRange -from darker.linting import ( - DiffLineMapping, - LinterMessage, - MessageLocation, - make_linter_env, -) -from darker.tests.helpers import raises_if_exception -from darker.utils import WINDOWS - -SKIP_ON_WINDOWS = [pytest.mark.skip] if WINDOWS else [] -SKIP_ON_UNIX = [] if WINDOWS else [pytest.mark.skip] - - -@pytest.mark.kwparametrize( - dict(column=0, expect=f"{Path('/path/to/file.py')}:42"), - dict(column=5, expect=f"{Path('/path/to/file.py')}:42:5"), -) -def test_message_location_str(column, expect): - """Null column number is hidden from string representation of message location""" - location = MessageLocation(Path("/path/to/file.py"), 42, column) - - result = str(location) - - assert result == expect - - -@pytest.mark.kwparametrize( - dict( - new_location=("/path/to/new_file.py", 43, 8), - old_location=("/path/to/old_file.py", 42, 13), - get_location=("/path/to/new_file.py", 43, 21), - expect_location=("/path/to/old_file.py", 42, 21), - ), - dict( - new_location=("/path/to/new_file.py", 43, 8), - old_location=("/path/to/old_file.py", 42, 13), - get_location=("/path/to/a_different_file.py", 43, 21), - expect_location=("", 0, 0), - ), - dict( - new_location=("/path/to/file.py", 43, 8), - old_location=("/path/to/file.py", 42, 13), - get_location=("/path/to/file.py", 42, 21), - expect_location=("", 0, 0), - ), -) -def test_diff_line_mapping_ignores_column( - new_location, old_location, get_location, expect_location -): - """Diff location mapping ignores column and attaches column of queried location""" - mapping = linting.DiffLineMapping() - new_location_ = MessageLocation(Path(new_location[0]), *new_location[1:]) - old_location = MessageLocation(Path(old_location[0]), *old_location[1:]) - get_location = MessageLocation(Path(get_location[0]), *get_location[1:]) - expect = MessageLocation(Path(expect_location[0]), *expect_location[1:]) - - mapping[new_location_] = old_location - result = mapping.get(get_location) - - assert result == expect - - -def test_normalize_whitespace(): - """Whitespace runs and leading/trailing whitespace is normalized""" - description = "module.py:42: \t indented message, trailing spaces and tabs \t " - message = LinterMessage("mylinter", description) - - result = linting.normalize_whitespace(message) - - assert result == LinterMessage( - "mylinter", "module.py:42: indented message, trailing spaces and tabs" - ) - - -@pytest.mark.kwparametrize( - dict( - line="module.py:42: Just a line number\n", - expect=(Path("module.py"), 42, 0, "Just a line number"), - ), - dict( - line="module.py:42:5: With column \n", - expect=(Path("module.py"), 42, 5, "With column"), - ), - dict( - line="{git_root_absolute}{sep}mod.py:42: Full path\n", - expect=(Path("mod.py"), 42, 0, "Full path"), - ), - dict( - line="{git_root_absolute}{sep}mod.py:42:5: Full path with column\n", - expect=(Path("mod.py"), 42, 5, "Full path with column"), - ), - dict( - line="mod.py:42: 123 digits start the description\n", - expect=(Path("mod.py"), 42, 0, "123 digits start the description"), - ), - dict( - line="mod.py:42: indented description\n", - expect=(Path("mod.py"), 42, 0, " indented description"), - ), - dict( - line="mod.py:42:5: indented description\n", - expect=(Path("mod.py"), 42, 5, " indented description"), - ), - dict( - line="nonpython.txt:5: Non-Python file\n", - expect=(Path("nonpython.txt"), 5, 0, "Non-Python file"), - ), - dict(line="mod.py: No line number\n", expect=(Path(), 0, 0, "")), - dict(line="mod.py:foo:5: Invalid line number\n", expect=(Path(), 0, 0, "")), - dict(line="mod.py:42:bar: Invalid column\n", expect=(Path(), 0, 0, "")), - dict( - line="/outside/mod.py:5: Outside the repo\n", - expect=(Path(), 0, 0, ""), - marks=SKIP_ON_WINDOWS, - ), - dict( - line="C:\\outside\\mod.py:5: Outside the repo\n", - expect=(Path(), 0, 0, ""), - marks=SKIP_ON_UNIX, - ), - dict(line="invalid linter output\n", expect=(Path(), 0, 0, "")), - dict(line=" leading:42: whitespace\n", expect=(Path(), 0, 0, "")), - dict(line=" leading:42:5 whitespace and column\n", expect=(Path(), 0, 0, "")), - dict(line="trailing :42: filepath whitespace\n", expect=(Path(), 0, 0, "")), - dict(line="leading: 42: linenum whitespace\n", expect=(Path(), 0, 0, "")), - dict(line="trailing:42 : linenum whitespace\n", expect=(Path(), 0, 0, "")), - dict(line="plus:+42: before linenum\n", expect=(Path(), 0, 0, "")), - dict(line="minus:-42: before linenum\n", expect=(Path(), 0, 0, "")), - dict(line="plus:42:+5 before column\n", expect=(Path(), 0, 0, "")), - dict(line="minus:42:-5 before column\n", expect=(Path(), 0, 0, "")), -) -def test_parse_linter_line(git_repo, monkeypatch, line, expect): - """Linter output is parsed correctly""" - monkeypatch.chdir(git_repo.root) - root_abs = git_repo.root.absolute() - line_expanded = line.format(git_root_absolute=root_abs, sep=os.sep) - - result = linting._parse_linter_line("linter", line_expanded, git_repo.root) - - assert result == (MessageLocation(*expect[:3]), LinterMessage("linter", expect[3])) - - -@pytest.mark.kwparametrize( - dict(rev2="master", expect=NotImplementedError), - dict(rev2=WORKTREE, expect=None), -) -def test_require_rev2_worktree(rev2, expect): - """``_require_rev2_worktree`` raises an exception if rev2 is not ``WORKTREE``""" - with raises_if_exception(expect): - - linting._require_rev2_worktree(rev2) - - -@pytest.mark.kwparametrize( - dict(cmdline="echo", expect=["first.py the 2nd.py\n"]), - dict(cmdline="echo words before", expect=["words before first.py the 2nd.py\n"]), - dict( - cmdline='echo "two spaces"', - expect=["two spaces first.py the 2nd.py\n"], - marks=[ - pytest.mark.xfail( - reason=( - "Quotes not removed on Windows." - " See https://github.com/akaihola/darker/issues/456" - ) - ) - ] - if WINDOWS - else [], - ), - dict(cmdline="echo eat spaces", expect=["eat spaces first.py the 2nd.py\n"]), -) -def test_check_linter_output(tmp_path, cmdline, expect): - """``_check_linter_output()`` runs linter and returns the stdout stream""" - with linting._check_linter_output( - cmdline, - tmp_path, - {Path("first.py"), Path("the 2nd.py")}, - make_linter_env(tmp_path, "WORKTREE"), - ) as stdout: - lines = list(stdout) - - assert lines == expect - - -@pytest.mark.kwparametrize( - dict( - _descr="New message for test.py", - messages_after=["test.py:1: new message"], - expect_output=["", "test.py:1: new message [cat]"], - ), - dict( - _descr="New message for test.py, including column number", - messages_after=["test.py:1:42: new message with column number"], - expect_output=["", "test.py:1:42: new message with column number [cat]"], - ), - dict( - _descr="Identical message on an unmodified unmoved line in test.py", - messages_before=["test.py:1:42: same message on same line"], - messages_after=["test.py:1:42: same message on same line"], - ), - dict( - _descr="Identical message on an unmodified moved line in test.py", - messages_before=["test.py:3:42: same message on a moved line"], - messages_after=["test.py:4:42: same message on a moved line"], - ), - dict( - _descr="Additional message on an unmodified moved line in test.py", - messages_before=["test.py:3:42: same message"], - messages_after=[ - "test.py:4:42: same message", - "test.py:4:42: additional message", - ], - expect_output=["", "test.py:4:42: additional message [cat]"], - ), - dict( - _descr="Changed message on an unmodified moved line in test.py", - messages_before=["test.py:4:42: old message"], - messages_after=["test.py:4:42: new message"], - expect_output=["", "test.py:4:42: new message [cat]"], - ), - dict( - _descr="Identical message but on an inserted line in test.py", - messages_before=["test.py:1:42: same message also on an inserted line"], - messages_after=[ - "test.py:1:42: same message also on an inserted line", - "test.py:2:42: same message also on an inserted line", - ], - expect_output=["", "test.py:2:42: same message also on an inserted line [cat]"], - ), - dict( - _descr="Warning for a file missing from the working tree", - messages_after=["missing.py:1: a missing Python file"], - expect_log=["WARNING Missing file missing.py from cat messages"], - ), - dict( - _descr="Linter message for a non-Python file is ignored with a warning", - messages_after=["nonpython.txt:1: non-py"], - expect_log=[ - "WARNING Linter message for a non-Python file: nonpython.txt:1: non-py" - ], - ), - dict( - _descr="Message for file outside common root is ignored with a warning (Unix)", - messages_after=["/elsewhere/mod.py:1: elsewhere"], - expect_log=[ - "WARNING Linter message for a file /elsewhere/mod.py outside root" - " directory {root}" - ], - marks=SKIP_ON_WINDOWS, - ), - dict( - _descr="Message for file outside common root is ignored with a warning (Win)", - messages_after=["C:\\elsewhere\\mod.py:1: elsewhere"], - expect_log=[ - "WARNING Linter message for a file C:\\elsewhere\\mod.py outside root" - " directory {root}" - ], - marks=SKIP_ON_UNIX, - ), - messages_before=[], - expect_output=[], - expect_log=[], -) -def test_run_linters( - git_repo, - capsys, - caplog, - _descr, - messages_before, - messages_after, - expect_output, - expect_log, -): - """Linter gets correct paths on command line and outputs just changed lines - - We use ``echo`` as our "linter". It just adds the paths of each file to lint as an - "error" on a line of ``test.py``. What this test does is the equivalent of e.g.:: - - - creating a ``test.py`` such that the first line is modified after the last commit - - creating and committing ``one.py`` and ``two.py`` - - running:: - - $ darker -L 'echo test.py:1:' one.py two.py - test.py:1: git-repo-root/one.py git-repo-root/two.py - - """ - src_paths = git_repo.add( - { - "test.py": "1 unmoved\n2 modify\n3 to 4 moved\n", - "nonpython.txt": "hello\n", - "messages": "\n".join(messages_before), - }, - commit="Initial commit", - ) - src_paths["test.py"].write_bytes( - b"1 unmoved\n2 modified\n3 inserted\n3 to 4 moved\n" - ) - src_paths["messages"].write_text("\n".join(messages_after)) - cmdlines: List[Union[str, List[str]]] = ["cat messages"] - revrange = RevisionRange("HEAD", ":WORKTREE:") - - linting.run_linters( - cmdlines, git_repo.root, {Path("dummy path")}, revrange, use_color=False - ) - - # We can now verify that the linter received the correct paths on its command line - # by checking standard output from the our `echo` "linter". - # The test cases also verify that only linter reports on modified lines are output. - result = capsys.readouterr().out.splitlines() - assert result == git_repo.expand_root(expect_output) - logs = [ - f"{record.levelname} {record.message}" - for record in caplog.records - if record.levelname != "DEBUG" - ] - assert logs == git_repo.expand_root(expect_log) - - -def test_run_linters_non_worktree(): - """``run_linters()`` doesn't support linting commits, only the worktree""" - with pytest.raises(NotImplementedError): - - linting.run_linters( - ["dummy-linter"], - Path("/dummy"), - {Path("dummy.py")}, - RevisionRange.parse_with_common_ancestor( - "..HEAD", Path("dummy cwd"), stdin_mode=False - ), - use_color=False, - ) - - -@pytest.mark.parametrize( - "message, expect", - [ - ("", 0), - ("test.py:1: message on modified line", 1), - ("test.py:2: message on unmodified line", 0), - ], -) -def test_run_linters_return_value(git_repo, message, expect): - """``run_linters()`` returns the number of linter errors on modified lines""" - src_paths = git_repo.add({"test.py": "1\n2\n"}, commit="Initial commit") - src_paths["test.py"].write_bytes(b"one\n2\n") - cmdline = f"echo {message}" - - result = linting.run_linters( - [cmdline], - git_repo.root, - {Path("test.py")}, - RevisionRange("HEAD", ":WORKTREE:"), - use_color=False, - ) - - assert result == expect - - -def test_run_linters_on_new_file(git_repo, capsys): - """``run_linters()`` considers file missing from history as empty - - Passes through all linter errors as if the original file was empty. - - """ - git_repo.add({"file1.py": "1\n"}, commit="Initial commit") - git_repo.create_tag("initial") - (git_repo.root / "file2.py").write_bytes(b"1\n2\n") - - linting.run_linters( - ["echo file2.py:1: message on a file not seen in Git history"], - Path(git_repo.root), - {Path("file2.py")}, - RevisionRange("initial", ":WORKTREE:"), - use_color=False, - ) - - output = capsys.readouterr().out.splitlines() - assert output == [ - "", - "file2.py:1: message on a file not seen in Git history file2.py [echo]", - ] - - -def test_run_linters_line_separation(git_repo, capsys): - """``run_linters`` separates contiguous blocks of linter output with empty lines""" - paths = git_repo.add({"a.py": "1\n2\n3\n4\n5\n6\n"}, commit="Initial commit") - paths["a.py"].write_bytes(b"a\nb\nc\nd\ne\nf\n") - linter_output = git_repo.root / "dummy-linter-output.txt" - linter_output.write_text( - dedent( - """ - a.py:2: first block - a.py:3: of linter output - a.py:5: second block - a.py:6: of linter output - """ - ) - ) - cat_command = "cmd /c type" if WINDOWS else "cat" - - linting.run_linters( - [f"{cat_command} {linter_output}"], - git_repo.root, - {Path(p) for p in paths}, - RevisionRange("HEAD", ":WORKTREE:"), - use_color=False, - ) - - result = capsys.readouterr().out - cat_cmd = "cmd" if WINDOWS else "cat" - assert result == dedent( - f""" - a.py:2: first block [{cat_cmd}] - a.py:3: of linter output [{cat_cmd}] - - a.py:5: second block [{cat_cmd}] - a.py:6: of linter output [{cat_cmd}] - """ - ) - - -def test_run_linters_stdin(): - """`linting.run_linters` raises a `NotImplementeError` on ``--stdin-filename``""" - with pytest.raises( - NotImplementedError, - match=r"^The -l/--lint option isn't yet available with --stdin-filename$", - ): - # end of test setup - - _ = linting.run_linters( - ["dummy-linter-command"], - Path("/dummy-dir"), - {Path("dummy.py")}, - RevisionRange("HEAD", ":STDIN:"), - use_color=False, - ) - - -def _build_messages( - lines_and_messages: Iterable[Union[Tuple[int, str], Tuple[int, str, str]]], -) -> Dict[MessageLocation, List[LinterMessage]]: - return { - MessageLocation(Path("a.py"), line, 0): [ - LinterMessage(*msg.split(":")) for msg in msgs - ] - for line, *msgs in lines_and_messages - } - - -def test_print_new_linter_messages(capsys): - """`linting._print_new_linter_messages()` hides old intact linter messages""" - baseline = _build_messages( - [ - (2, "mypy:single message on an unmodified line"), - (4, "mypy:single message on a disappearing line"), - (6, "mypy:single message on a moved line"), - (8, "mypy:single message on a modified line"), - (10, "mypy:multiple messages", "pylint:on the same moved line"), - ( - 12, - "mypy:old message which will be replaced", - "pylint:on an unmodified line", - ), - (14, "mypy:old message on a modified line"), - ] - ) - new_messages = _build_messages( - [ - (2, "mypy:single message on an unmodified line"), - (5, "mypy:single message on a moved line"), - (8, "mypy:single message on a modified line"), - (11, "mypy:multiple messages", "pylint:on the same moved line"), - ( - 12, - "mypy:new message replacing the old one", - "pylint:on an unmodified line", - ), - (14, "mypy:new message on a modified line"), - (16, "mypy:multiple messages", "pylint:on the same new line"), - ] - ) - diff_line_mapping = DiffLineMapping() - for new_line, old_line in {2: 2, 5: 6, 11: 10, 12: 12}.items(): - diff_line_mapping[MessageLocation(Path("a.py"), new_line)] = MessageLocation( - Path("a.py"), old_line - ) - - linting._print_new_linter_messages( - baseline, new_messages, diff_line_mapping, use_color=False - ) - - result = capsys.readouterr().out.splitlines() - assert result == [ - "", - "a.py:8: single message on a modified line [mypy]", - "", - "a.py:12: new message replacing the old one [mypy]", - "", - "a.py:14: new message on a modified line [mypy]", - "", - "a.py:16: multiple messages [mypy]", - "a.py:16: on the same new line [pylint]", - ] - - -LINT_EMPTY_LINES_CMD = [ - "python", - "-c", - dedent( - """ - from pathlib import Path - for path in Path(".").glob("**/*.py"): - for linenum, line in enumerate(path.open(), start=1): - if not line.strip(): - print(f"{path}:{linenum}: EMPTY") - """ - ), -] - -LINT_NONEMPTY_LINES_CMD = [ - "python", - "-c", - dedent( - """ - from pathlib import Path - for path in Path(".").glob("**/*.py"): - for linenum, line in enumerate(path.open(), start=1): - if line.strip(): - print(f"{path}:{linenum}: {line.strip()}") - """ - ), -] - - -def test_get_messages_from_linters_for_baseline(git_repo): - """Test for `linting._get_messages_from_linters_for_baseline`""" - git_repo.add({"a.py": "First line\n\nThird line\n"}, commit="Initial commit") - initial = git_repo.get_hash() - git_repo.add({"a.py": "Just one line\n"}, commit="Second commit") - git_repo.create_branch("baseline", initial) - - result = linting._get_messages_from_linters_for_baseline( - linter_cmdlines=[LINT_EMPTY_LINES_CMD, LINT_NONEMPTY_LINES_CMD], - root=git_repo.root, - paths=[Path("a.py"), Path("subdir/b.py")], - revision="baseline", - ) - - a_py = Path("a.py") - expect = { - MessageLocation(a_py, 1): [LinterMessage("python", "First line")], - MessageLocation(a_py, 2): [LinterMessage("python", "EMPTY")], - MessageLocation(a_py, 3): [LinterMessage("python", "Third line")], - } - assert result == expect - - -class AssertEmptyStderrPopen( - Popen # type: ignore[type-arg] -): # pylint: disable=too-few-public-methods - # When support for Python 3.8 is dropped, inherit `Popen[str]` instead and remove - # the `type-arg` ignore above. - """A Popen to use for the following test; asserts that its stderr is empty""" - - def __init__(self, args: List[str], **kwargs: Any): # type: ignore[misc] - super().__init__(args, stderr=PIPE, **kwargs) - assert self.stderr is not None - assert self.stderr.read() == "" - - -def test_get_messages_from_linters_for_baseline_no_mypy_errors(git_repo): - """Ensure Mypy does not fail early when ``__init__.py`` is at the repository root - - Regression test for #498 - - """ - git_repo.add({"__init__.py": ""}, commit="Initial commit") - initial = git_repo.get_hash() - with patch.object(linting, "Popen", AssertEmptyStderrPopen): - # end of test setup - - _ = linting._get_messages_from_linters_for_baseline( - linter_cmdlines=["mypy"], - root=git_repo.root, - paths=[Path("__init__.py")], - revision=initial, - ) diff --git a/src/darker/tests/test_main.py b/src/darker/tests/test_main.py index 52c4609a1..7ccee1c8e 100644 --- a/src/darker/tests/test_main.py +++ b/src/darker/tests/test_main.py @@ -19,7 +19,6 @@ import darker.__main__ import darker.import_sorting -import darker.linting from darker.config import Exclusions from darker.exceptions import MissingPackageError from darker.git import WORKTREE, EditedLinenumsDiffer, RevisionRange From a43bc33fdc9ed1eb28facb16a56b45a293525e4f Mon Sep 17 00:00:00 2001 From: Antti Kaihola <13725+akaihola@users.noreply.github.com> Date: Thu, 23 Feb 2023 23:55:04 +0200 Subject: [PATCH 05/30] Inherit `DarkerConfig` from `BaseConfig` It's in `darkgraylib.config`. --- src/darker/config.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/src/darker/config.py b/src/darker/config.py index 02f47584c..d954235a1 100644 --- a/src/darker/config.py +++ b/src/darker/config.py @@ -5,11 +5,12 @@ from argparse import ArgumentParser, Namespace from dataclasses import dataclass, field from pathlib import Path -from typing import Dict, Iterable, List, Optional, Set, TypedDict, Union, cast +from typing import Dict, Iterable, List, Optional, Set, Union, cast import toml from darker.black_compat import find_project_root +from darkgraylib.config import BaseConfig class TomlArrayLinesEncoder(toml.TomlEncoder): # type: ignore @@ -23,24 +24,17 @@ def dump_list(self, v: Iterable[object]) -> str: UnvalidatedConfig = Dict[str, Union[List[str], str, bool, int]] -class DarkerConfig(TypedDict, total=False): +class DarkerConfig(BaseConfig, total=False): """Dictionary representing ``[tool.darker]`` from ``pyproject.toml``""" - src: List[str] - revision: str diff: bool - stdout: bool check: bool isort: bool lint: List[str] - config: str - log_level: int - color: bool skip_string_normalization: bool skip_magic_trailing_comma: bool line_length: int target_version: str - workers: int class OutputMode: From c8170af9ccf83e749543b73d5644edd5e33ed8fc Mon Sep 17 00:00:00 2001 From: Antti Kaihola <13725+akaihola@users.noreply.github.com> Date: Fri, 24 Feb 2023 00:04:31 +0200 Subject: [PATCH 06/30] Move `RevisionRange` to `darkgraylib.git` --- src/darker/__main__.py | 2 +- src/darker/git.py | 110 +--------- src/darker/tests/test_command_line.py | 2 +- src/darker/tests/test_fstring.py | 3 +- src/darker/tests/test_git.py | 197 +----------------- src/darker/tests/test_import_sorting.py | 3 +- src/darker/tests/test_main.py | 3 +- ...test_main_blacken_and_flynt_single_file.py | 3 +- 8 files changed, 18 insertions(+), 305 deletions(-) diff --git a/src/darker/__main__.py b/src/darker/__main__.py index 64377775e..62f82add5 100644 --- a/src/darker/__main__.py +++ b/src/darker/__main__.py @@ -29,7 +29,6 @@ STDIN, WORKTREE, EditedLinenumsDiffer, - RevisionRange, get_missing_at_revision, get_path_in_repo, git_get_content_at_revision, @@ -50,6 +49,7 @@ from darker.verification import ASTVerifier, BinarySearch, NotEquivalentError from darkgraylib.command_line import parse_command_line from darkgraylib.config import show_config_if_debug +from darkgraylib.git import RevisionRange from darkgraylib.log import setup_logging from darkgraylib.main import resolve_paths from graylint.linting import run_linters diff --git a/src/darker/git.py b/src/darker/git.py index 370a92777..6d1dc743b 100644 --- a/src/darker/git.py +++ b/src/darker/git.py @@ -28,7 +28,7 @@ from darker.diff import diff_and_get_opcodes, opcodes_to_edit_linenums from darker.multiline_strings import get_multiline_string_ranges from darker.utils import GIT_DATEFORMAT, TextDocument - +from darkgraylib.git import RevisionRange logger = logging.getLogger(__name__) @@ -143,114 +143,6 @@ def git_get_content_at_revision(path: Path, revision: str, cwd: Path) -> TextDoc return TextDocument() -@dataclass(frozen=True) -class RevisionRange: - """Represent a range of commits in a Git repository for comparing differences - - ``rev1`` is the "old" revision, and ``rev2``, the "new" revision which should be - compared against ``rev1``. - - When parsing a revision range expression with triple dots (e.g. ``master...HEAD``), - the branch point, or common ancestor of the revisions, is used instead of the - provided ``rev1``. This is useful e.g. when CI is doing a check - on a feature branch, and there have been commits in the main branch after the branch - point. Without the ability to compare to the branch point, Darker would suggest - corrections to formatting on lines changes in the main branch even if those lines - haven't been touched in the feature branch. - - """ - - rev1: str - rev2: str - - @classmethod - def parse_with_common_ancestor( - cls, revision_range: str, cwd: Path, stdin_mode: bool - ) -> "RevisionRange": - """Convert a range expression to a ``RevisionRange`` object - - If the expression contains triple dots (e.g. ``master...HEAD``), finds the - common ancestor of the two revisions and uses that as the first revision. - - :param revision_range: The revision range as a string to parse - :param cwd: The working directory to use if invoking Git - :param stdin_mode: If `True`, the default for ``rev2`` is ``:STDIN:`` - :return: The range parsed into a `RevisionRange` object - - """ - rev1, rev2, use_common_ancestor = cls._parse(revision_range, stdin_mode) - if use_common_ancestor: - return cls._with_common_ancestor(rev1, rev2, cwd) - return cls(rev1, rev2) - - @staticmethod - def _parse(revision_range: str, stdin_mode: bool) -> Tuple[str, str, bool]: - """Convert a range expression to revisions, using common ancestor if appropriate - - A `ValueError` is raised if ``--stdin-filename`` is used by the revision range - is ``:PRE-COMMIT:`` or the end of the range is not ``:STDIN:``. - - :param revision_range: The revision range as a string to parse - :param stdin_mode: If `True`, the default for ``rev2`` is ``:STDIN:`` - :raises ValueError: for an invalid revision when ``--stdin-filename`` is used - :return: The range parsed into a `RevisionRange` object - - >>> RevisionRange._parse("a..b", stdin_mode=False) - ('a', 'b', False) - >>> RevisionRange._parse("a...b", stdin_mode=False) - ('a', 'b', True) - >>> RevisionRange._parse("a..", stdin_mode=False) - ('a', ':WORKTREE:', False) - >>> RevisionRange._parse("a...", stdin_mode=False) - ('a', ':WORKTREE:', True) - >>> RevisionRange._parse("a..", stdin_mode=True) - ('a', ':STDIN:', False) - >>> RevisionRange._parse("a...", stdin_mode=True) - ('a', ':STDIN:', True) - - """ - if revision_range == PRE_COMMIT_FROM_TO_REFS: - if stdin_mode: - raise ValueError( - f"With --stdin-filename, revision {revision_range!r} is not allowed" - ) - try: - return ( - os.environ["PRE_COMMIT_FROM_REF"], - os.environ["PRE_COMMIT_TO_REF"], - True, - ) - except KeyError: - # Fallback to running against HEAD - revision_range = "HEAD" - match = COMMIT_RANGE_RE.match(revision_range) - default_rev2 = STDIN if stdin_mode else WORKTREE - if match: - rev1, range_dots, rev2 = match.groups() - use_common_ancestor = range_dots == "..." - effective_rev2 = rev2 or default_rev2 - if stdin_mode and effective_rev2 != STDIN: - raise ValueError( - f"With --stdin-filename, rev2 in {revision_range} must be" - f" {STDIN!r}, not {effective_rev2!r}" - ) - return (rev1 or "HEAD", rev2 or default_rev2, use_common_ancestor) - return ( - revision_range or "HEAD", - default_rev2, - revision_range not in ["", "HEAD"], - ) - - @classmethod - def _with_common_ancestor(cls, rev1: str, rev2: str, cwd: Path) -> "RevisionRange": - """Find common ancestor for revisions and return a ``RevisionRange`` object""" - rev2_for_merge_base = "HEAD" if rev2 in [WORKTREE, STDIN] else rev2 - merge_base_cmd = ["merge-base", rev1, rev2_for_merge_base] - common_ancestor = _git_check_output_lines(merge_base_cmd, cwd)[0] - rev1_hash = _git_check_output_lines(["show", "-s", "--pretty=%H", rev1], cwd)[0] - return cls(rev1 if common_ancestor == rev1_hash else common_ancestor, rev2) - - def get_path_in_repo(path: Path) -> Path: """Return the relative path to the file in the old revision diff --git a/src/darker/tests/test_command_line.py b/src/darker/tests/test_command_line.py index e21745297..cd655e615 100644 --- a/src/darker/tests/test_command_line.py +++ b/src/darker/tests/test_command_line.py @@ -19,7 +19,6 @@ from darker.__main__ import main from darker.command_line import make_argument_parser, parse_command_line from darker.config import ConfigurationError, Exclusions -from darker.git import RevisionRange from darker.tests.helpers import ( filter_dict, flynt_present, @@ -27,6 +26,7 @@ raises_if_exception, ) from darker.utils import TextDocument, joinlines +from darkgraylib.git import RevisionRange pytestmark = pytest.mark.usefixtures("find_project_root_cache_clear") diff --git a/src/darker/tests/test_fstring.py b/src/darker/tests/test_fstring.py index 0d4ba2c31..d952d46fd 100644 --- a/src/darker/tests/test_fstring.py +++ b/src/darker/tests/test_fstring.py @@ -8,9 +8,10 @@ import pytest import darker.fstring -from darker.git import EditedLinenumsDiffer, RevisionRange +from darker.git import EditedLinenumsDiffer from darker.tests.helpers import flynt_present from darker.utils import TextDocument, joinlines +from darkgraylib.git import RevisionRange ORIGINAL_SOURCE = ("'{}'.format(x)", "#", "'{0}'.format(42)") MODIFIED_SOURCE = ("'{}'.format( x)", "#", "'{0}'.format( 42)") diff --git a/src/darker/tests/test_git.py b/src/darker/tests/test_git.py index 864330ac6..d6cf8a3d6 100644 --- a/src/darker/tests/test_git.py +++ b/src/darker/tests/test_git.py @@ -16,8 +16,9 @@ from darker import git from darker.tests.conftest import GitRepoFixture -from darker.tests.helpers import raises_if_exception, raises_or_matches +from darker.tests.helpers import raises_or_matches from darker.utils import GIT_DATEFORMAT, TextDocument +from darkgraylib.git import RevisionRange def test_tmp_path_sanity(tmp_path): @@ -118,73 +119,6 @@ def test_git_get_content_at_revision(git_repo, revision, expect_lines, expect_mt assert result.encoding == "utf-8" -@pytest.mark.kwparametrize( - dict(revision_range="", stdin_mode=False, expect=("HEAD", ":WORKTREE:", False)), - dict(revision_range="HEAD", stdin_mode=False, expect=("HEAD", ":WORKTREE:", False)), - dict(revision_range="a", stdin_mode=False, expect=("a", ":WORKTREE:", True)), - dict(revision_range="a..", stdin_mode=False, expect=("a", ":WORKTREE:", False)), - dict(revision_range="a...", stdin_mode=False, expect=("a", ":WORKTREE:", True)), - dict(revision_range="..HEAD", stdin_mode=False, expect=("HEAD", "HEAD", False)), - dict(revision_range="...HEAD", stdin_mode=False, expect=("HEAD", "HEAD", True)), - dict(revision_range="a..HEAD", stdin_mode=False, expect=("a", "HEAD", False)), - dict(revision_range="a...HEAD", stdin_mode=False, expect=("a", "HEAD", True)), - dict(revision_range="a..b", stdin_mode=False, expect=("a", "b", False)), - dict(revision_range="a...b", stdin_mode=False, expect=("a", "b", True)), - dict(revision_range="", stdin_mode=True, expect=("HEAD", ":STDIN:", False)), - dict(revision_range="HEAD", stdin_mode=True, expect=("HEAD", ":STDIN:", False)), - dict(revision_range="a", stdin_mode=True, expect=("a", ":STDIN:", True)), - dict(revision_range="a..", stdin_mode=True, expect=("a", ":STDIN:", False)), - dict(revision_range="a...", stdin_mode=True, expect=("a", ":STDIN:", True)), - dict( - revision_range="..HEAD", - stdin_mode=True, - expect=ValueError( - "With --stdin-filename, rev2 in ..HEAD must be ':STDIN:', not 'HEAD'" - ), - ), - dict( - revision_range="...HEAD", - stdin_mode=True, - expect=ValueError( - "With --stdin-filename, rev2 in ...HEAD must be ':STDIN:', not 'HEAD'" - ), - ), - dict( - revision_range="a..HEAD", - stdin_mode=True, - expect=ValueError( - "With --stdin-filename, rev2 in a..HEAD must be ':STDIN:', not 'HEAD'" - ), - ), - dict( - revision_range="a...HEAD", - stdin_mode=True, - expect=ValueError( - "With --stdin-filename, rev2 in a...HEAD must be ':STDIN:', not 'HEAD'" - ), - ), - dict( - revision_range="a..b", - stdin_mode=True, - expect=ValueError( - "With --stdin-filename, rev2 in a..b must be ':STDIN:', not 'b'" - ), - ), - dict( - revision_range="a...b", - stdin_mode=True, - expect=ValueError( - "With --stdin-filename, rev2 in a...b must be ':STDIN:', not 'b'" - ), - ), -) -def test_revisionrange_parse(revision_range, stdin_mode, expect): - """Test for :meth:`RevisionRange.parse`""" - with raises_or_matches(expect, ["args"]) as check: - - check(git.RevisionRange._parse(revision_range, stdin_mode)) - - def git_call(cmd, encoding=None): """Returns a mocked call to git""" return call( @@ -250,59 +184,6 @@ def test_git_get_content_at_revision_obtain_file_content( assert text_document_class.method_calls == expect_textdocument_calls -@pytest.mark.kwparametrize( - dict(revrange="HEAD", stdin_mode=False, expect="HEAD..:WORKTREE:"), - dict(revrange="{initial}", stdin_mode=False, expect="{initial}..:WORKTREE:"), - dict(revrange="{initial}..", stdin_mode=False, expect="{initial}..:WORKTREE:"), - dict(revrange="{initial}..HEAD", stdin_mode=False, expect="{initial}..HEAD"), - dict(revrange="{initial}..feature", stdin_mode=False, expect="{initial}..feature"), - dict(revrange="{initial}...", stdin_mode=False, expect="{initial}..:WORKTREE:"), - dict(revrange="{initial}...HEAD", stdin_mode=False, expect="{initial}..HEAD"), - dict(revrange="{initial}...feature", stdin_mode=False, expect="{initial}..feature"), - dict(revrange="master", stdin_mode=False, expect="{initial}..:WORKTREE:"), - dict(revrange="master..", stdin_mode=False, expect="master..:WORKTREE:"), - dict(revrange="master..HEAD", stdin_mode=False, expect="master..HEAD"), - dict(revrange="master..feature", stdin_mode=False, expect="master..feature"), - dict(revrange="master...", stdin_mode=False, expect="{initial}..:WORKTREE:"), - dict(revrange="master...HEAD", stdin_mode=False, expect="{initial}..HEAD"), - dict(revrange="master...feature", stdin_mode=False, expect="{initial}..feature"), - dict(revrange="HEAD", stdin_mode=True, expect="HEAD..:STDIN:"), - dict(revrange="{initial}", stdin_mode=True, expect="{initial}..:STDIN:"), - dict(revrange="{initial}..", stdin_mode=True, expect="{initial}..:STDIN:"), - dict(revrange="{initial}..HEAD", stdin_mode=True, expect=ValueError), - dict(revrange="{initial}..feature", stdin_mode=True, expect=ValueError), - dict(revrange="{initial}...", stdin_mode=True, expect="{initial}..:STDIN:"), - dict(revrange="{initial}...HEAD", stdin_mode=True, expect=ValueError), - dict(revrange="{initial}...feature", stdin_mode=True, expect=ValueError), - dict(revrange="master", stdin_mode=True, expect="{initial}..:STDIN:"), - dict(revrange="master..", stdin_mode=True, expect="master..:STDIN:"), - dict(revrange="master..HEAD", stdin_mode=True, expect=ValueError), - dict(revrange="master..feature", stdin_mode=True, expect=ValueError), - dict(revrange="master...", stdin_mode=True, expect="{initial}..:STDIN:"), - dict(revrange="master...HEAD", stdin_mode=True, expect=ValueError), - dict(revrange="master...feature", stdin_mode=True, expect=ValueError), -) -def test_revisionrange_parse_with_common_ancestor( - git_repo, revrange, stdin_mode, expect -): - """``_git_get_old_revision()`` gets common ancestor using Git when necessary""" - git_repo.add({"a": "i"}, commit="Initial commit") - initial = git_repo.get_hash() - git_repo.add({"a": "m"}, commit="in master") - master = git_repo.get_hash() - git_repo.create_branch("feature", initial) - git_repo.add({"a": "f"}, commit="in feature") - with raises_if_exception(expect): - - result = git.RevisionRange.parse_with_common_ancestor( - revrange.format(initial=initial), git_repo.root, stdin_mode - ) - - rev1, rev2 = expect.format(initial=initial, master=master).split("..") - assert result.rev1 == rev1 - assert result.rev2 == rev2 - - @pytest.mark.kwparametrize( dict(path="file.py", expect="file.py"), dict(path="subdir/file.py", expect="subdir/file.py"), @@ -686,7 +567,7 @@ def test_git_get_modified_python_files(git_repo, modify_paths, paths, expect): else: absolute_path.parent.mkdir(parents=True, exist_ok=True) absolute_path.write_bytes(content.encode("ascii")) - revrange = git.RevisionRange("HEAD", ":WORKTREE:") + revrange = RevisionRange("HEAD", ":WORKTREE:") result = git.git_get_modified_python_files( {root / p for p in paths}, revrange, cwd=root @@ -843,7 +724,7 @@ def test_git_get_modified_python_files_revision_range( """Test for :func:`darker.git.git_get_modified_python_files` with revision range""" result = git.git_get_modified_python_files( [Path(branched_repo.root)], - git.RevisionRange.parse_with_common_ancestor( + RevisionRange.parse_with_common_ancestor( revrange, branched_repo.root, stdin_mode=False ), Path(branched_repo.root), @@ -954,70 +835,6 @@ def test_git_get_root_not_found(tmp_path, path): assert root is None -@pytest.mark.kwparametrize( - dict( - expect_rev1="HEAD", - expect_rev2=":WORKTREE:", - ), - dict( - environ={"PRE_COMMIT_FROM_REF": "old"}, - expect_rev1="HEAD", - expect_rev2=":WORKTREE:", - ), - dict( - environ={"PRE_COMMIT_TO_REF": "new"}, - expect_rev1="HEAD", - expect_rev2=":WORKTREE:", - ), - dict( - environ={"PRE_COMMIT_FROM_REF": "old", "PRE_COMMIT_TO_REF": "new"}, - expect_rev1="old", - expect_rev2="new", - expect_use_common_ancestor=True, - ), - dict( - stdin_mode=True, - expect_rev1=ValueError( - "With --stdin-filename, revision ':PRE-COMMIT:' is not allowed" - ), - ), - dict( - environ={"PRE_COMMIT_FROM_REF": "old"}, - stdin_mode=True, - expect_rev1=ValueError( - "With --stdin-filename, revision ':PRE-COMMIT:' is not allowed" - ), - ), - dict( - environ={"PRE_COMMIT_TO_REF": "new"}, - stdin_mode=True, - expect_rev1=ValueError( - "With --stdin-filename, revision ':PRE-COMMIT:' is not allowed" - ), - ), - dict( - environ={"PRE_COMMIT_FROM_REF": "old", "PRE_COMMIT_TO_REF": "new"}, - stdin_mode=True, - expect_rev1=ValueError( - "With --stdin-filename, revision ':PRE-COMMIT:' is not allowed" - ), - ), - environ={}, - stdin_mode=False, - expect_rev2=None, - expect_use_common_ancestor=False, -) -def test_revisionrange_parse_pre_commit( - environ, stdin_mode, expect_rev1, expect_rev2, expect_use_common_ancestor -): - """RevisionRange._parse(':PRE-COMMIT:') gets the range from environment variables""" - with patch.dict(os.environ, environ), raises_if_exception(expect_rev1): - - result = git.RevisionRange._parse(":PRE-COMMIT:", stdin_mode) - - assert result == (expect_rev1, expect_rev2, expect_use_common_ancestor) - - edited_linenums_differ_cases = pytest.mark.kwparametrize( dict(context_lines=0, expect=[3, 7]), dict(context_lines=1, expect=[2, 3, 4, 6, 7, 8]), @@ -1031,7 +848,7 @@ def test_edited_linenums_differ_compare_revisions(git_repo, context_lines, expec """Tests for EditedLinenumsDiffer.revision_vs_worktree()""" paths = git_repo.add({"a.py": "1\n2\n3\n4\n5\n6\n7\n8\n"}, commit="Initial commit") paths["a.py"].write_bytes(b"1\n2\nthree\n4\n5\n6\nseven\n8\n") - revrange = git.RevisionRange("HEAD", ":WORKTREE:") + revrange = RevisionRange("HEAD", ":WORKTREE:") differ = git.EditedLinenumsDiffer(git_repo.root, revrange) linenums = differ.compare_revisions(Path("a.py"), context_lines) @@ -1044,7 +861,7 @@ def test_edited_linenums_differ_revision_vs_lines(git_repo, context_lines, expec """Tests for EditedLinenumsDiffer.revision_vs_lines()""" git_repo.add({"a.py": "1\n2\n3\n4\n5\n6\n7\n8\n"}, commit="Initial commit") content = TextDocument.from_lines(["1", "2", "three", "4", "5", "6", "seven", "8"]) - revrange = git.RevisionRange("HEAD", ":WORKTREE:") + revrange = RevisionRange("HEAD", ":WORKTREE:") differ = git.EditedLinenumsDiffer(git_repo.root, revrange) linenums = differ.revision_vs_lines(Path("a.py"), content, context_lines) @@ -1089,7 +906,7 @@ def test_edited_linenums_differ_revision_vs_lines_multiline_strings( "CHANGED", ] ) - revrange = git.RevisionRange("HEAD", ":WORKTREE:") + revrange = RevisionRange("HEAD", ":WORKTREE:") differ = git.EditedLinenumsDiffer(git_repo.root, revrange) linenums = differ.revision_vs_lines(Path("a.py"), content, context_lines) diff --git a/src/darker/tests/test_import_sorting.py b/src/darker/tests/test_import_sorting.py index 13b0ec36f..dc9390583 100644 --- a/src/darker/tests/test_import_sorting.py +++ b/src/darker/tests/test_import_sorting.py @@ -9,9 +9,10 @@ import pytest import darker.import_sorting -from darker.git import EditedLinenumsDiffer, RevisionRange +from darker.git import EditedLinenumsDiffer from darker.tests.helpers import isort_present from darker.utils import TextDocument, joinlines +from darkgraylib.git import RevisionRange ORIGINAL_SOURCE = ("import sys", "import os", "", "print(42)") ISORTED_SOURCE = ("import os", "import sys", "", "print(42)") diff --git a/src/darker/tests/test_main.py b/src/darker/tests/test_main.py index 7ccee1c8e..35a9a94b4 100644 --- a/src/darker/tests/test_main.py +++ b/src/darker/tests/test_main.py @@ -21,12 +21,13 @@ import darker.import_sorting from darker.config import Exclusions from darker.exceptions import MissingPackageError -from darker.git import WORKTREE, EditedLinenumsDiffer, RevisionRange +from darker.git import WORKTREE, EditedLinenumsDiffer from darker.tests.helpers import isort_present from darker.tests.test_fstring import FLYNTED_SOURCE, MODIFIED_SOURCE, ORIGINAL_SOURCE from darker.tests.test_highlighting import BLUE, CYAN, RESET, WHITE, YELLOW from darker.utils import TextDocument, joinlines from darker.verification import NotEquivalentError +from darkgraylib.git import RevisionRange def randomword(length: int) -> str: diff --git a/src/darker/tests/test_main_blacken_and_flynt_single_file.py b/src/darker/tests/test_main_blacken_and_flynt_single_file.py index 8e620d275..faf4f78bc 100644 --- a/src/darker/tests/test_main_blacken_and_flynt_single_file.py +++ b/src/darker/tests/test_main_blacken_and_flynt_single_file.py @@ -9,8 +9,9 @@ from darker.__main__ import _blacken_and_flynt_single_file from darker.config import Exclusions -from darker.git import EditedLinenumsDiffer, RevisionRange +from darker.git import EditedLinenumsDiffer from darker.utils import TextDocument +from darkgraylib.git import RevisionRange @pytest.mark.kwparametrize( From 77af0de837cd3a53c46c5356d5b077f263e860d2 Mon Sep 17 00:00:00 2001 From: Antti Kaihola <13725+akaihola@users.noreply.github.com> Date: Sat, 25 Mar 2023 10:35:06 +0200 Subject: [PATCH 07/30] Bring `--output` validation back --- src/darker/__main__.py | 16 +- src/darker/command_line.py | 29 ++- src/darker/tests/test_command_line.py | 242 +------------------ src/darker/tests/test_config.py | 28 --- src/darker/tests/test_main_stdin_filename.py | 2 +- 5 files changed, 34 insertions(+), 283 deletions(-) diff --git a/src/darker/__main__.py b/src/darker/__main__.py index 62f82add5..8998f64f1 100644 --- a/src/darker/__main__.py +++ b/src/darker/__main__.py @@ -18,9 +18,9 @@ run_black, ) from darker.chooser import choose_lines -from darker.command_line import make_argument_parser +from darker.command_line import parse_command_line from darker.concurrency import get_executor -from darker.config import Exclusions, OutputMode, dump_config, validate_config_output_mode +from darker.config import Exclusions, OutputMode, validate_config_output_mode from darker.diff import diff_chunks from darker.exceptions import DependencyError, MissingPackageError from darker.fstring import apply_flynt, flynt @@ -38,16 +38,8 @@ from darker.help import get_extra_instruction from darker.highlighting import colorize, should_use_color from darker.import_sorting import apply_isort, isort -from darker.utils import ( - GIT_DATEFORMAT, - DiffChunk, - TextDocument, - debug_dump, - get_common_root, - glob_any, -) +from darker.utils import GIT_DATEFORMAT, DiffChunk, TextDocument, debug_dump, glob_any from darker.verification import ASTVerifier, BinarySearch, NotEquivalentError -from darkgraylib.command_line import parse_command_line from darkgraylib.config import show_config_if_debug from darkgraylib.git import RevisionRange from darkgraylib.log import setup_logging @@ -484,7 +476,7 @@ def main( # pylint: disable=too-many-locals,too-many-branches,too-many-statemen should be) reformatted; 0 otherwise. """ - args, config, config_nondefault = parse_command_line(make_argument_parser, argv) + args, config, config_nondefault = parse_command_line(argv) # Make sure there aren't invalid option combinations after merging configuration and # command line options. diff --git a/src/darker/command_line.py b/src/darker/command_line.py index 709f84949..04dd17078 100644 --- a/src/darker/command_line.py +++ b/src/darker/command_line.py @@ -1,12 +1,14 @@ """Command line parsing for the ``darker`` binary""" -from argparse import ArgumentParser +from argparse import ArgumentParser, Namespace from functools import partial +from typing import List, Optional, Tuple from black import TargetVersion import darkgraylib.command_line from darker import help as hlp +from darker.config import DarkerConfig, OutputMode from darkgraylib.command_line import add_parser_argument from graylint.command_line import add_lint_arg @@ -74,3 +76,28 @@ def make_argument_parser(require_src: bool) -> ArgumentParser: choices=[v.name.lower() for v in TargetVersion], ) return parser + + +def parse_command_line( + argv: Optional[List[str]], +) -> Tuple[Namespace, DarkerConfig, DarkerConfig]: + """Return the parsed command line, using defaults from a configuration file + + Also return the effective configuration which combines defaults, the configuration + read from ``pyproject.toml`` (or the path given in ``--config``), environment + variables, and command line arguments. + + Finally, also return the set of configuration options which differ from defaults. + + :param argv: Command line arguments to parse (excluding the path of the script). If + ``None``, use ``sys.argv``. + :return: A tuple of the parsed command line, the effective configuration, and the + set of modified configuration options from the defaults. + + """ + args, effective_cfg, modified_cfg = darkgraylib.command_line.parse_command_line( + make_argument_parser, argv, "darker", DarkerConfig + ) + OutputMode.validate_diff_stdout(args.diff, args.stdout) + OutputMode.validate_stdout_src(args.stdout, args.src, args.stdin_filename) + return args, effective_cfg, modified_cfg diff --git a/src/darker/tests/test_command_line.py b/src/darker/tests/test_command_line.py index cd655e615..f2688d58b 100644 --- a/src/darker/tests/test_command_line.py +++ b/src/darker/tests/test_command_line.py @@ -11,7 +11,6 @@ from unittest.mock import DEFAULT, Mock, call, patch import pytest -import toml from black import TargetVersion import darker.help @@ -19,12 +18,7 @@ from darker.__main__ import main from darker.command_line import make_argument_parser, parse_command_line from darker.config import ConfigurationError, Exclusions -from darker.tests.helpers import ( - filter_dict, - flynt_present, - isort_present, - raises_if_exception, -) +from darker.tests.helpers import flynt_present, isort_present, raises_if_exception from darker.utils import TextDocument, joinlines from darkgraylib.git import RevisionRange @@ -54,104 +48,7 @@ def get_darker_help_output(capsys): return re.sub(r"\s+", " ", capsys.readouterr().out) -@pytest.mark.kwparametrize( - dict(config=None, argv=[], expect=SystemExit), - dict( - config=None, - argv=["file.py"], - expect={"src": ["file.py"]}, - ), - dict( - config={"src": ["file.py"]}, - argv=[], - expect={"src": ["file.py"]}, - ), - dict( - config={"src": ["file.py"]}, - argv=["file.py"], - expect={"src": ["file.py"]}, - ), - dict( - config={"src": ["file1.py"]}, - argv=["file2.py"], - expect={"src": ["file2.py"]}, - ), -) -def test_parse_command_line_config_src( - tmpdir, - monkeypatch, - config, - argv, - expect, -): - """The ``src`` positional argument from config and cmdline is handled correctly""" - monkeypatch.chdir(tmpdir) - if config is not None: - toml.dump({"tool": {"darker": config}}, tmpdir / "pyproject.toml") - with raises_if_exception(expect): - - args, effective_cfg, modified_cfg = parse_command_line(argv) - - assert filter_dict(args.__dict__, "src") == expect - assert filter_dict(dict(effective_cfg), "src") == expect - assert filter_dict(dict(modified_cfg), "src") == expect - - -@pytest.mark.kwparametrize( - dict(argv=["."], expect="pylint"), - dict(argv=["./subdir/"], expect="flake8"), - dict(argv=["--config", "./pyproject.toml", "."], expect="pylint"), - dict(argv=["--config", "./subdir/pyproject.toml", "."], expect="flake8"), - dict(argv=["--config", "./pyproject.toml", "subdir/"], expect="pylint"), - dict(argv=["--config", "./subdir/pyproject.toml", "subdir/"], expect="flake8"), -) -def test_parse_command_line_config_location_specified( - tmp_path, - monkeypatch, - argv, - expect, -): - """Darker configuration is read from file pointed to using ``-c``/``--config``""" - monkeypatch.chdir(tmp_path) - subdir = tmp_path / "subdir" - subdir.mkdir() - root_config = tmp_path / "pyproject.toml" - subdir_config = subdir / "pyproject.toml" - root_config.write_text(toml.dumps({"tool": {"darker": {"lint": "pylint"}}})) - subdir_config.write_text(toml.dumps({"tool": {"darker": {"lint": "flake8"}}})) - - args, effective_cfg, modified_cfg = parse_command_line(argv) - - assert args.lint == expect - assert effective_cfg["lint"] == expect - assert modified_cfg["lint"] == expect - - @pytest.mark.kwparametrize( - dict( - argv=["."], - expect_value=("src", ["."]), - expect_config=("src", ["."]), - expect_modified=("src", ["."]), - ), - dict( - argv=["."], - expect_value=("revision", "HEAD"), - expect_config=("revision", "HEAD"), - expect_modified=("revision", ...), - ), - dict( - argv=["-rmaster", "."], - expect_value=("revision", "master"), - expect_config=("revision", "master"), - expect_modified=("revision", "master"), - ), - dict( - argv=["--revision", "HEAD", "."], - expect_value=("revision", "HEAD"), - expect_config=("revision", "HEAD"), - expect_modified=("revision", ...), - ), dict( argv=["."], expect_value=("diff", False), @@ -230,129 +127,6 @@ def test_parse_command_line_config_location_specified( expect_config=("lint", ["flake8", "mypy"]), expect_modified=("lint", ["flake8", "mypy"]), ), - dict( - argv=["."], - expect_value=("config", None), - expect_config=("config", None), - expect_modified=("config", ...), - ), - dict( - argv=["-c", "my.cfg", "."], - expect_value=("config", "my.cfg"), - expect_config=("config", "my.cfg"), - expect_modified=("config", "my.cfg"), - ), - dict( - argv=["--config=my.cfg", "."], - expect_value=("config", "my.cfg"), - expect_config=("config", "my.cfg"), - expect_modified=("config", "my.cfg"), - ), - dict( - argv=["-c", "subdir_with_config", "."], - expect_value=("config", "subdir_with_config"), - expect_config=("config", "subdir_with_config"), - expect_modified=("config", "subdir_with_config"), - ), - dict( - argv=["--config=subdir_with_config", "."], - expect_value=("config", "subdir_with_config"), - expect_config=("config", "subdir_with_config"), - expect_modified=("config", "subdir_with_config"), - ), - dict( - argv=["."], - expect_value=("log_level", 30), - expect_config=("log_level", "WARNING"), - expect_modified=("log_level", ...), - ), - dict( - argv=["-v", "."], - expect_value=("log_level", 20), - expect_config=("log_level", "INFO"), - expect_modified=("log_level", "INFO"), - ), - dict( - argv=["--verbose", "-v", "."], - expect_value=("log_level", 10), - expect_config=("log_level", "DEBUG"), - expect_modified=("log_level", "DEBUG"), - ), - dict( - argv=["-q", "."], - expect_value=("log_level", 40), - expect_config=("log_level", "ERROR"), - expect_modified=("log_level", "ERROR"), - ), - dict( - argv=["--quiet", "-q", "."], - expect_value=("log_level", 50), - expect_config=("log_level", "CRITICAL"), - expect_modified=("log_level", "CRITICAL"), - ), - dict( - argv=["."], - environ={}, - expect_value=("color", None), - expect_config=("color", None), - expect_modified=("color", ...), - ), - dict( - argv=["."], - environ={"PY_COLORS": "0"}, - expect_value=("color", False), - expect_config=("color", False), - expect_modified=("color", False), - ), - dict( - argv=["."], - environ={"PY_COLORS": "1"}, - expect_value=("color", True), - expect_config=("color", True), - expect_modified=("color", True), - ), - dict( - argv=["--color", "."], - environ={}, - expect_value=("color", True), - expect_config=("color", True), - expect_modified=("color", True), - ), - dict( - argv=["--color", "."], - environ={"PY_COLORS": "0"}, - expect_value=("color", True), - expect_config=("color", True), - expect_modified=("color", True), - ), - dict( - argv=["--color", "."], - environ={"PY_COLORS": "1"}, - expect_value=("color", True), - expect_config=("color", True), - expect_modified=("color", True), - ), - dict( - argv=["--no-color", "."], - environ={}, - expect_value=("color", False), - expect_config=("color", False), - expect_modified=("color", False), - ), - dict( - argv=["--no-color", "."], - environ={"PY_COLORS": "0"}, - expect_value=("color", False), - expect_config=("color", False), - expect_modified=("color", False), - ), - dict( - argv=["--no-color", "."], - environ={"PY_COLORS": "1"}, - expect_value=("color", False), - expect_config=("color", False), - expect_modified=("color", False), - ), dict( argv=["."], expect_value=("skip_string_normalization", None), @@ -431,20 +205,6 @@ def test_parse_command_line_config_location_specified( expect_config=None, expect_modified=None, ), - dict( - # this is accepted as a path, but would later fail if a file or directory with - # that funky name doesn't exist - argv=["--suspicious path"], - expect_value=("src", ["--suspicious path"]), - expect_config=("src", ["--suspicious path"]), - expect_modified=("src", ["--suspicious path"]), - ), - dict( - argv=["valid/path", "another/valid/path"], - expect_value=("src", ["valid/path", "another/valid/path"]), - expect_config=("src", ["valid/path", "another/valid/path"]), - expect_modified=("src", ["valid/path", "another/valid/path"]), - ), environ={}, ) def test_parse_command_line( diff --git a/src/darker/tests/test_config.py b/src/darker/tests/test_config.py index 0713ef42b..ef99c6382 100644 --- a/src/darker/tests/test_config.py +++ b/src/darker/tests/test_config.py @@ -12,14 +12,12 @@ from darker.config import ( ConfigurationError, - DarkerConfig, OutputMode, TomlArrayLinesEncoder, dump_config, get_effective_config, get_modified_config, load_config, - replace_log_level_name, ) from darker.tests.helpers import raises_if_exception @@ -57,32 +55,6 @@ def test_toml_array_lines_encoder(list_value, expect): assert result == expect -@pytest.mark.kwparametrize( - dict(log_level=None, expect={}), - dict(log_level=0, expect={"log_level": "NOTSET"}), - dict(log_level=10, expect={"log_level": "DEBUG"}), - dict(log_level=20, expect={"log_level": "INFO"}), - dict(log_level=30, expect={"log_level": "WARNING"}), - dict(log_level=40, expect={"log_level": "ERROR"}), - dict(log_level=50, expect={"log_level": "CRITICAL"}), - dict(log_level="DEBUG", expect={"log_level": 10}), - dict(log_level="INFO", expect={"log_level": 20}), - dict(log_level="WARNING", expect={"log_level": 30}), - dict(log_level="WARN", expect={"log_level": 30}), - dict(log_level="ERROR", expect={"log_level": 40}), - dict(log_level="CRITICAL", expect={"log_level": 50}), - dict(log_level="FOOBAR", expect={"log_level": "Level FOOBAR"}), -) -def test_replace_log_level_name(log_level, expect): - """``replace_log_level_name()`` converts between log level names and numbers""" - config = DarkerConfig() if log_level is None else DarkerConfig(log_level=log_level) - - replace_log_level_name(config) - - result = {k: v for k, v in config.items() if k == "log_level"} - assert result == expect - - @pytest.mark.kwparametrize( dict(diff=False, stdout=False, expect=None), dict(diff=False, stdout=True, expect=None), diff --git a/src/darker/tests/test_main_stdin_filename.py b/src/darker/tests/test_main_stdin_filename.py index d6dbd5f4a..76446a54d 100644 --- a/src/darker/tests/test_main_stdin_filename.py +++ b/src/darker/tests/test_main_stdin_filename.py @@ -10,9 +10,9 @@ import toml import darker.__main__ -from darker.config import ConfigurationError from darker.tests.conftest import GitRepoFixture from darker.tests.helpers import raises_if_exception +from darkgraylib.config import ConfigurationError pytestmark = pytest.mark.usefixtures("find_project_root_cache_clear") From 3124fc4514f8b144802026d0dc582297c13b326b Mon Sep 17 00:00:00 2001 From: Antti Kaihola <13725+akaihola@users.noreply.github.com> Date: Sat, 25 Mar 2023 10:35:49 +0200 Subject: [PATCH 08/30] Update to `lint-action@v2.3.0` --- .github/workflows/isort.yml | 2 +- .github/workflows/mypy.yml | 2 +- .github/workflows/pylint.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/isort.yml b/.github/workflows/isort.yml index b6e7961a2..32c156289 100644 --- a/.github/workflows/isort.yml +++ b/.github/workflows/isort.yml @@ -11,7 +11,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 - run: pip install 'isort>=5.0.1' - - uses: wearerequired/lint-action@v2.1.0 + - uses: wearerequired/lint-action@v2.3.0 with: github_token: ${{ secrets.GITHUB_TOKEN }} isort: true diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index 62f2480ee..382304326 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -19,7 +19,7 @@ jobs: pytest \ types-requests \ types-toml - - uses: wearerequired/lint-action@v2.1.0 + - uses: wearerequired/lint-action@v2.3.0 with: github_token: ${{ secrets.GITHUB_TOKEN }} mypy: true diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index 4587c1a3f..8e6338a8f 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -26,7 +26,7 @@ jobs: ruamel.yaml \ toml pip list - - uses: wearerequired/lint-action@v2.1.0 + - uses: wearerequired/lint-action@v2.3.0 with: github_token: ${{ secrets.GITHUB_TOKEN }} pylint: true From 66738911d4743f780af46c817b8bb9a92de99826 Mon Sep 17 00:00:00 2001 From: Antti Kaihola <13725+akaihola@users.noreply.github.com> Date: Sat, 25 Mar 2023 10:36:00 +0200 Subject: [PATCH 09/30] Mark the Darker package as typed --- src/darker/py.typed | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 src/darker/py.typed diff --git a/src/darker/py.typed b/src/darker/py.typed new file mode 100644 index 000000000..e69de29bb From 0cccd56078360208df2603238bda6fc2f3dcd644 Mon Sep 17 00:00:00 2001 From: Antti Kaihola <13725+akaihola@users.noreply.github.com> Date: Sun, 9 Jul 2023 22:01:08 +0300 Subject: [PATCH 10/30] Add `darkgraylib` and `graylint` as dependencies --- setup.cfg | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.cfg b/setup.cfg index e17d91bc4..8a4235c11 100644 --- a/setup.cfg +++ b/setup.cfg @@ -28,6 +28,8 @@ packages = find: install_requires = # NOTE: remember to keep `constraints-oldest.txt` in sync with these black>=21.5b1,<24.2 # upper limit until incompatibility fixed + darkgraylib @ git+https://github.com/akaihola/darkgraylib.git@main + graylint @ git+https://github.com/akaihola/graylint.git@main toml>=0.10.0 # NOTE: remember to keep `.github/workflows/python-package.yml` in sync # with the minimum required Python version From 20a67cb6a0cda89cc5e9c33cedb5b6dc727509ec Mon Sep 17 00:00:00 2001 From: Antti Kaihola <13725+akaihola@users.noreply.github.com> Date: Sun, 9 Jul 2023 22:08:26 +0300 Subject: [PATCH 11/30] Add `darkgraylib` and `graylint` for Pylint & Mypy --- .github/workflows/mypy.yml | 2 ++ .github/workflows/pylint.yml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index 382304326..07600d6e3 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -13,7 +13,9 @@ jobs: - run: | pip install -U \ black \ + git+https://github.com/akaihola/darkgraylib.git@main \ flynt \ + git+https://github.com/akaihola/graylint.git@main \ isort \ mypy>=0.990 \ pytest \ diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index 8e6338a8f..360275474 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -15,7 +15,9 @@ jobs: pip install -U \ airium \ black \ + git+https://github.com/akaihola/darkgraylib.git@main \ defusedxml \ + git+https://github.com/akaihola/graylint.git@main \ pip-requirements-parser \ pygments \ pylint \ From c91fd654d2d5d9014a15945ccf8a3d21b77f4ad0 Mon Sep 17 00:00:00 2001 From: Antti Kaihola <13725+akaihola@users.noreply.github.com> Date: Sun, 9 Jul 2023 22:36:11 +0300 Subject: [PATCH 12/30] Consider `darkgraylib` and `graylint` first party Affects import ordering done by isort. --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 4a64ebeb5..1429ef464 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,6 +8,7 @@ target-version = ["py311"] [tool.isort] profile = "black" +known_first_party = ["darkgraylib", "graylint"] known_third_party = ["pytest"] [tool.darker] From 095c769cd8161493a570e3c57784ba7621204f73 Mon Sep 17 00:00:00 2001 From: Antti Kaihola <13725+akaihola@users.noreply.github.com> Date: Sun, 9 Jul 2023 22:58:55 +0300 Subject: [PATCH 13/30] Don't use Pylint to check import order --- pyproject.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 1429ef464..fc691d28b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,3 +17,8 @@ src = [ "src", ] revision = "origin/master..." + +[tool.pylint."messages control"] +# Check import order only with isort. Pylint doesn't support a custom list of first +# party packages. We want to consider "darkgraylib" and "graylint" as first party. +disable = ["wrong-import-order"] From b4752fee090bdffea7fb54ef9d769b4653fdc1a9 Mon Sep 17 00:00:00 2001 From: Antti Kaihola <13725+akaihola@users.noreply.github.com> Date: Sun, 9 Jul 2023 23:25:49 +0300 Subject: [PATCH 14/30] Move `TextDocument` class to `darkgraylib` --- src/darker/__main__.py | 3 +- src/darker/black_diff.py | 2 +- src/darker/diff.py | 3 +- src/darker/fstring.py | 2 +- src/darker/git.py | 3 +- src/darker/import_sorting.py | 3 +- src/darker/multiline_strings.py | 2 +- src/darker/tests/test_black_diff.py | 2 +- src/darker/tests/test_command_line.py | 3 +- src/darker/tests/test_diff.py | 2 +- src/darker/tests/test_fstring.py | 3 +- src/darker/tests/test_git.py | 3 +- src/darker/tests/test_import_sorting.py | 3 +- src/darker/tests/test_main.py | 3 +- ...test_main_blacken_and_flynt_single_file.py | 2 +- src/darker/tests/test_multiline_strings.py | 2 +- src/darker/tests/test_utils.py | 2 +- src/darker/tests/test_verification.py | 3 +- src/darker/utils.py | 154 ------------------ src/darker/verification.py | 3 +- 20 files changed, 30 insertions(+), 173 deletions(-) diff --git a/src/darker/__main__.py b/src/darker/__main__.py index 8998f64f1..c6d036586 100644 --- a/src/darker/__main__.py +++ b/src/darker/__main__.py @@ -38,12 +38,13 @@ from darker.help import get_extra_instruction from darker.highlighting import colorize, should_use_color from darker.import_sorting import apply_isort, isort -from darker.utils import GIT_DATEFORMAT, DiffChunk, TextDocument, debug_dump, glob_any +from darker.utils import GIT_DATEFORMAT, DiffChunk, debug_dump, glob_any from darker.verification import ASTVerifier, BinarySearch, NotEquivalentError from darkgraylib.config import show_config_if_debug from darkgraylib.git import RevisionRange from darkgraylib.log import setup_logging from darkgraylib.main import resolve_paths +from darkgraylib.utils import TextDocument from graylint.linting import run_linters logger = logging.getLogger(__name__) diff --git a/src/darker/black_diff.py b/src/darker/black_diff.py index 4fc42963c..62b91c493 100644 --- a/src/darker/black_diff.py +++ b/src/darker/black_diff.py @@ -54,7 +54,7 @@ from black.report import Report from darker.config import ConfigurationError -from darker.utils import TextDocument +from darkgraylib.utils import TextDocument __all__ = ["BlackConfig", "Mode", "run_black"] diff --git a/src/darker/diff.py b/src/darker/diff.py index 46343b10b..7e5e96e01 100644 --- a/src/darker/diff.py +++ b/src/darker/diff.py @@ -69,7 +69,8 @@ from typing import Dict, Generator, List, Sequence, Tuple from darker.multiline_strings import find_overlap -from darker.utils import DiffChunk, TextDocument +from darker.utils import DiffChunk +from darkgraylib.utils import TextDocument logger = logging.getLogger(__name__) diff --git a/src/darker/fstring.py b/src/darker/fstring.py index ab6b56122..233500a2a 100644 --- a/src/darker/fstring.py +++ b/src/darker/fstring.py @@ -6,7 +6,7 @@ from darker.exceptions import MissingPackageError from darker.git import EditedLinenumsDiffer -from darker.utils import TextDocument +from darkgraylib.utils import TextDocument try: import flynt diff --git a/src/darker/git.py b/src/darker/git.py index 6d1dc743b..d4512f3d2 100644 --- a/src/darker/git.py +++ b/src/darker/git.py @@ -27,8 +27,9 @@ from darker.diff import diff_and_get_opcodes, opcodes_to_edit_linenums from darker.multiline_strings import get_multiline_string_ranges -from darker.utils import GIT_DATEFORMAT, TextDocument +from darker.utils import GIT_DATEFORMAT from darkgraylib.git import RevisionRange +from darkgraylib.utils import TextDocument logger = logging.getLogger(__name__) diff --git a/src/darker/import_sorting.py b/src/darker/import_sorting.py index 95d3f7867..72584d316 100644 --- a/src/darker/import_sorting.py +++ b/src/darker/import_sorting.py @@ -8,7 +8,8 @@ from darker.diff import diff_chunks from darker.exceptions import IncompatiblePackageError, MissingPackageError from darker.git import EditedLinenumsDiffer -from darker.utils import DiffChunk, TextDocument, glob_any +from darker.utils import DiffChunk, glob_any +from darkgraylib.utils import TextDocument try: import isort diff --git a/src/darker/multiline_strings.py b/src/darker/multiline_strings.py index 62af0131b..068f5531b 100644 --- a/src/darker/multiline_strings.py +++ b/src/darker/multiline_strings.py @@ -3,7 +3,7 @@ from tokenize import STRING, tokenize from typing import Generator, Optional, Sequence, Tuple -from darker.utils import TextDocument +from darkgraylib.utils import TextDocument if sys.version_info >= (3, 12): from tokenize import FSTRING_END, FSTRING_START diff --git a/src/darker/tests/test_black_diff.py b/src/darker/tests/test_black_diff.py index 6415605e5..1f110eb7d 100644 --- a/src/darker/tests/test_black_diff.py +++ b/src/darker/tests/test_black_diff.py @@ -24,7 +24,7 @@ ) from darker.config import ConfigurationError from darker.tests.helpers import raises_or_matches -from darker.utils import TextDocument +from darkgraylib.utils import TextDocument if sys.version_info >= (3, 11): try: diff --git a/src/darker/tests/test_command_line.py b/src/darker/tests/test_command_line.py index f2688d58b..3c5b28461 100644 --- a/src/darker/tests/test_command_line.py +++ b/src/darker/tests/test_command_line.py @@ -19,8 +19,9 @@ from darker.command_line import make_argument_parser, parse_command_line from darker.config import ConfigurationError, Exclusions from darker.tests.helpers import flynt_present, isort_present, raises_if_exception -from darker.utils import TextDocument, joinlines +from darker.utils import joinlines from darkgraylib.git import RevisionRange +from darkgraylib.utils import TextDocument pytestmark = pytest.mark.usefixtures("find_project_root_cache_clear") diff --git a/src/darker/tests/test_diff.py b/src/darker/tests/test_diff.py index 68a67f156..ecd8ebe6a 100644 --- a/src/darker/tests/test_diff.py +++ b/src/darker/tests/test_diff.py @@ -13,7 +13,7 @@ opcodes_to_chunks, opcodes_to_edit_linenums, ) -from darker.utils import TextDocument +from darkgraylib.utils import TextDocument FUNCTIONS2_PY = dedent( """\ diff --git a/src/darker/tests/test_fstring.py b/src/darker/tests/test_fstring.py index d952d46fd..c283942d2 100644 --- a/src/darker/tests/test_fstring.py +++ b/src/darker/tests/test_fstring.py @@ -10,8 +10,9 @@ import darker.fstring from darker.git import EditedLinenumsDiffer from darker.tests.helpers import flynt_present -from darker.utils import TextDocument, joinlines +from darker.utils import joinlines from darkgraylib.git import RevisionRange +from darkgraylib.utils import TextDocument ORIGINAL_SOURCE = ("'{}'.format(x)", "#", "'{0}'.format(42)") MODIFIED_SOURCE = ("'{}'.format( x)", "#", "'{0}'.format( 42)") diff --git a/src/darker/tests/test_git.py b/src/darker/tests/test_git.py index d6cf8a3d6..70c049e3e 100644 --- a/src/darker/tests/test_git.py +++ b/src/darker/tests/test_git.py @@ -17,8 +17,9 @@ from darker import git from darker.tests.conftest import GitRepoFixture from darker.tests.helpers import raises_or_matches -from darker.utils import GIT_DATEFORMAT, TextDocument +from darker.utils import GIT_DATEFORMAT from darkgraylib.git import RevisionRange +from darkgraylib.utils import TextDocument def test_tmp_path_sanity(tmp_path): diff --git a/src/darker/tests/test_import_sorting.py b/src/darker/tests/test_import_sorting.py index dc9390583..9c6b99abb 100644 --- a/src/darker/tests/test_import_sorting.py +++ b/src/darker/tests/test_import_sorting.py @@ -11,8 +11,9 @@ import darker.import_sorting from darker.git import EditedLinenumsDiffer from darker.tests.helpers import isort_present -from darker.utils import TextDocument, joinlines +from darker.utils import joinlines from darkgraylib.git import RevisionRange +from darkgraylib.utils import TextDocument ORIGINAL_SOURCE = ("import sys", "import os", "", "print(42)") ISORTED_SOURCE = ("import os", "import sys", "", "print(42)") diff --git a/src/darker/tests/test_main.py b/src/darker/tests/test_main.py index 35a9a94b4..53f62d6af 100644 --- a/src/darker/tests/test_main.py +++ b/src/darker/tests/test_main.py @@ -25,9 +25,10 @@ from darker.tests.helpers import isort_present from darker.tests.test_fstring import FLYNTED_SOURCE, MODIFIED_SOURCE, ORIGINAL_SOURCE from darker.tests.test_highlighting import BLUE, CYAN, RESET, WHITE, YELLOW -from darker.utils import TextDocument, joinlines +from darker.utils import joinlines from darker.verification import NotEquivalentError from darkgraylib.git import RevisionRange +from darkgraylib.utils import TextDocument def randomword(length: int) -> str: diff --git a/src/darker/tests/test_main_blacken_and_flynt_single_file.py b/src/darker/tests/test_main_blacken_and_flynt_single_file.py index faf4f78bc..18d1cbcc4 100644 --- a/src/darker/tests/test_main_blacken_and_flynt_single_file.py +++ b/src/darker/tests/test_main_blacken_and_flynt_single_file.py @@ -10,8 +10,8 @@ from darker.__main__ import _blacken_and_flynt_single_file from darker.config import Exclusions from darker.git import EditedLinenumsDiffer -from darker.utils import TextDocument from darkgraylib.git import RevisionRange +from darkgraylib.utils import TextDocument @pytest.mark.kwparametrize( diff --git a/src/darker/tests/test_multiline_strings.py b/src/darker/tests/test_multiline_strings.py index 524630588..ba4a6f85d 100644 --- a/src/darker/tests/test_multiline_strings.py +++ b/src/darker/tests/test_multiline_strings.py @@ -5,7 +5,7 @@ import pytest from darker import multiline_strings -from darker.utils import TextDocument +from darkgraylib.utils import TextDocument def test_get_multiline_string_ranges(): diff --git a/src/darker/tests/test_utils.py b/src/darker/tests/test_utils.py index 9811268a8..655f1dca2 100644 --- a/src/darker/tests/test_utils.py +++ b/src/darker/tests/test_utils.py @@ -10,13 +10,13 @@ import pytest from darker.utils import ( - TextDocument, debug_dump, detect_newline, get_common_root, get_path_ancestry, joinlines, ) +from darkgraylib.utils import TextDocument @pytest.fixture(params=[TextDocument.from_file, TextDocument.from_bytes]) diff --git a/src/darker/tests/test_verification.py b/src/darker/tests/test_verification.py index 0ec7f2e94..70e209bf3 100644 --- a/src/darker/tests/test_verification.py +++ b/src/darker/tests/test_verification.py @@ -6,13 +6,14 @@ import pytest -from darker.utils import DiffChunk, TextDocument +from darker.utils import DiffChunk from darker.verification import ( ASTVerifier, BinarySearch, NotEquivalentError, verify_ast_unchanged, ) +from darkgraylib.utils import TextDocument @pytest.mark.kwparametrize( diff --git a/src/darker/utils.py b/src/darker/utils.py index a992240fc..41e097017 100644 --- a/src/darker/utils.py +++ b/src/darker/utils.py @@ -1,10 +1,7 @@ """Miscellaneous utility functions""" -import io import logging import sys -import tokenize -from datetime import datetime from itertools import chain from pathlib import Path from typing import Collection, Iterable, List, Tuple @@ -26,157 +23,6 @@ def detect_newline(string: str) -> str: return "\n" -class TextDocument: - """Store & handle a multi-line text document, either as a string or list of lines""" - - DEFAULT_ENCODING = "utf-8" - DEFAULT_NEWLINE = "\n" - - def __init__( # pylint: disable=too-many-arguments - self, - string: str = None, - lines: Iterable[str] = None, - encoding: str = DEFAULT_ENCODING, - newline: str = DEFAULT_NEWLINE, - mtime: str = "", - ): - self._string = string - self._lines = None if lines is None else tuple(lines) - self._encoding = encoding - self._newline = newline - self._mtime = mtime - - def string_with_newline(self, newline: str) -> str: - """Return the document as a string, using the given newline sequence""" - if self._string is None or detect_newline(self._string) != newline: - return joinlines(self.lines or (), newline) - return self._string - - @property - def string(self) -> str: - """Return the document as a string, converting and caching if necessary""" - if self._string is None: - self._string = self.string_with_newline(self.newline) - return self._string - - @property - def encoded_string(self) -> bytes: - """Return the document as a bytestring, converting and caching if necessary""" - return self.string.encode(self.encoding) - - @property - def lines(self) -> TextLines: - """Return the document as a list of lines converting and caching if necessary""" - if self._lines is None: - self._lines = tuple((self._string or "").splitlines()) - return self._lines - - @property - def encoding(self) -> str: - """Return the encoding used in the document""" - return self._encoding - - @property - def newline(self) -> str: - """Return the newline character sequence used in the document""" - return self._newline - - @property - def mtime(self) -> str: - """Return the last modification time of the document""" - return self._mtime - - @classmethod - def from_str( - cls, - string: str, - encoding: str = DEFAULT_ENCODING, - override_newline: str = None, - mtime: str = "", - ) -> "TextDocument": - """Create a document object from a string - - :param string: The contents of the new text document - :param encoding: The character encoding to be used when writing out the bytes - :param override_newline: Replace existing newlines with the given newline string - :param mtime: The modification time of the original file - - """ - newline = detect_newline(string) - if override_newline and override_newline != newline: - string = string.replace(newline, override_newline) - newline = override_newline - return cls(string, None, encoding=encoding, newline=newline, mtime=mtime) - - @classmethod - def from_bytes(cls, data: bytes, mtime: str = "") -> "TextDocument": - """Create a document object from a binary string - - :param data: The binary content of the new text document - :param mtime: The modification time of the original file - - """ - srcbuf = io.BytesIO(data) - encoding, lines = tokenize.detect_encoding(srcbuf.readline) - if not lines: - return cls(lines=[], encoding=encoding, mtime=mtime) - return cls.from_str(data.decode(encoding), encoding=encoding, mtime=mtime) - - @classmethod - def from_file(cls, path: Path) -> "TextDocument": - """Create a document object by reading a text file - - Also store the last modification time of the file. - - """ - mtime = datetime.utcfromtimestamp(path.stat().st_mtime).strftime(GIT_DATEFORMAT) - with path.open("rb") as srcbuf: - return cls.from_bytes(srcbuf.read(), mtime) - - @classmethod - def from_lines( - cls, - lines: Iterable[str], - encoding: str = DEFAULT_ENCODING, - newline: str = DEFAULT_NEWLINE, - mtime: str = "", - ) -> "TextDocument": - """Create a document object from a list of lines - - The lines should be strings without trailing newlines. They should be encoded in - UTF-8 unless a different encoding is specified with the ``encoding`` argument. - - """ - return cls(None, lines, encoding=encoding, newline=newline, mtime=mtime) - - def __eq__(self, other: object) -> bool: - """Compare the equality two text documents, ignoring the modification times""" - if not isinstance(other, TextDocument): - return NotImplemented - if not self._string and not self._lines: - return not other._string and not other._lines - return self.lines == other.lines - - def __repr__(self) -> str: - """Return a Python representation of the document object""" - encoding = ( - "" - if self._encoding == self.DEFAULT_ENCODING - else f", encoding={self.encoding!r}" - ) - newline = ( - "" - if self.newline == self.DEFAULT_NEWLINE - else f", newline={self.newline!r}" - ) - mtime = "" if not self._mtime else f", mtime={self._mtime!r}" - return ( - f"{type(self).__name__}(" - f"[{len(self.lines)} lines]" - f"{encoding}{newline}{mtime})" - ) - - DiffChunk = Tuple[int, TextLines, TextLines] diff --git a/src/darker/verification.py b/src/darker/verification.py index 8184a3a2c..ebbc4ce61 100644 --- a/src/darker/verification.py +++ b/src/darker/verification.py @@ -4,7 +4,8 @@ from black import assert_equivalent, parse_ast, stringify_ast -from darker.utils import DiffChunk, TextDocument, debug_dump +from darker.utils import DiffChunk, debug_dump +from darkgraylib.utils import TextDocument class NotEquivalentError(Exception): From 56c34b3c5ff0d96a1842b2ca3784e6d36c852e03 Mon Sep 17 00:00:00 2001 From: Antti Kaihola <13725+akaihola@users.noreply.github.com> Date: Wed, 12 Jul 2023 21:48:58 +0300 Subject: [PATCH 15/30] Move `git_repo` fixture to `darkgraylib` --- CHANGES.rst | 1 + src/darker/tests/conftest.py | 118 ------------------- src/darker/tests/test_git.py | 2 +- src/darker/tests/test_main_stdin_filename.py | 2 +- 4 files changed, 3 insertions(+), 120 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 03b773d77..591f1f9d3 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -37,6 +37,7 @@ Added - Allow ``-`` as the single source filename when using the ``--stdin-filename`` option. This makes the option compatible with Black. - Upgrade NixOS tests to use Python 3.11 on both Linux and macOS. +- Move ``git_repo`` fixture to ``darkgraylib``. Fixed ----- diff --git a/src/darker/tests/conftest.py b/src/darker/tests/conftest.py index 88cf210fb..b8f763c6f 100644 --- a/src/darker/tests/conftest.py +++ b/src/darker/tests/conftest.py @@ -1,126 +1,8 @@ """Configuration and fixtures for the Pytest based test suite""" -import os -import re -from pathlib import Path -from subprocess import check_call # nosec -from typing import Dict, Iterable, List, Union - import pytest from black import find_project_root as black_find_project_root -from darker.git import _git_check_output_lines, git_get_version - - -class GitRepoFixture: - """Fixture for managing temporary Git repositories""" - def __init__(self, root: Path, env: Dict[str, str]): - self.root = root - self.env = env - - @classmethod - def create_repository(cls, root: Path) -> "GitRepoFixture": - """Fixture method for creating a Git repository in the given directory""" - # For testing, ignore ~/.gitconfig settings like templateDir and defaultBranch. - # Also, this makes sure GIT_DIR or other GIT_* variables are not set, and that - # Git's messages are in English. - env = {"HOME": str(root), "LC_ALL": "C", "PATH": os.environ["PATH"]} - instance = cls(root, env) - # pylint: disable=protected-access - force_master = ( - ["--initial-branch=master"] if git_get_version() >= (2, 28) else [] - ) - instance._run("init", *force_master) - instance._run("config", "user.email", "ci@example.com") - instance._run("config", "user.name", "CI system") - return instance - - def _run(self, *args: str) -> None: - """Helper method to run a Git command line in the repository root""" - check_call(["git"] + list(args), cwd=self.root, env=self.env) # nosec - - def _run_and_get_first_line(self, *args: str) -> str: - """Helper method to run Git in repo root and return first line of output""" - return _git_check_output_lines(list(args), Path(self.root))[0] - - def add( - self, paths_and_contents: Dict[str, Union[str, bytes, None]], commit: str = None - ) -> Dict[str, Path]: - """Add/remove/modify files and optionally commit the changes - - :param paths_and_contents: Paths of the files relative to repository root, and - new contents for the files as strings. ``None`` can - be specified as the contents in order to remove a - file. - :param commit: The message for the commit, or ``None`` to skip making a commit. - - """ - absolute_paths = { - relative_path: self.root / relative_path - for relative_path in paths_and_contents - } - for relative_path, content in paths_and_contents.items(): - path = absolute_paths[relative_path] - if content is None: - self._run("rm", "--", relative_path) - continue - if isinstance(content, str): - content = content.encode("utf-8") - path.parent.mkdir(parents=True, exist_ok=True) - path.write_bytes(content) - self._run("add", "--", relative_path) - if commit: - self._run("commit", "-m", commit) - return absolute_paths - - def get_hash(self, revision: str = "HEAD") -> str: - """Return the commit hash at the given revision in the Git repository""" - return self._run_and_get_first_line("rev-parse", revision) - - def get_branch(self) -> str: - """Return the active branch name in the Git repository""" - return self._run_and_get_first_line("symbolic-ref", "--short", "HEAD") - - def create_tag(self, tag_name: str) -> None: - """Create a tag at current HEAD""" - self._run("tag", tag_name) - - def create_branch(self, new_branch: str, start_point: str) -> None: - """Fixture method to create and check out new branch at given starting point""" - self._run("checkout", "-b", new_branch, start_point) - - def expand_root(self, lines: Iterable[str]) -> List[str]: - """Replace "{root/}" in strings with the path in the temporary Git repo - - This is used to generate expected strings corresponding to locations of files in - the temporary Git repository. - - :param lines: The lines of text to process - :return: Given lines with paths processed - - """ - return [ - re.sub( - r"\{root(/.*?)?\}", - lambda m: str(self.root / (str(m.group(1)[1:]) if m.group(1) else "")), - line, - ) - for line in lines - ] - - -@pytest.fixture -def git_repo(tmp_path, monkeypatch): - """Create a temporary Git repository and change current working directory into it""" - repository = GitRepoFixture.create_repository(tmp_path) - monkeypatch.chdir(tmp_path) - # While `GitRepoFixture.create_repository()` already deletes `GIT_*` environment - # variables for any Git commands run by the fixture, let's explicitly remove - # `GIT_DIR` in case a test should call Git directly: - monkeypatch.delenv("GIT_DIR", raising=False) - - yield repository - @pytest.fixture def find_project_root_cache_clear(): diff --git a/src/darker/tests/test_git.py b/src/darker/tests/test_git.py index 70c049e3e..95ae78e9b 100644 --- a/src/darker/tests/test_git.py +++ b/src/darker/tests/test_git.py @@ -15,10 +15,10 @@ import pytest from darker import git -from darker.tests.conftest import GitRepoFixture from darker.tests.helpers import raises_or_matches from darker.utils import GIT_DATEFORMAT from darkgraylib.git import RevisionRange +from darkgraylib.teststools.git_repo_plugin import GitRepoFixture from darkgraylib.utils import TextDocument diff --git a/src/darker/tests/test_main_stdin_filename.py b/src/darker/tests/test_main_stdin_filename.py index 76446a54d..7cfafc0d7 100644 --- a/src/darker/tests/test_main_stdin_filename.py +++ b/src/darker/tests/test_main_stdin_filename.py @@ -10,9 +10,9 @@ import toml import darker.__main__ -from darker.tests.conftest import GitRepoFixture from darker.tests.helpers import raises_if_exception from darkgraylib.config import ConfigurationError +from darkgraylib.testtools.git_repo_plugin import GitRepoFixture pytestmark = pytest.mark.usefixtures("find_project_root_cache_clear") From c39ca16e033b5c5be5e25538c1192ef6d327b00c Mon Sep 17 00:00:00 2001 From: Antti Kaihola <13725+akaihola@users.noreply.github.com> Date: Mon, 7 Aug 2023 07:08:34 +0300 Subject: [PATCH 16/30] Move parts of `darker.config` into `darkgraylib` --- src/darker/config.py | 164 +------------------------------------------ 1 file changed, 3 insertions(+), 161 deletions(-) diff --git a/src/darker/config.py b/src/darker/config.py index d954235a1..7beb307d3 100644 --- a/src/darker/config.py +++ b/src/darker/config.py @@ -1,25 +1,11 @@ """Load and save configuration in TOML format""" -import logging -import os -from argparse import ArgumentParser, Namespace +from argparse import Namespace from dataclasses import dataclass, field from pathlib import Path -from typing import Dict, Iterable, List, Optional, Set, Union, cast - -import toml - -from darker.black_compat import find_project_root -from darkgraylib.config import BaseConfig - - -class TomlArrayLinesEncoder(toml.TomlEncoder): # type: ignore - """Format TOML so list items are each on their own line""" - - def dump_list(self, v: Iterable[object]) -> str: - """Format a list value""" - return "[{}\n]".format("".join(f"\n {self.dump_value(item)}," for item in v)) +from typing import Dict, List, Optional, Set, Union +from darkgraylib.config import BaseConfig, ConfigurationError UnvalidatedConfig = Dict[str, Union[List[str], str, bool, int]] @@ -79,50 +65,6 @@ def validate_stdout_src( ) -class ConfigurationError(Exception): - """Exception class for invalid configuration values""" - - -def convert_config_characters( - config: UnvalidatedConfig, pattern: str, replacement: str -) -> UnvalidatedConfig: - """Convert a character in config keys to a different character""" - return {key.replace(pattern, replacement): value for key, value in config.items()} - - -def convert_hyphens_to_underscores(config: UnvalidatedConfig) -> UnvalidatedConfig: - """Convert hyphenated config keys to underscored keys""" - return convert_config_characters(config, "-", "_") - - -def convert_underscores_to_hyphens(config: DarkerConfig) -> UnvalidatedConfig: - """Convert underscores in config keys to hyphens""" - return convert_config_characters(cast(UnvalidatedConfig, config), "_", "-") - - -def validate_config_keys(config: UnvalidatedConfig) -> None: - """Raise an exception if any keys in the configuration are invalid. - - :param config: The configuration read from ``pyproject.toml`` - :raises ConfigurationError: Raised if unknown options are present - - """ - if set(config).issubset(DarkerConfig.__annotations__): - return - unknown_keys = ", ".join( - sorted(set(config).difference(DarkerConfig.__annotations__)) - ) - raise ConfigurationError( - f"Invalid [tool.darker] keys in pyproject.toml: {unknown_keys}" - ) - - -def replace_log_level_name(config: DarkerConfig) -> None: - """Replace numeric log level in configuration with the name of the log level""" - if "log_level" in config: - config["log_level"] = logging.getLevelName(config["log_level"]) - - def validate_config_output_mode(config: DarkerConfig) -> None: """Make sure both ``diff`` and ``stdout`` aren't enabled in configuration""" OutputMode.validate_diff_stdout( @@ -130,106 +72,6 @@ def validate_config_output_mode(config: DarkerConfig) -> None: ) -def validate_stdin_src(stdin_filename: Optional[str], src: List[str]) -> None: - """Make sure both ``stdin`` mode and paths/directories are specified""" - if stdin_filename is None: - return - if len(src) == 0 or src == ["-"]: - return - raise ConfigurationError( - "No Python source files are allowed when using the `stdin-filename` option" - ) - - -def override_color_with_environment(pyproject_config: DarkerConfig) -> DarkerConfig: - """Override ``color`` if the ``PY_COLORS`` environment variable is '0' or '1' - - :param config: The configuration read from ``pyproject.toml`` - :return: The modified configuration - - """ - config = pyproject_config.copy() - py_colors = os.getenv("PY_COLORS") - if py_colors in {"0", "1"}: - config["color"] = py_colors == "1" - elif os.getenv("NO_COLOR") is not None: - config["color"] = False - elif os.getenv("FORCE_COLOR") is not None: - config["color"] = True - return config - - -def load_config(path: Optional[str], srcs: Iterable[str]) -> DarkerConfig: - """Find and load Darker configuration from a TOML configuration file - - Darker determines the location for the configuration file by trying the following: - - the file path in the `path` argument, given using the ``-c``/``--config`` command - line option - - ``pyproject.toml`` inside the directory specified by the `path` argument - - ``pyproject.toml`` from a common parent directory to all items in `srcs` - - ``pyproject.toml`` in the current working directory if `srcs` is empty - - :param path: The file or directory specified using the ``-c``/``--config`` command - line option, or `None` if the option was omitted. - :param srcs: File(s) and directory/directories to be processed by Darker. - - """ - if path: - for candidate_path in [Path(path), Path(path, "pyproject.toml")]: - if candidate_path.is_file(): - config_path = candidate_path - break - else: - if Path(path).is_dir() or path.endswith(os.sep): - raise ConfigurationError( - f"Configuration file {Path(path, 'pyproject.toml')} not found" - ) - raise ConfigurationError(f"Configuration file {path} not found") - else: - config_path = find_project_root(tuple(srcs or ["."])) / "pyproject.toml" - if not config_path.is_file(): - return {} - pyproject_toml = toml.load(config_path) - tool_darker_config = convert_hyphens_to_underscores( - pyproject_toml.get("tool", {}).get("darker", {}) or {} - ) - validate_config_keys(tool_darker_config) - config = cast(DarkerConfig, tool_darker_config) - replace_log_level_name(config) - validate_config_output_mode(config) - return config - - -def get_effective_config(args: Namespace) -> DarkerConfig: - """Return all configuration options""" - config = cast(DarkerConfig, vars(args).copy()) - replace_log_level_name(config) - validate_config_output_mode(config) - return config - - -def get_modified_config(parser: ArgumentParser, args: Namespace) -> DarkerConfig: - """Return configuration options which are set to non-default values""" - not_default = cast( - DarkerConfig, - { - argument: value - for argument, value in vars(args).items() - if value != parser.get_default(argument) - }, - ) - replace_log_level_name(not_default) - return not_default - - -def dump_config(config: DarkerConfig) -> str: - """Return the configuration in TOML format""" - dump = toml.dumps( - convert_underscores_to_hyphens(config), encoder=TomlArrayLinesEncoder() - ) - return f"[tool.darker]\n{dump}" - - @dataclass class Exclusions: """File exclusions patterns for pre-processing steps From 92d12a0244f82b53abdcad3708a8b8f4566ab90f Mon Sep 17 00:00:00 2001 From: Antti Kaihola <13725+akaihola@users.noreply.github.com> Date: Mon, 7 Aug 2023 08:48:53 +0300 Subject: [PATCH 17/30] Move Black cache clear plugin to `darkgraylib` --- src/darker/tests/conftest.py | 16 ---------------- 1 file changed, 16 deletions(-) delete mode 100644 src/darker/tests/conftest.py diff --git a/src/darker/tests/conftest.py b/src/darker/tests/conftest.py deleted file mode 100644 index b8f763c6f..000000000 --- a/src/darker/tests/conftest.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Configuration and fixtures for the Pytest based test suite""" - -import pytest -from black import find_project_root as black_find_project_root - - -@pytest.fixture -def find_project_root_cache_clear(): - """Clear LRU caching in :func:`black.find_project_root` before each test - - NOTE: We use `darker.black_compat.find_project_root` to wrap Black's original - function since its signature has changed along the way. However, clearing the cache - needs to be done on the original of course. - - """ - black_find_project_root.cache_clear() From 496513832fcdf1e212deef6ed846c244ec07cdde Mon Sep 17 00:00:00 2001 From: Antti Kaihola <13725+akaihola@users.noreply.github.com> Date: Mon, 7 Aug 2023 08:50:07 +0300 Subject: [PATCH 18/30] Move some `darker.config` tests into `darkgraylib` --- src/darker/black_diff.py | 2 +- src/darker/tests/test_black_diff.py | 2 +- src/darker/tests/test_command_line.py | 3 +- src/darker/tests/test_config.py | 499 +------------------------- 4 files changed, 7 insertions(+), 499 deletions(-) diff --git a/src/darker/black_diff.py b/src/darker/black_diff.py index 62b91c493..8bf5a0c04 100644 --- a/src/darker/black_diff.py +++ b/src/darker/black_diff.py @@ -53,7 +53,7 @@ from black.files import gen_python_files from black.report import Report -from darker.config import ConfigurationError +from darkgraylib.config import ConfigurationError from darkgraylib.utils import TextDocument __all__ = ["BlackConfig", "Mode", "run_black"] diff --git a/src/darker/tests/test_black_diff.py b/src/darker/tests/test_black_diff.py index 1f110eb7d..efa2264cb 100644 --- a/src/darker/tests/test_black_diff.py +++ b/src/darker/tests/test_black_diff.py @@ -22,8 +22,8 @@ read_black_config, run_black, ) -from darker.config import ConfigurationError from darker.tests.helpers import raises_or_matches +from darkgraylib.config import ConfigurationError from darkgraylib.utils import TextDocument if sys.version_info >= (3, 11): diff --git a/src/darker/tests/test_command_line.py b/src/darker/tests/test_command_line.py index 3c5b28461..f7b6f9cc9 100644 --- a/src/darker/tests/test_command_line.py +++ b/src/darker/tests/test_command_line.py @@ -17,9 +17,10 @@ from darker import black_diff from darker.__main__ import main from darker.command_line import make_argument_parser, parse_command_line -from darker.config import ConfigurationError, Exclusions +from darker.config import Exclusions from darker.tests.helpers import flynt_present, isort_present, raises_if_exception from darker.utils import joinlines +from darkgraylib.config import ConfigurationError from darkgraylib.git import RevisionRange from darkgraylib.utils import TextDocument diff --git a/src/darker/tests/test_config.py b/src/darker/tests/test_config.py index ef99c6382..fc92ecb20 100644 --- a/src/darker/tests/test_config.py +++ b/src/darker/tests/test_config.py @@ -2,57 +2,14 @@ # pylint: disable=unused-argument,too-many-arguments,use-dict-literal -import os -import re -from argparse import ArgumentParser, Namespace +from argparse import Namespace from pathlib import Path -from textwrap import dedent import pytest -from darker.config import ( - ConfigurationError, - OutputMode, - TomlArrayLinesEncoder, - dump_config, - get_effective_config, - get_modified_config, - load_config, -) +from darker.config import OutputMode from darker.tests.helpers import raises_if_exception - - -@pytest.mark.kwparametrize( - dict(list_value=[], expect="[\n]"), - dict(list_value=["one value"], expect='[\n "one value",\n]'), - dict(list_value=["two", "values"], expect='[\n "two",\n "values",\n]'), - dict( - list_value=[ - "a", - "dozen", - "short", - "string", - "values", - "in", - "the", - "list", - "of", - "strings", - "to", - "format", - ], - expect=( - '[\n "a",\n "dozen",\n "short",\n "string",\n "values"' - ',\n "in",\n "the",\n "list",\n "of",\n "strings"' - ',\n "to",\n "format",\n]' - ), - ), -) -def test_toml_array_lines_encoder(list_value, expect): - """``TomlArrayLinesEncoder`` formats lists with each item on its own line""" - result = TomlArrayLinesEncoder().dump_list(list_value) - - assert result == expect +from darkgraylib.config import ConfigurationError @pytest.mark.kwparametrize( @@ -170,453 +127,3 @@ def test_output_mode_from_args(diff, stdout, expect): result = OutputMode.from_args(args) assert result == expect - - -@pytest.mark.kwparametrize( - dict(), # pylint: disable=use-dict-literal - dict(cwd="lvl1"), - dict(cwd="lvl1/lvl2"), - dict(cwd="has_git", expect={}), - dict(cwd="has_git/lvl1", expect={}), - dict(cwd="has_pyp", expect={"config": "has_pyp"}), - dict(cwd="has_pyp/lvl1", expect={"config": "has_pyp"}), - dict(srcs=["root.py"]), - dict(srcs=["../root.py"], cwd="lvl1"), - dict(srcs=["../root.py"], cwd="has_git"), - dict(srcs=["../root.py"], cwd="has_pyp"), - dict(srcs=["root.py", "lvl1/lvl1.py"]), - dict(srcs=["../root.py", "lvl1.py"], cwd="lvl1"), - dict(srcs=["../root.py", "../lvl1/lvl1.py"], cwd="has_git"), - dict(srcs=["../root.py", "../lvl1/lvl1.py"], cwd="has_pyp"), - dict(srcs=["has_pyp/pyp.py", "lvl1/lvl1.py"]), - dict(srcs=["../has_pyp/pyp.py", "lvl1.py"], cwd="lvl1"), - dict(srcs=["../has_pyp/pyp.py", "../lvl1/lvl1.py"], cwd="has_git"), - dict(srcs=["pyp.py", "../lvl1/lvl1.py"], cwd="has_pyp"), - dict( - srcs=["has_pyp/lvl1/l1.py", "has_pyp/lvl1b/l1b.py"], - expect={"config": "has_pyp"}, - ), - dict( - srcs=["../has_pyp/lvl1/l1.py", "../has_pyp/lvl1b/l1b.py"], - cwd="lvl1", - expect={"config": "has_pyp"}, - ), - dict( - srcs=["../has_pyp/lvl1/l1.py", "../has_pyp/lvl1b/l1b.py"], - cwd="has_git", - expect={"config": "has_pyp"}, - ), - dict( - srcs=["lvl1/l1.py", "lvl1b/l1b.py"], - cwd="has_pyp", - expect={"config": "has_pyp"}, - ), - dict( - srcs=["full_example/full.py"], - expect={ - "check": True, - "diff": True, - "isort": True, - "lint": ["flake8", "mypy", "pylint"], - "log_level": 10, - "revision": "main", - "src": ["src", "tests"], - }, - ), - dict(srcs=["stdout_example/dummy.py"], expect={"stdout": True}), - dict(confpath="c", expect={"lint": ["PYP_TOML"]}), - dict(confpath="c/pyproject.toml", expect={"lint": ["PYP_TOML"]}), - dict(cwd="lvl1", confpath="../c", expect={"lint": ["PYP_TOML"]}), - dict(cwd="lvl1", confpath="../c/pyproject.toml", expect={"lint": ["PYP_TOML"]}), - dict(cwd="lvl1/lvl2", confpath="../../c", expect={"lint": ["PYP_TOML"]}), - dict( - cwd="lvl1/lvl2", - confpath="../../c/pyproject.toml", - expect={"lint": ["PYP_TOML"]}, - ), - dict(cwd="has_git", confpath="../c", expect={"lint": ["PYP_TOML"]}), - dict(cwd="has_git", confpath="../c/pyproject.toml", expect={"lint": ["PYP_TOML"]}), - dict(cwd="has_git/lvl1", confpath="../../c", expect={"lint": ["PYP_TOML"]}), - dict( - cwd="has_git/lvl1", - confpath="../../c/pyproject.toml", - expect={"lint": ["PYP_TOML"]}, - ), - dict(cwd="has_pyp", confpath="../c", expect={"lint": ["PYP_TOML"]}), - dict(cwd="has_pyp", confpath="../c/pyproject.toml", expect={"lint": ["PYP_TOML"]}), - dict(cwd="has_pyp/lvl1", confpath="../../c", expect={"lint": ["PYP_TOML"]}), - dict( - cwd="has_pyp/lvl1", - confpath="../../c/pyproject.toml", - expect={"lint": ["PYP_TOML"]}, - ), - dict(srcs=["root.py"], confpath="c", expect={"lint": ["PYP_TOML"]}), - dict(srcs=["root.py"], confpath="c/pyproject.toml", expect={"lint": ["PYP_TOML"]}), - dict( - srcs=["../root.py"], cwd="lvl1", confpath="../c", expect={"lint": ["PYP_TOML"]} - ), - dict( - srcs=["../root.py"], - cwd="lvl1", - confpath="../c/pyproject.toml", - expect={"lint": ["PYP_TOML"]}, - ), - dict( - srcs=["../root.py"], - cwd="has_git", - confpath="../c", - expect={"lint": ["PYP_TOML"]}, - ), - dict( - srcs=["../root.py"], - cwd="has_git", - confpath="../c/pyproject.toml", - expect={"lint": ["PYP_TOML"]}, - ), - dict( - srcs=["../root.py"], - cwd="has_pyp", - confpath="../c", - expect={"lint": ["PYP_TOML"]}, - ), - dict( - srcs=["../root.py"], - cwd="has_pyp", - confpath="../c/pyproject.toml", - expect={"lint": ["PYP_TOML"]}, - ), - dict(srcs=["root.py", "lvl1/lvl1.py"], confpath="c", expect={"lint": ["PYP_TOML"]}), - dict( - srcs=["root.py", "lvl1/lvl1.py"], - confpath="c/pyproject.toml", - expect={"lint": ["PYP_TOML"]}, - ), - dict( - srcs=["../root.py", "lvl1.py"], - cwd="lvl1", - confpath="../c", - expect={"lint": ["PYP_TOML"]}, - ), - dict( - srcs=["../root.py", "lvl1.py"], - cwd="lvl1", - confpath="../c/pyproject.toml", - expect={"lint": ["PYP_TOML"]}, - ), - dict( - srcs=["../root.py", "../lvl1/lvl1.py"], - cwd="has_git", - confpath="../c", - expect={"lint": ["PYP_TOML"]}, - ), - dict( - srcs=["../root.py", "../lvl1/lvl1.py"], - cwd="has_git", - confpath="../c/pyproject.toml", - expect={"lint": ["PYP_TOML"]}, - ), - dict( - srcs=["../root.py", "../lvl1/lvl1.py"], - cwd="has_pyp", - confpath="../c", - expect={"lint": ["PYP_TOML"]}, - ), - dict( - srcs=["../root.py", "../lvl1/lvl1.py"], - cwd="has_pyp", - confpath="../c/pyproject.toml", - expect={"lint": ["PYP_TOML"]}, - ), - dict( - srcs=["has_pyp/pyp.py", "lvl1/lvl1.py"], - confpath="c", - expect={"lint": ["PYP_TOML"]}, - ), - dict( - srcs=["has_pyp/pyp.py", "lvl1/lvl1.py"], - confpath="c/pyproject.toml", - expect={"lint": ["PYP_TOML"]}, - ), - dict( - srcs=["../has_pyp/pyp.py", "lvl1.py"], - cwd="lvl1", - confpath="../c", - expect={"lint": ["PYP_TOML"]}, - ), - dict( - srcs=["../has_pyp/pyp.py", "lvl1.py"], - cwd="lvl1", - confpath="../c/pyproject.toml", - expect={"lint": ["PYP_TOML"]}, - ), - dict( - srcs=["../has_pyp/pyp.py", "../lvl1/lvl1.py"], - cwd="has_git", - confpath="../c", - expect={"lint": ["PYP_TOML"]}, - ), - dict( - srcs=["../has_pyp/pyp.py", "../lvl1/lvl1.py"], - cwd="has_git", - confpath="../c/pyproject.toml", - expect={"lint": ["PYP_TOML"]}, - ), - dict( - srcs=["pyp.py", "../lvl1/lvl1.py"], - cwd="has_pyp", - confpath="../c", - expect={"lint": ["PYP_TOML"]}, - ), - dict( - srcs=["pyp.py", "../lvl1/lvl1.py"], - cwd="has_pyp", - confpath="../c/pyproject.toml", - expect={"lint": ["PYP_TOML"]}, - ), - dict( - srcs=["has_pyp/lvl1/l1.py", "has_pyp/lvl1b/l1b.py"], - confpath="c", - expect={"lint": ["PYP_TOML"]}, - ), - dict( - srcs=["has_pyp/lvl1/l1.py", "has_pyp/lvl1b/l1b.py"], - confpath="c/pyproject.toml", - expect={"lint": ["PYP_TOML"]}, - ), - dict( - srcs=["../has_pyp/lvl1/l1.py", "../has_pyp/lvl1b/l1b.py"], - cwd="lvl1", - confpath="../c", - expect={"lint": ["PYP_TOML"]}, - ), - dict( - srcs=["../has_pyp/lvl1/l1.py", "../has_pyp/lvl1b/l1b.py"], - cwd="lvl1", - confpath="../c/pyproject.toml", - expect={"lint": ["PYP_TOML"]}, - ), - dict( - srcs=["../has_pyp/lvl1/l1.py", "../has_pyp/lvl1b/l1b.py"], - cwd="has_git", - confpath="../c", - expect={"lint": ["PYP_TOML"]}, - ), - dict( - srcs=["../has_pyp/lvl1/l1.py", "../has_pyp/lvl1b/l1b.py"], - cwd="has_git", - confpath="../c/pyproject.toml", - expect={"lint": ["PYP_TOML"]}, - ), - dict( - srcs=["lvl1/l1.py", "lvl1b/l1b.py"], - cwd="has_pyp", - confpath="../c", - expect={"lint": ["PYP_TOML"]}, - ), - dict( - srcs=["lvl1/l1.py", "lvl1b/l1b.py"], - cwd="has_pyp", - confpath="../c/pyproject.toml", - expect={"lint": ["PYP_TOML"]}, - ), - dict(srcs=["full_example/full.py"], confpath="c", expect={"lint": ["PYP_TOML"]}), - dict( - srcs=["full_example/full.py"], - confpath="c/pyproject.toml", - expect={"lint": ["PYP_TOML"]}, - ), - dict(srcs=["stdout_example/dummy.py"], confpath="c", expect={"lint": ["PYP_TOML"]}), - dict( - srcs=["stdout_example/dummy.py"], - confpath="c/pyproject.toml", - expect={"lint": ["PYP_TOML"]}, - ), - srcs=[], - cwd=".", - confpath=None, - expect={"config": "no_pyp"}, -) -def test_load_config( # pylint: disable=too-many-arguments - find_project_root_cache_clear, tmp_path, monkeypatch, srcs, cwd, confpath, expect -): - """``load_config()`` finds and loads configuration based on source file paths""" - (tmp_path / ".git").mkdir() - (tmp_path / "pyproject.toml").write_text('[tool.darker]\nconfig = "no_pyp"\n') - (tmp_path / "lvl1/lvl2").mkdir(parents=True) - (tmp_path / "has_git/.git").mkdir(parents=True) - (tmp_path / "has_git/lvl1").mkdir() - (tmp_path / "has_pyp/lvl1").mkdir(parents=True) - (tmp_path / "has_pyp/pyproject.toml").write_text( - '[tool.darker]\nconfig = "has_pyp"\n' - ) - (tmp_path / "full_example").mkdir() - (tmp_path / "full_example/pyproject.toml").write_text( - dedent( - """ - [tool.darker] - src = [ - "src", - "tests", - ] - revision = "main" - diff = true - check = true - isort = true - lint = [ - "flake8", - "mypy", - "pylint", - ] - log_level = "DEBUG" - """ - ) - ) - (tmp_path / "stdout_example").mkdir() - (tmp_path / "stdout_example/pyproject.toml").write_text( - "[tool.darker]\nstdout = true\n" - ) - (tmp_path / "c").mkdir() - (tmp_path / "c" / "pyproject.toml").write_text( - "[tool.darker]\nlint = ['PYP_TOML']\n" - ) - monkeypatch.chdir(tmp_path / cwd) - - result = load_config(confpath, srcs) - - assert result == expect - - -@pytest.mark.kwparametrize( - dict(path=".", expect="Configuration file pyproject.toml not found"), - dict(path="./foo.toml", expect="Configuration file ./foo.toml not found"), - dict( - path="empty", expect=f"Configuration file empty{os.sep}pyproject.toml not found" - ), - dict( - path="empty/", - expect=f"Configuration file empty{os.sep}pyproject.toml not found", - ), - dict(path="subdir/foo.toml", expect="Configuration file subdir/foo.toml not found"), - dict( - path="missing_dir", - expect="Configuration file missing_dir not found", - ), - dict( - path=f"missing_dir{os.sep}", - expect=f"Configuration file missing_dir{os.sep}pyproject.toml not found", - ), - dict( - path="missing_dir/foo.toml", - expect="Configuration file missing_dir/foo.toml not found", - ), -) -def test_load_config_explicit_path_errors(tmp_path, monkeypatch, path, expect): - """``load_config()`` raises an error if given path is not a file""" - monkeypatch.chdir(tmp_path) - (tmp_path / "subdir").mkdir() - (tmp_path / "subdir" / "pyproject.toml").write_text("") - (tmp_path / "empty").mkdir() - with pytest.raises(ConfigurationError, match=re.escape(expect)): - - _ = load_config(path, ["."]) - - -@pytest.mark.kwparametrize( - dict(args=Namespace(), expect={}), - dict(args=Namespace(one="option"), expect={"one": "option"}), - dict(args=Namespace(log_level=10), expect={"log_level": "DEBUG"}), - dict( - args=Namespace(two="options", log_level=20), - expect={"two": "options", "log_level": "INFO"}, - ), - dict(args=Namespace(diff=True, stdout=True), expect=ConfigurationError), -) -def test_get_effective_config(args, expect): - """``get_effective_config()`` converts command line options correctly""" - with raises_if_exception(expect): - - result = get_effective_config(args) - - assert result == expect - - -@pytest.mark.kwparametrize( - dict(args=Namespace(), expect={}), - dict(args=Namespace(unknown="option"), expect={"unknown": "option"}), - dict(args=Namespace(log_level=10), expect={"log_level": "DEBUG"}), - dict(args=Namespace(names=[], int=42, string="fourty-two"), expect={"names": []}), - dict( - args=Namespace(names=["bar"], int=42, string="fourty-two"), - expect={"names": ["bar"]}, - ), - dict( - args=Namespace(names=["foo"], int=43, string="fourty-two"), expect={"int": 43} - ), - dict(args=Namespace(names=["foo"], int=42, string="one"), expect={"string": "one"}), -) -def test_get_modified_config(args, expect): - """``get_modified_config()`` only includes non-default configuration options""" - parser = ArgumentParser() - parser.add_argument("names", nargs="*", default=["foo"]) - parser.add_argument("--int", dest="int", default=42) - parser.add_argument("--string", default="fourty-two") - - result = get_modified_config(parser, args) - - assert result == expect - - -@pytest.mark.kwparametrize( - dict(config={}, expect="[tool.darker]\n"), - dict(config={"str": "value"}, expect='[tool.darker]\nstr = "value"\n'), - dict(config={"int": 42}, expect="[tool.darker]\nint = 42\n"), - dict(config={"float": 4.2}, expect="[tool.darker]\nfloat = 4.2\n"), - dict( - config={"list": ["foo", "bar"]}, - expect=dedent( - """\ - [tool.darker] - list = [ - "foo", - "bar", - ] - """ - ), - ), - dict( - config={ - "src": ["main.py"], - "revision": "master", - "diff": False, - "stdout": False, - "check": False, - "isort": False, - "lint": [], - "config": None, - "log_level": "DEBUG", - "skip_string_normalization": None, - "line_length": None, - }, - expect=dedent( - """\ - [tool.darker] - src = [ - "main.py", - ] - revision = "master" - diff = false - stdout = false - check = false - isort = false - lint = [ - ] - log-level = "DEBUG" - """ - ), - ), -) -def test_dump_config(config, expect): - """``dump_config()`` outputs configuration correctly in the TOML format""" - result = dump_config(config) - - assert result == expect From e86947c51d4c9b061c5b8882730fc545029d52a6 Mon Sep 17 00:00:00 2001 From: Antti Kaihola <13725+akaihola@users.noreply.github.com> Date: Mon, 7 Aug 2023 08:50:54 +0300 Subject: [PATCH 19/30] fix darkgraylib.testtools import typo --- src/darker/tests/test_git.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/darker/tests/test_git.py b/src/darker/tests/test_git.py index 95ae78e9b..a0914ea3a 100644 --- a/src/darker/tests/test_git.py +++ b/src/darker/tests/test_git.py @@ -18,7 +18,7 @@ from darker.tests.helpers import raises_or_matches from darker.utils import GIT_DATEFORMAT from darkgraylib.git import RevisionRange -from darkgraylib.teststools.git_repo_plugin import GitRepoFixture +from darkgraylib.testtools.git_repo_plugin import GitRepoFixture from darkgraylib.utils import TextDocument From e2cdd62078c9d1f88721dc7481005537819dfc62 Mon Sep 17 00:00:00 2001 From: Antti Kaihola <13725+akaihola@users.noreply.github.com> Date: Mon, 7 Aug 2023 09:08:58 +0300 Subject: [PATCH 20/30] Move highlighting support into `darkgraylib` --- mypy.ini | 4 - setup.cfg | 3 - src/darker/__main__.py | 2 +- src/darker/highlighting/__init__.py | 55 --- src/darker/highlighting/lexers.py | 88 ---- src/darker/tests/test_highlighting.py | 657 -------------------------- src/darker/tests/test_main.py | 2 +- 7 files changed, 2 insertions(+), 809 deletions(-) delete mode 100644 src/darker/highlighting/__init__.py delete mode 100644 src/darker/highlighting/lexers.py delete mode 100644 src/darker/tests/test_highlighting.py diff --git a/mypy.ini b/mypy.ini index d478e24e0..556691eb9 100644 --- a/mypy.ini +++ b/mypy.ini @@ -57,10 +57,6 @@ disallow_any_explicit = False [mypy-darker.config] disallow_subclassing_any = False -[mypy-darker.highlighting.lexers] -disallow_any_unimported = False -disallow_subclassing_any = False - [mypy-darker.tests.conftest] disallow_any_unimported = False diff --git a/setup.cfg b/setup.cfg index 8a4235c11..c70e705b4 100644 --- a/setup.cfg +++ b/setup.cfg @@ -41,9 +41,6 @@ where = src [options.entry_points] console_scripts = darker = darker.__main__:main_with_error_handling -pygments.lexers = - lint_location = darker.highlighting.lexers:LocationLexer - lint_description = darker.highlighting.lexers:DescriptionLexer [options.extras_require] flynt = diff --git a/src/darker/__main__.py b/src/darker/__main__.py index c6d036586..3a08f3fea 100644 --- a/src/darker/__main__.py +++ b/src/darker/__main__.py @@ -36,12 +36,12 @@ git_is_repository, ) from darker.help import get_extra_instruction -from darker.highlighting import colorize, should_use_color from darker.import_sorting import apply_isort, isort from darker.utils import GIT_DATEFORMAT, DiffChunk, debug_dump, glob_any from darker.verification import ASTVerifier, BinarySearch, NotEquivalentError from darkgraylib.config import show_config_if_debug from darkgraylib.git import RevisionRange +from darkgraylib.highlighting import colorize, should_use_color from darkgraylib.log import setup_logging from darkgraylib.main import resolve_paths from darkgraylib.utils import TextDocument diff --git a/src/darker/highlighting/__init__.py b/src/darker/highlighting/__init__.py deleted file mode 100644 index 6467de043..000000000 --- a/src/darker/highlighting/__init__.py +++ /dev/null @@ -1,55 +0,0 @@ -"""Highlighting of terminal output""" - -# pylint: disable=import-outside-toplevel,unused-import - -import sys -from typing import Optional, cast - - -def should_use_color(config_color: Optional[bool]) -> bool: - """Return ``True`` if configuration and package support allow output highlighting - - In ``config_color``, the combination of ``color =`` in ``pyproject.toml``, the - ``PY_COLORS`` environment variable, and the ``--color``/``--no-color`` command line - options determine whether the user wants to force enable or disable highlighting. - - If highlighting isn't forced either way, it is automatically enabled for terminal - output. - - Finally, if ``pygments`` isn't installed, highlighting is disabled. - - :param config_color: The configuration as parsed from ``pyproject.toml`` and - overridden using environment variables and/or command line - options - :return: ``True`` if highlighting should be used - - """ - if config_color is not None: - use_color = config_color - else: - use_color = sys.stdout.isatty() - - if use_color: - try: - import pygments # noqa - - return True - except ImportError: - pass - return False - - -def colorize(output: str, lexer_name: str, use_color: bool) -> str: - """Return the output highlighted for terminal if Pygments is available""" - if not use_color: - return output - from pygments import highlight - from pygments.formatters.terminal import TerminalFormatter - from pygments.lexers import get_lexer_by_name - - lexer = get_lexer_by_name(lexer_name) - highlighted = highlight(output, lexer, TerminalFormatter()) - if "\n" not in output: - # see https://github.com/pygments/pygments/issues/1107 - highlighted = highlighted.rstrip("\n") - return cast(str, highlighted) diff --git a/src/darker/highlighting/lexers.py b/src/darker/highlighting/lexers.py deleted file mode 100644 index c9212050d..000000000 --- a/src/darker/highlighting/lexers.py +++ /dev/null @@ -1,88 +0,0 @@ -"""Custom Pygments lexers for highlighting linter output""" - -from typing import Generator, Tuple - -from pygments.lexer import Lexer, RegexLexer, bygroups, combined -from pygments.lexers.python import PythonLexer -from pygments.token import Error, Number, String, Text, _TokenType - - -class LocationLexer(Lexer): - """Lexer for linter output ``path:line:col:`` prefix""" - - aliases = ["lint_location"] - - def get_tokens_unprocessed( - self, text: str - ) -> Generator[Tuple[int, _TokenType, str], None, None]: - """Tokenize and generate (index, tokentype, value) tuples for highlighted tokens - - "index" is the starting position of the token within the input text. - - """ - path, *positions = text.split(":") - yield 0, String, path - pos = len(path) - for position in positions: - yield pos, Text, ":" - yield pos + 1, Number, position - pos += 1 + len(position) - - -class DescriptionLexer(RegexLexer): - """Lexer for linter output descriptions - - Highlights embedded Python code and expressions using the Python 3 lexer. - - """ - - aliases = "lint_description" - - # Make normal text in linter messages look like strings in source code. - # This is a decent choice since it lets source code stand out fairly well. - message = String - - # Customize the Python lexer - tokens = PythonLexer.tokens.copy() - - # Move the main Python lexer into a separate state - tokens["python"] = tokens["root"] - tokens["python"].insert(0, ('"', message, "#pop")) - tokens["python"].insert(0, ("'", message, "#pop")) - - # The root state manages a possible prefix for the description. - # It highlights error codes, and also catches coverage output and assumes that - # Python code follows and uses the Python lexer to highlight that. - tokens["root"] = [ - (r"\s*no coverage: ", message, "python"), - (r"[CEFNW]\d{3,4}\b|error\b", Error, "description"), - (r"", Text, "description"), - ] - - # Highlight a single space-separated word using the Python lexer - tokens["one-python-identifier"] = [ - (" ", message, "#pop"), - ] - - # The description state handles everything after the description prefix - tokens["description"] = [ - # Highlight quoted expressions using the Python lexer. - ('"', message, combined("python", "dqs")), - ("'", message, combined("python", "sqs")), - # Also catch a few common patterns which are followed by Python expressions, - # but exclude a couple of special cases. - (r"\bUnused (argument|variable) ", message), - ( - r"\b(Returning|Unused|Base type|imported from) ", - message, - combined("one-python-identifier", "python"), - ), - # Highlight parenthesized message identifiers at the end of messages - ( - r"(\()([a-z][a-z-]+[a-z])(\))(\s*)$", - bygroups(message, Error, message, message), - ), - # Everything else is considered just plain non-highlighted text - (r"\s+", message), - (r"\S+", message), - ] diff --git a/src/darker/tests/test_highlighting.py b/src/darker/tests/test_highlighting.py deleted file mode 100644 index c0d8cb608..000000000 --- a/src/darker/tests/test_highlighting.py +++ /dev/null @@ -1,657 +0,0 @@ -"""Unit tests for :mod:`darker.highlighting`""" - -# pylint: disable=too-many-arguments,redefined-outer-name,unused-argument -# pylint: disable=protected-access - -import os -import sys -from pathlib import Path -from shlex import shlex -from typing import Dict, Generator -from unittest.mock import patch - -import pytest -from _pytest.fixtures import SubRequest -from pygments.token import Token - -from darker.command_line import parse_command_line -from darker.highlighting import colorize, lexers, should_use_color - -RESET = "\x1b[39;49;00m" -RED = "\x1b[31;01m" -GREEN = "\x1b[32m" -YELLOW = "\x1b[33m" -BLUE = "\x1b[34m" -CYAN = "\x1b[36m" -WHITE = "\x1b[37m" -BR_RED = "\x1b[91m" - - -@pytest.fixture(scope="module") -def module_tmp_path(tmp_path_factory: pytest.TempPathFactory) -> Path: - """Fixture for creating a module-scope temporary directory - - :param tmp_path_factory: The temporary path factory fixture from Pytest - :return: The created directory path - - """ - return tmp_path_factory.mktemp("test_highlighting") - - -def unset_our_env_vars(): - """Unset the environment variables used in this test module""" - os.environ.pop("PY_COLORS", None) - os.environ.pop("NO_COLOR", None) - os.environ.pop("FORCE_COLOR", None) - - -@pytest.fixture(scope="module", autouse=True) -def clean_environ(): - """Fixture for clearing unwanted environment variables - - The ``NO_COLOR`` and ``PY_COLORS`` environment variables are tested in this module, - so we need to ensure they aren't already set. - - In all `os.environ` patching, we use our own low-level custom code instead of - `unittest.mock.patch.dict` for performance reasons. - - """ - old = os.environ - os.environ = old.copy() # type: ignore # noqa: B003 - unset_our_env_vars() - - yield - - os.environ = old # noqa: B003 - - -@pytest.fixture(params=["", "color = false", "color = true"]) -def pyproject_toml_color( - request: SubRequest, module_tmp_path: Path -) -> Generator[None, None, None]: - """Parametrized fixture for the ``color =`` option in ``pyproject.toml`` - - Creates three versions of ``pyproject.toml`` in ``module_tmp_path`` for a test - function: - - Without the ``color =`` option:: - - [tool.darker] - - With color turned off:: - - [tool.darker] - color = false - - With color turned on:: - - [tool.darker] - color = true - - :param request: The Pytest ``request`` object - :param module_tmp_path: A temporary directory created by Pytest - :yield: The ``color =`` option line in ``pyproject.toml``, or an empty string - - """ - pyproject_toml_path = module_tmp_path / "pyproject.toml" - with pyproject_toml_path.open("w") as pyproject_toml: - print(f"[tool.darker]\n{request.param}\n", file=pyproject_toml) - - yield request.param - - pyproject_toml_path.unlink() - - -@pytest.fixture(params=["", "tty"]) -def tty(request: SubRequest) -> Generator[bool, None, None]: - """Parametrized fixture for patching `sys.stdout.isatty` - - Patches `sys.stdout.isatty` to return either `False` or `True`. The parameter values - are strings and not booleans in order to improve readability of parametrized tests. - Custom patching for performance. - - :param request: The Pytest ``request`` object - :yield: The patched `False` or `True` return value for `sys.stdout.isatty` - - """ - old_isatty = sys.stdout.isatty - is_a_tty: bool = request.param == "tty" - sys.stdout.isatty = lambda: is_a_tty # type: ignore[method-assign] - - yield is_a_tty - - sys.stdout.isatty = old_isatty # type: ignore[method-assign] - - -def _parse_environment_variables(definitions: str) -> Dict[str, str]: - """Parse a ``"= ="`` formatted string into a dictionary - - :param definitions: The string to parse - :return: The parsed dictionary - - """ - return dict(item.split("=") for item in shlex(definitions, punctuation_chars=" ")) - - -@pytest.fixture(params=["", "NO_COLOR=", "NO_COLOR=foo"]) -def env_no_color(request: SubRequest) -> Generator[Dict[str, str], None, None]: - """Parametrized fixture for patching ``NO_COLOR`` - - This fixture must come before `config_from_env_and_argv` in test function - signatures. - - Patches the environment with or without the ``NO_COLOR`` environment variable. The - environment is expressed as a space-separated string to improve readability of - parametrized tests. - - :param request: The Pytest ``request`` object - :yield: The patched items in the environment - - """ - os.environ.update(_parse_environment_variables(request.param)) - yield request.param - unset_our_env_vars() - - -@pytest.fixture(params=["", "FORCE_COLOR=", "FORCE_COLOR=foo"]) -def env_force_color(request: SubRequest) -> Generator[Dict[str, str], None, None]: - """Parametrized fixture for patching ``FORCE_COLOR`` - - This fixture must come before `config_from_env_and_argv` in test function - signatures. - - Patches the environment with or without the ``FORCE_COLOR`` environment variable. - The environment is expressed as a space-separated string to improve readability of - parametrized tests. - - :param request: The Pytest ``request`` object - :yield: The patched items in the environment - - """ - os.environ.update(_parse_environment_variables(request.param)) - yield request.param - unset_our_env_vars() - - -@pytest.fixture(params=["", "PY_COLORS=0", "PY_COLORS=1"]) -def env_py_colors(request: SubRequest) -> Generator[Dict[str, str], None, None]: - """Parametrized fixture for patching ``PY_COLORS`` - - This fixture must come before `config_from_env_and_argv` in test function - signatures. - - Patches the environment with or without the ``PY_COLORS`` environment variable. The - environment is expressed as a space-separated string to improve readability of - parametrized tests. - - :param request: The Pytest ``request`` object - :yield: The patched items in the environment - - """ - os.environ.update(_parse_environment_variables(request.param)) - yield request.param - unset_our_env_vars() - - -@pytest.fixture -def uninstall_pygments() -> Generator[None, None, None]: - """Fixture for uninstalling ``pygments`` temporarily""" - mods = sys.modules.copy() - del mods["darker.highlighting"] - # cause an ImportError for `import pygments`: - mods["pygments"] = None # type: ignore[assignment] - with patch.dict(sys.modules, mods, clear=True): - - yield - - -config_cache = {} - - -@pytest.fixture(params=[[], ["--no-color"], ["--color"]]) -def config_from_env_and_argv( - request: SubRequest, module_tmp_path: Path -) -> Generator[bool, None, None]: - """Parametrized fixture for the ``--color`` / ``--no-color`` arguments - - Yields ``color`` configuration boolean values resulting from the current environment - variables and a command line - - with no color argument, - - with the ``--color`` argument, and - - with the ``--no--color`` argument. - - The ``NO_COLOR`` and ``PY_COLORS`` environment variables affect the resulting - configuration, and must precede `config_from_env_and_argv` in test function - signatures (if they are being used). - - :param request: The Pytest ``request`` object - :param module_tmp_path: A temporary directory created by Pytest - :yield: The list of arguments for the Darker command line - - """ - argv = request.param + [str(module_tmp_path / "dummy.py")] - cache_key = ( - tuple(request.param), - os.getenv("NO_COLOR"), - os.getenv("FORCE_COLOR"), - os.getenv("PY_COLORS"), - (module_tmp_path / "pyproject.toml").read_bytes(), - ) - if cache_key not in config_cache: - _, config, _ = parse_command_line(argv) - config_cache[cache_key] = config["color"] - yield config_cache[cache_key] - - -def test_should_use_color_no_pygments( - uninstall_pygments: None, - pyproject_toml_color: str, - env_no_color: str, - env_force_color: str, - env_py_colors: str, - config_from_env_and_argv: bool, - tty: bool, -) -> None: - """Color output is never used if `pygments` is not installed - - All combinations of ``pyproject.toml`` options, environment variables and command - line options affecting syntax highlighting are tested without `pygments`. - - """ - result = should_use_color(config_from_env_and_argv) - - assert result is False - - -@pytest.mark.parametrize( - "config_from_env_and_argv, expect", - [(["--no-color"], False), (["--color"], True)], - indirect=["config_from_env_and_argv"], -) -def test_should_use_color_pygments_and_command_line_argument( - pyproject_toml_color: str, - env_no_color: str, - env_force_color: str, - env_py_colors: str, - config_from_env_and_argv: bool, - expect: bool, - tty: bool, -) -> None: - """--color / --no-color determines highlighting if `pygments` is installed - - All combinations of ``pyproject.toml`` options, environment variables and command - line options affecting syntax highlighting are tested with `pygments` installed. - - """ - result = should_use_color(config_from_env_and_argv) - - assert result == expect - - -@pytest.mark.parametrize( - "env_py_colors, expect", - [("PY_COLORS=0", False), ("PY_COLORS=1", True)], - indirect=["env_py_colors"], -) -@pytest.mark.parametrize("config_from_env_and_argv", [[]], indirect=True) -def test_should_use_color_pygments_and_py_colors( - pyproject_toml_color: str, - env_no_color: str, - env_force_color: str, - env_py_colors: str, - config_from_env_and_argv: bool, - tty: bool, - expect: bool, -) -> None: - """PY_COLORS determines highlighting when `pygments` installed and no cmdline args - - These tests are set up so that it appears as if - - ``pygments`` is installed - - there is no ``--color`` or `--no-color`` command line option - - All combinations of ``pyproject.toml`` options and environment variables affecting - syntax highlighting are tested. - - """ - result = should_use_color(config_from_env_and_argv) - - assert result == expect - - -@pytest.mark.parametrize( - "env_no_color, env_force_color, expect", - [ - (" ", "FORCE_COLOR= ", "should_use_color() == True"), - (" ", "FORCE_COLOR=foo", "should_use_color() == True"), - ("NO_COLOR= ", "FORCE_COLOR= ", " "), - ("NO_COLOR= ", "FORCE_COLOR=foo", " "), - ("NO_COLOR=foo", "FORCE_COLOR= ", " "), - ("NO_COLOR=foo", "FORCE_COLOR=foo", " "), - ], - indirect=["env_no_color", "env_force_color"], -) -@pytest.mark.parametrize("config_from_env_and_argv", [[]], indirect=True) -def test_should_use_color_no_color_force_color( - pyproject_toml_color: str, - env_no_color: str, - env_force_color: str, - config_from_env_and_argv: bool, - tty: bool, - expect: str, -) -> None: - """NO_COLOR/FORCE_COLOR determine highlighting in absence of PY_COLORS/cmdline args - - These tests are set up so that it appears as if - - ``pygments`` is installed - - the ``PY_COLORS`` environment variable is unset - - there is no ``--color`` or `--no-color`` command line option - - All combinations of ``pyproject.toml`` options, ``NO_COLOR``, ``FORCE_COLOR`` and - `sys.stdout.isatty` are tested. - - """ - result = should_use_color(config_from_env_and_argv) - - assert result == (expect == "should_use_color() == True") - - -@pytest.mark.parametrize("config_from_env_and_argv", [[]], indirect=True) -@pytest.mark.parametrize( - "pyproject_toml_color, tty, expect", - [ - # for readability, padded strings are used for parameters and the expectation - (" ", " ", " "), - (" ", "tty", "should_use_color() == True"), - ("color = false", " ", " "), - ("color = false", "tty", " "), - ("color = true ", " ", "should_use_color() == True"), - ("color = true ", "tty", "should_use_color() == True"), - ], - indirect=["pyproject_toml_color", "tty"], -) -def test_should_use_color_pygments( - pyproject_toml_color: str, - tty: bool, - config_from_env_and_argv: bool, - expect: str, -) -> None: - """Color output is enabled only if correct configuration options are in place - - These tests are set up so that it appears as if - - ``pygments`` is installed (required for running the tests) - - there is no ``--color`` or `--no-color`` command line option - - the ``PY_COLORS`` environment variable isn't set to ``0`` or ``1`` (cleared by - the auto-use ``clear_environ`` fixture) - - This test exercises the remaining combinations of ``pyproject.toml`` options and - environment variables affecting syntax highlighting. - - """ - result = should_use_color(config_from_env_and_argv) - - assert result == (expect == "should_use_color() == True") - - -def test_colorize_with_no_color(): - """``colorize()`` does nothing when Pygments isn't available""" - result = colorize("print(42)", "python", use_color=False) - - assert result == "print(42)" - - -@pytest.mark.parametrize( - "text, lexer, use_color, expect", - [ - ( - "except RuntimeError:", - "python", - True, - { - # Pygments <2.14.0: - f"{BLUE}except{RESET} {CYAN}RuntimeError{RESET}:", - # Pygments 2.14.0: - f"{BLUE}except{RESET} {CYAN}RuntimeError{RESET}:{WHITE}{RESET}", - }, - ), - ("except RuntimeError:", "python", False, {"except RuntimeError:"}), - ( - "a = 1", - "python", - True, - { - # Pygments <2.14.0: - f"a = {BLUE}1{RESET}", - # Pygments 2.14.0: - f"a = {BLUE}1{RESET}{WHITE}{RESET}", - }, - ), - ( - "a = 1\n", - "python", - True, - { - # Pygments <2.14.0: - f"a = {BLUE}1{RESET}\n", - # Pygments 2.14.0: - f"a = {BLUE}1{RESET}{WHITE}{RESET}\n", - }, - ), - ( - "- a\n+ b\n", - "diff", - True, - { - # Pygments 2.4.0: - f"{RED}- a{RESET}\n{GREEN}+ b{RESET}\n", - # Pygments 2.10.0: - f"{BR_RED}- a{RESET}\n{GREEN}+ b{RESET}\n", - # Pygments 2.11.2: - f"{BR_RED}- a{RESET}{WHITE}{RESET}\n" - f"{GREEN}+ b{RESET}{WHITE}{RESET}\n", - }, - ), - ( - "- a\n+ b\n", - "diff", - False, - {"- a\n+ b\n"}, - ), - ], -) -def test_colorize(text, lexer, use_color, expect): - """``colorize()`` produces correct highlighted terminal output""" - result = colorize(text, lexer, use_color) - - assert result in expect - - -@pytest.mark.parametrize( - "text, expect", - [ - ( - "path/to/file.py:42:", - [ - (0, Token.Literal.String, "path/to/file.py"), - (15, Token.Text, ":"), - (16, Token.Literal.Number, "42"), - (18, Token.Text, ":"), - (19, Token.Literal.Number, ""), - ], - ), - ( - "path/to/file.py:42:43:", - [ - (0, Token.Literal.String, "path/to/file.py"), - (15, Token.Text, ":"), - (16, Token.Literal.Number, "42"), - (18, Token.Text, ":"), - (19, Token.Literal.Number, "43"), - (21, Token.Text, ":"), - (22, Token.Literal.Number, ""), - ], - ), - ], -) -def test_location_lexer(text, expect): - """Linter "path:linenum:colnum:" prefixes are lexed correctly""" - location_lexer = lexers.LocationLexer() - - result = list(location_lexer.get_tokens_unprocessed(text)) - - assert result == expect - - -@pytest.mark.parametrize( - "text, expect", - [ - ( - " no coverage: a = 1", - [ - (0, Token.Literal.String, " no coverage: "), - (15, Token.Text, " "), - (19, Token.Name, "a"), - (20, Token.Text, " "), - (21, Token.Operator, "="), - (22, Token.Text, " "), - (23, Token.Literal.Number.Integer, "1"), - ], - ), - ( - "C000 python(code) = not(highlighted)", - [ - (0, Token.Error, "C000"), - (4, Token.Literal.String, " "), - (5, Token.Literal.String, "python(code)"), - (17, Token.Literal.String, " "), - (18, Token.Literal.String, "="), - (19, Token.Literal.String, " "), - (20, Token.Literal.String, "not(highlighted)"), - ], - ), - ( - "C0000 Unused argument not highlighted", - [ - (0, Token.Error, "C0000"), - (5, Token.Literal.String, " "), - (6, Token.Literal.String, "Unused argument "), - (22, Token.Literal.String, "not"), - (25, Token.Literal.String, " "), - (26, Token.Literal.String, "highlighted"), - ], - ), - ( - "E000 Unused variable not highlighted", - [ - (0, Token.Error, "E000"), - (4, Token.Literal.String, " "), - (5, Token.Literal.String, "Unused variable "), - (21, Token.Literal.String, "not"), - (24, Token.Literal.String, " "), - (25, Token.Literal.String, "highlighted"), - ], - ), - ( - "E0000 Returning python_expression - is highlighted", - [ - (0, Token.Error, "E0000"), - (5, Token.Literal.String, " "), - (6, Token.Literal.String, "Returning "), - (16, Token.Name, "python_expression"), - (33, Token.Literal.String, " "), - (34, Token.Literal.String, "-"), - (35, Token.Literal.String, " "), - (36, Token.Literal.String, "is"), - (38, Token.Literal.String, " "), - (39, Token.Literal.String, "highlighted"), - ], - ), - ( - "F000 Unused python_expression_highlighted", - [ - (0, Token.Error, "F000"), - (4, Token.Literal.String, " "), - (5, Token.Literal.String, "Unused "), - (12, Token.Name, "python_expression_highlighted"), - ], - ), - ( - "F0000 Base type PythonClassHighlighted whatever", - [ - (0, Token.Error, "F0000"), - (5, Token.Literal.String, " "), - (6, Token.Literal.String, "Base type "), - (16, Token.Name, "PythonClassHighlighted"), - (38, Token.Literal.String, " "), - (39, Token.Literal.String, "whatever"), - ], - ), - ( - "N000 imported from python.module.highlighted", - [ - (0, Token.Error, "N000"), - (4, Token.Literal.String, " "), - (5, Token.Literal.String, "imported from "), - (19, Token.Name, "python"), - (25, Token.Operator, "."), - (26, Token.Name, "module"), - (32, Token.Operator, "."), - (33, Token.Name, "highlighted"), - ], - ), - ( - "N0000 (message-identifier) not-highlighted-in-the-middle", - [ - (0, Token.Error, "N0000"), - (5, Token.Literal.String, " "), - (6, Token.Literal.String, "(message-identifier)"), - (26, Token.Literal.String, " "), - (27, Token.Literal.String, "not-highlighted-in-the-middle"), - ], - ), - ( - "W000 at-the-end-highlight (message-identifier)", - [ - (0, Token.Error, "W000"), - (4, Token.Literal.String, " "), - (5, Token.Literal.String, "at-the-end-highlight"), - (25, Token.Literal.String, " "), - (26, Token.Literal.String, "("), - (27, Token.Error, "message-identifier"), - (45, Token.Literal.String, ")"), - ], - ), - ( - "W0000 four-digit-warning", - [ - (0, Token.Error, "W0000"), - (5, Token.Literal.String, " "), - (6, Token.Literal.String, "four-digit-warning"), - ], - ), - ( - "E00 two-digit-message-id-not-highlighted", - [ - (0, Token.Text, ""), - (0, Token.Literal.String, "E00"), - (3, Token.Literal.String, " "), - (4, Token.Literal.String, "two-digit-message-id-not-highlighted"), - ], - ), - ( - "E00000 five-digit-message-id-not-highlighted", - [ - (0, Token.Text, ""), - (0, Token.Literal.String, "E00000"), - (6, Token.Literal.String, " "), - (7, Token.Literal.String, "five-digit-message-id-not-highlighted"), - ], - ), - ], -) -def test_description_lexer(text, expect): - """The description parts of linter output are lexed correctly""" - description_lexer = lexers.DescriptionLexer() - - result = list(description_lexer.get_tokens_unprocessed(text)) - - assert result == expect diff --git a/src/darker/tests/test_main.py b/src/darker/tests/test_main.py index 53f62d6af..a08d10680 100644 --- a/src/darker/tests/test_main.py +++ b/src/darker/tests/test_main.py @@ -24,10 +24,10 @@ from darker.git import WORKTREE, EditedLinenumsDiffer from darker.tests.helpers import isort_present from darker.tests.test_fstring import FLYNTED_SOURCE, MODIFIED_SOURCE, ORIGINAL_SOURCE -from darker.tests.test_highlighting import BLUE, CYAN, RESET, WHITE, YELLOW from darker.utils import joinlines from darker.verification import NotEquivalentError from darkgraylib.git import RevisionRange +from darkgraylib.tests.test_highlighting import BLUE, CYAN, RESET, WHITE, YELLOW from darkgraylib.utils import TextDocument From 825fa7acb7b527b796c60ac5686d2e597f061431 Mon Sep 17 00:00:00 2001 From: Antti Kaihola <13725+akaihola@users.noreply.github.com> Date: Mon, 7 Aug 2023 09:39:09 +0300 Subject: [PATCH 21/30] Move parts of `darker.diff` into `darkgraylib` --- src/darker/diff.py | 77 +------------------ src/darker/git.py | 3 +- src/darker/tests/test_diff.py | 138 ++-------------------------------- 3 files changed, 11 insertions(+), 207 deletions(-) diff --git a/src/darker/diff.py b/src/darker/diff.py index 7e5e96e01..b3c2020f4 100644 --- a/src/darker/diff.py +++ b/src/darker/diff.py @@ -65,51 +65,16 @@ """ import logging -from difflib import SequenceMatcher -from typing import Dict, Generator, List, Sequence, Tuple +from typing import Generator, List, Sequence, Tuple from darker.multiline_strings import find_overlap from darker.utils import DiffChunk +from darkgraylib.diff import diff_and_get_opcodes, validate_opcodes from darkgraylib.utils import TextDocument logger = logging.getLogger(__name__) -def diff_and_get_opcodes( - src: TextDocument, dst: TextDocument -) -> List[Tuple[str, int, int, int, int]]: - """Return opcodes and line numbers for chunks in the diff of two lists of strings - - The opcodes are 5-tuples for each chunk with - - - the tag of the operation ('equal', 'delete', 'replace' or 'insert') - - the number of the first line in the chunk in the from-file - - the number of the last line in the chunk in the from-file - - the number of the first line in the chunk in the to-file - - the number of the last line in the chunk in the to-file - - Line numbers are zero based. - - """ - matcher = SequenceMatcher(None, src.lines, dst.lines, autojunk=False) - opcodes = matcher.get_opcodes() - logger.debug( - "Diff between edited and reformatted has %s opcode%s", - len(opcodes), - "s" if len(opcodes) > 1 else "", - ) - return opcodes - - -def _validate_opcodes(opcodes: List[Tuple[str, int, int, int, int]]) -> None: - """Make sure every other opcode is an 'equal' tag""" - if not all( - (tag1 == "equal") != (tag2 == "equal") - for (tag1, _, _, _, _), (tag2, _, _, _, _) in zip(opcodes[:-1], opcodes[1:]) - ): - raise ValueError(f"Unexpected opcodes in {opcodes!r}") - - def opcodes_to_edit_linenums( # pylint: disable=too-many-locals opcodes: List[Tuple[str, int, int, int, int]], context_lines: int, @@ -132,7 +97,7 @@ def opcodes_to_edit_linenums( # pylint: disable=too-many-locals """ if not opcodes: return - _validate_opcodes(opcodes) + validate_opcodes(opcodes) # Calculate the last line number beyond which we won't extend with extra context # lines @@ -175,7 +140,7 @@ def opcodes_to_chunks( lines for each chunk and concatenating them together. """ - _validate_opcodes(opcodes) + validate_opcodes(opcodes) for _tag, src_start, src_end, dst_start, dst_end in opcodes: yield src_start + 1, src.lines[src_start:src_end], dst.lines[dst_start:dst_end] @@ -207,37 +172,3 @@ def diff_chunks(src: TextDocument, dst: TextDocument) -> List[DiffChunk]: # 5. convert the diff into chunks, keeping original and reformatted content for each # chunk return list(opcodes_to_chunks(opcodes, src, dst)) - - -def map_unmodified_lines(src: TextDocument, dst: TextDocument) -> Dict[int, int]: - """Return a mapping of line numbers of unmodified lines between dst and src docs - - After doing a diff between ``src`` and ``dst``, some identical chunks of lines may - be identified. For each such chunk, a mapping from every line number of the chunk in - ``dst`` to corresponding line number in ``src`` is added. - - :param src: The original text document - :param dst: The modified text document - :return: A mapping from ``dst`` lines to corresponding unmodified ``src`` lines. - Line numbers are 1-based. - :raises RuntimeError: if blocks in opcodes don't make sense - - """ - opcodes = diff_and_get_opcodes(src, dst) - _validate_opcodes(opcodes) - if not src.string and not dst.string: - # empty files may get linter messages on line 1 - return {1: 1} - result = {} - for tag, src_start, src_end, dst_start, dst_end in opcodes: - if tag != "equal": - continue - for line_delta in range(dst_end - dst_start): - result[dst_start + line_delta + 1] = src_start + line_delta + 1 - if line_delta != src_end - src_start - 1: - raise RuntimeError( - "Something is wrong, 'equal' diff blocks should have the same length." - f" src_start={src_start}, src_end={src_end}," - f" dst_start={dst_start}, dst_end={dst_end}" - ) - return result diff --git a/src/darker/git.py b/src/darker/git.py index d4512f3d2..1fac38ac0 100644 --- a/src/darker/git.py +++ b/src/darker/git.py @@ -25,9 +25,10 @@ overload, ) -from darker.diff import diff_and_get_opcodes, opcodes_to_edit_linenums +from darker.diff import opcodes_to_edit_linenums from darker.multiline_strings import get_multiline_string_ranges from darker.utils import GIT_DATEFORMAT +from darkgraylib.diff import diff_and_get_opcodes from darkgraylib.git import RevisionRange from darkgraylib.utils import TextDocument diff --git a/src/darker/tests/test_diff.py b/src/darker/tests/test_diff.py index ecd8ebe6a..4bce26b4f 100644 --- a/src/darker/tests/test_diff.py +++ b/src/darker/tests/test_diff.py @@ -3,112 +3,19 @@ # pylint: disable=use-dict-literal from itertools import chain -from textwrap import dedent import pytest from darker.diff import ( - diff_and_get_opcodes, - map_unmodified_lines, opcodes_to_chunks, opcodes_to_edit_linenums, ) -from darkgraylib.utils import TextDocument - -FUNCTIONS2_PY = dedent( - """\ - def f( - a, - **kwargs, - ) -> A: - with cache_dir(): - if something: - result = ( - CliRunner().invoke(black.main, [str(src1), str(src2), "--diff", "--check"]) - ) - limited.append(-limited.pop()) # negate top - return A( - very_long_argument_name1=very_long_value_for_the_argument, - very_long_argument_name2=-very.long.value.for_the_argument, - **kwargs, - ) - def g(): - "Docstring." - def inner(): - pass - print("Inner defs should breathe a little.") - def h(): - def inner(): - pass - print("Inner defs should breathe a little.") -""" # noqa: E501 -) - - -FUNCTIONS2_PY_REFORMATTED = dedent( - """\ - def f( - a, - **kwargs, - ) -> A: - with cache_dir(): - if something: - result = CliRunner().invoke( - black.main, [str(src1), str(src2), "--diff", "--check"] - ) - limited.append(-limited.pop()) # negate top - return A( - very_long_argument_name1=very_long_value_for_the_argument, - very_long_argument_name2=-very.long.value.for_the_argument, - **kwargs, - ) - - - def g(): - "Docstring." - - def inner(): - pass - - print("Inner defs should breathe a little.") - - - def h(): - def inner(): - pass - - print("Inner defs should breathe a little.") - """ +from darkgraylib.tests.test_diff import ( + EXPECT_OPCODES, + FUNCTIONS2_PY, + FUNCTIONS2_PY_REFORMATTED, ) - - -EXPECT_OPCODES = [ - ("equal", 0, 1, 0, 1), - ("replace", 1, 3, 1, 3), - ("equal", 3, 6, 3, 6), - ("replace", 6, 8, 6, 8), - ("equal", 8, 15, 8, 15), - ("insert", 15, 15, 15, 17), - ("equal", 15, 17, 17, 19), - ("insert", 17, 17, 19, 20), - ("equal", 17, 19, 20, 22), - ("insert", 19, 19, 22, 23), - ("equal", 19, 20, 23, 24), - ("insert", 20, 20, 24, 26), - ("equal", 20, 23, 26, 29), - ("insert", 23, 23, 29, 30), - ("equal", 23, 24, 30, 31), -] - - -def test_diff_and_get_opcodes(): - """``diff_and_get_opcodes()`` produces correct opcodes for the example sources""" - src = TextDocument.from_str(FUNCTIONS2_PY) - dst = TextDocument.from_str(FUNCTIONS2_PY_REFORMATTED) - - opcodes = diff_and_get_opcodes(src, dst) - - assert opcodes == EXPECT_OPCODES +from darkgraylib.utils import TextDocument def test_opcodes_to_chunks(): @@ -269,38 +176,3 @@ def test_opcodes_to_edit_linenums_empty_opcodes(): ) assert result == [] # pylint: disable=use-implicit-booleaness-not-comparison - - -@pytest.mark.kwparametrize( - dict( - expect={1: 1}, - ), - dict( - lines2=["file", "was", "empty", "but", "eventually", "not"], - expect={}, - ), - dict( - lines1=["file", "had", "content", "but", "becomes", "empty"], - expect={}, - ), - dict( - lines1=["1 unmoved", "2 modify", "3 to 4 moved"], - lines2=["1 unmoved", "2 modified", "3 inserted", "3 to 4 moved"], - expect={1: 1, 4: 3}, - ), - dict( - lines1=["can't", "follow", "both", "when", "order", "is", "changed"], - lines2=["when", "order", "is", "changed", "can't", "follow", "both"], - expect={1: 4, 2: 5, 3: 6, 4: 7}, - ), - lines1=[], - lines2=[], -) -def test_map_unmodified_lines(lines1, lines2, expect): - """``map_unmodified_lines`` returns a 1-based mapping from new to old linenums""" - doc1 = TextDocument.from_lines(lines1) - doc2 = TextDocument.from_lines(lines2) - - result = map_unmodified_lines(doc1, doc2) - - assert result == expect From 68fe8486e4a188e6a20e7d3f4045cdb24c8278d6 Mon Sep 17 00:00:00 2001 From: Antti Kaihola <13725+akaihola@users.noreply.github.com> Date: Sat, 2 Mar 2024 20:53:51 +0200 Subject: [PATCH 22/30] Update `--help` output in README --- README.rst | 41 +++++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/README.rst b/README.rst index 0383eb5a3..740c81d0c 100644 --- a/README.rst +++ b/README.rst @@ -323,6 +323,26 @@ The following `command line arguments`_ can also be used to modify the defaults: revision range from the ``PRE_COMMIT_FROM_REF`` and ``PRE_COMMIT_TO_REF`` environment variables. If those are not found, Darker works against ``HEAD``. Also see ``--stdin-filename=`` for the ``:STDIN:`` special value. +--stdin-filename PATH + The path to the file when passing it through stdin. Useful so Darker can find the + previous version from Git. Only valid with ``--revision=..:STDIN:`` + (``HEAD..:STDIN:`` being the default if ``--stdin-filename`` is enabled). +-c PATH, --config PATH + Make ``darker``, ``black`` and ``isort`` read configuration from ``PATH``. Note + that other tools like ``flynt``, ``mypy``, ``pylint`` or ``flake8`` won't use + this configuration file. +-v, --verbose + Show steps taken and summarize modifications +-q, --quiet + Reduce amount of output +--color + Enable syntax highlighting even for non-terminal output. Overrides the + environment variable PY_COLORS=0 +--no-color + Disable syntax highlighting even for terminal output. Overrides the environment + variable PY_COLORS=1 +-W WORKERS, --workers WORKERS + How many parallel workers to allow, or ``0`` for one per core [default: 1] --diff Don't write the files back, just output a diff for each file on stdout. Highlight syntax if on a terminal and the ``pygments`` package is available, or if enabled @@ -331,10 +351,6 @@ The following `command line arguments`_ can also be used to modify the defaults: Force complete reformatted output to stdout, instead of in-place. Only valid if there's just one file to reformat. Highlight syntax if on a terminal and the ``pygments`` package is available, or if enabled by configuration. ---stdin-filename PATH - The path to the file when passing it through stdin. Useful so Darker can find the - previous version from Git. Only valid with ``--revision=..:STDIN:`` - (``HEAD..:STDIN:`` being the default if ``--stdin-filename`` is enabled). --check Don't write the files back, just return the status. Return code 0 means nothing would change. Return code 1 means some files would be reformatted. @@ -343,24 +359,11 @@ The following `command line arguments`_ can also be used to modify the defaults: -i, --isort Also sort imports using the ``isort`` package -L CMD, --lint CMD - Also run a linter on changed files. ``CMD`` can be a name or path of the linter + Run a linter on changed files. ``CMD`` can be a name or path of the linter binary, or a full quoted command line with the command and options. Linters read their configuration as normally, and aren't affected by ``-c`` / ``--config``. Linter output is syntax highlighted when the ``pygments`` package is available if run on a terminal and or enabled by explicitly (see ``--color``). --c PATH, --config PATH - Ask ``black`` and ``isort`` to read configuration from ``PATH``. Note that other - tools like flynt, Mypy, Pylint and Flake8 won't use this configuration file. --v, --verbose - Show steps taken and summarize modifications --q, --quiet - Reduce amount of output ---color - Enable syntax highlighting even for non-terminal output. Overrides the - environment variable PY_COLORS=0 ---no-color - Disable syntax highlighting even for terminal output. Overrides the environment - variable PY_COLORS=1 -S, --skip-string-normalization Don't normalize string quotes or prefixes --no-skip-string-normalization @@ -375,8 +378,6 @@ The following `command line arguments`_ can also be used to modify the defaults: -t VERSION, --target-version VERSION [py33|py34|py35|py36|py37|py38|py39|py310|py311|py312] Python versions that should be supported by Black's output. [default: per-file auto-detection] --W WORKERS, --workers WORKERS - How many parallel workers to allow, or ``0`` for one per core [default: 1] To change default values for these options for a given project, add a ``[tool.darker]`` or ``[tool.black]`` section to ``pyproject.toml`` in the From eb842ab34c8496c16e2f51cdac4ca8147d9465f4 Mon Sep 17 00:00:00 2001 From: Antti Kaihola <13725+akaihola@users.noreply.github.com> Date: Sat, 2 Mar 2024 20:54:28 +0200 Subject: [PATCH 23/30] Import testhelpers from new place in Darkgraylib --- src/darker/tests/test_diff.py | 2 +- src/darker/tests/test_main.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/darker/tests/test_diff.py b/src/darker/tests/test_diff.py index 4bce26b4f..ffdb1bb8e 100644 --- a/src/darker/tests/test_diff.py +++ b/src/darker/tests/test_diff.py @@ -10,7 +10,7 @@ opcodes_to_chunks, opcodes_to_edit_linenums, ) -from darkgraylib.tests.test_diff import ( +from darkgraylib.testtools.diff_helpers import ( EXPECT_OPCODES, FUNCTIONS2_PY, FUNCTIONS2_PY_REFORMATTED, diff --git a/src/darker/tests/test_main.py b/src/darker/tests/test_main.py index a08d10680..1e5b9ac2f 100644 --- a/src/darker/tests/test_main.py +++ b/src/darker/tests/test_main.py @@ -27,7 +27,7 @@ from darker.utils import joinlines from darker.verification import NotEquivalentError from darkgraylib.git import RevisionRange -from darkgraylib.tests.test_highlighting import BLUE, CYAN, RESET, WHITE, YELLOW +from darkgraylib.testtools.highlighting_helpers import BLUE, CYAN, RESET, WHITE, YELLOW from darkgraylib.utils import TextDocument From 1479fa4dba95850ee7d2d5651c22fdc4a9355748 Mon Sep 17 00:00:00 2001 From: Antti Kaihola <13725+akaihola@users.noreply.github.com> Date: Sun, 3 Mar 2024 17:05:03 +0200 Subject: [PATCH 24/30] Move `black_compat` module to Darkgraylib --- src/darker/__main__.py | 4 ++-- src/darker/black_compat.py | 21 --------------------- src/darker/import_sorting.py | 2 +- 3 files changed, 3 insertions(+), 24 deletions(-) delete mode 100644 src/darker/black_compat.py diff --git a/src/darker/__main__.py b/src/darker/__main__.py index 3a08f3fea..e0fbaff80 100644 --- a/src/darker/__main__.py +++ b/src/darker/__main__.py @@ -10,7 +10,6 @@ from pathlib import Path from typing import Collection, Generator, List, Optional, Tuple -import darker.black_compat from darker.black_diff import ( BlackConfig, filter_python_files, @@ -39,6 +38,7 @@ from darker.import_sorting import apply_isort, isort from darker.utils import GIT_DATEFORMAT, DiffChunk, debug_dump, glob_any from darker.verification import ASTVerifier, BinarySearch, NotEquivalentError +from darkgraylib.black_compat import find_project_root from darkgraylib.config import show_config_if_debug from darkgraylib.git import RevisionRange from darkgraylib.highlighting import colorize, should_use_color @@ -562,7 +562,7 @@ def main( # pylint: disable=too-many-locals,too-many-branches,too-many-statemen # In other modes, only reformat files which have been modified. if git_is_repository(root): # Get the modified files only. - repo_root = darker.black_compat.find_project_root([str(root)]) + repo_root = find_project_root([str(root)]) changed_files = { (repo_root / file).relative_to(root) for file in git_get_modified_python_files(paths, revrange, repo_root) diff --git a/src/darker/black_compat.py b/src/darker/black_compat.py deleted file mode 100644 index 81ad542c2..000000000 --- a/src/darker/black_compat.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Functions for maintaining compatibility with multiple Black versions""" - -from pathlib import Path -from typing import Any, Sequence, Tuple, cast - -from black import find_project_root as black_find_project_root - - -def find_project_root(srcs: Sequence[str]) -> Path: - """Hide changed return value type in Black behind this wrapper - - :param srcs: Files and directories to find the common root for - :return: Project root path - - """ - root = cast(Any, black_find_project_root(tuple(srcs or ["."]))) - if isinstance(root, tuple): - # Black >= 22.1 - return cast(Tuple[Path], root)[0] - # Black < 22 - return cast(Path, root) diff --git a/src/darker/import_sorting.py b/src/darker/import_sorting.py index 72584d316..d01dd55cd 100644 --- a/src/darker/import_sorting.py +++ b/src/darker/import_sorting.py @@ -4,11 +4,11 @@ from pathlib import Path from typing import Any, Collection, List, Optional, TypedDict -from darker.black_compat import find_project_root from darker.diff import diff_chunks from darker.exceptions import IncompatiblePackageError, MissingPackageError from darker.git import EditedLinenumsDiffer from darker.utils import DiffChunk, glob_any +from darkgraylib.black_compat import find_project_root from darkgraylib.utils import TextDocument try: From 3fca1dfa02652299752b744dbf21ae3df77f05d7 Mon Sep 17 00:00:00 2001 From: Antti Kaihola <13725+akaihola@users.noreply.github.com> Date: Sun, 3 Mar 2024 18:15:56 +0200 Subject: [PATCH 25/30] Move `argparse_helpers` and tests into Darkgraylib --- src/darker/argparse_helpers.py | 110 ---------------------- src/darker/tests/test_argparse_helpers.py | 103 -------------------- 2 files changed, 213 deletions(-) delete mode 100644 src/darker/argparse_helpers.py delete mode 100644 src/darker/tests/test_argparse_helpers.py diff --git a/src/darker/argparse_helpers.py b/src/darker/argparse_helpers.py deleted file mode 100644 index 9591c2e6b..000000000 --- a/src/darker/argparse_helpers.py +++ /dev/null @@ -1,110 +0,0 @@ -"""Custom formatter and action for argparse""" - -import logging -import re -import sys -from argparse import SUPPRESS, Action, ArgumentParser, HelpFormatter, Namespace -from textwrap import fill -from typing import Any, List, Optional, Sequence, Union - -WORD_RE = re.compile(r"\w") - - -def _fill_line(line: str, width: int, indent: str) -> str: - first_word_match = WORD_RE.search(line) - first_word_offset = first_word_match.start() if first_word_match else 0 - return fill( - line, - width, - initial_indent=indent, - subsequent_indent=indent + first_word_offset * " ", - ) - - -class NewlinePreservingFormatter(HelpFormatter): - """A command line help formatter which preserves newline characters""" - def _fill_text(self, text: str, width: int, indent: str) -> str: - if "\n" in text: - return "\n".join( - _fill_line(line, width, indent) for line in text.split("\n") - ) - return super()._fill_text(text, width, indent) - - -class OptionsForReadmeAction(Action): - """Implementation of the ``--options-for-readme`` argument - - This argparse action prints optional command line arguments in a format suitable for - inclusion in ``README.rst``. - - """ - - # pylint: disable=too-few-public-methods - - def __init__( - self, option_strings: List[str], dest: str = SUPPRESS, help: str = None - ): # pylint: disable=redefined-builtin - super().__init__(option_strings, dest, 0) - - def __call__( - self, - parser: ArgumentParser, - namespace: Namespace, - values: Optional[Union[str, Sequence[Any]]], - option_string: str = None, - ) -> None: - optional_arguments_group = next( - group - for group in parser._action_groups - # The group title for options differs between Python versions - if group.title in {"optional arguments", "options"} - ) - actions = [] - for action in optional_arguments_group._group_actions: - if action.dest in {"help", "version", "options_for_readme"}: - continue - if action.help is not None: - action.help = action.help.replace("`", "``") - actions.append(action) - formatter = HelpFormatter(parser.prog, max_help_position=7, width=88) - formatter.add_arguments(actions) - sys.stderr.write(formatter.format_help()) - parser.exit() - - -class LogLevelAction(Action): # pylint: disable=too-few-public-methods - """Support for command line actions which increment/decrement the log level""" - - def __init__( # pylint: disable=too-many-arguments - self, - option_strings: List[str], - dest: str, - const: int, - default: int = logging.WARNING, - required: bool = False, - help: str = None, # pylint: disable=redefined-builtin - metavar: str = None, - ): - super().__init__( - option_strings=option_strings, - dest=dest, - nargs=0, - const=const, - default=default, - required=required, - help=help, - metavar=metavar, - ) - - def __call__( - self, - parser: ArgumentParser, - namespace: Namespace, - values: Union[str, Sequence[Any], None], - option_string: str = None, - ) -> None: - current_level = getattr(namespace, self.dest, self.default) - new_level = current_level + self.const - new_level = max(new_level, logging.DEBUG) - new_level = min(new_level, logging.CRITICAL) - setattr(namespace, self.dest, new_level) diff --git a/src/darker/tests/test_argparse_helpers.py b/src/darker/tests/test_argparse_helpers.py deleted file mode 100644 index 8e332be15..000000000 --- a/src/darker/tests/test_argparse_helpers.py +++ /dev/null @@ -1,103 +0,0 @@ -"""Tests for the ``darker.argparse_helpers`` module""" - -# pylint: disable=use-dict-literal - -from argparse import ArgumentParser, Namespace -from logging import CRITICAL, DEBUG, ERROR, INFO, NOTSET, WARNING - -import pytest - -from darker import argparse_helpers -from darker.tests.helpers import raises_if_exception - - -@pytest.mark.kwparametrize( - dict(line="", width=0, expect=ValueError), - dict(line="", width=1, expect=[]), - dict( - line="lorem ipsum dolor sit amet", - width=9, - expect=[" lorem", " ipsum", " dolor", " sit", " amet"], - ), - dict( - line="lorem ipsum dolor sit amet", - width=15, - expect=[" lorem ipsum", " dolor sit", " amet"], - ), -) -def test_fill_line(line, width, expect): - """``_fill_line()`` wraps lines correctly""" - with raises_if_exception(expect): - - result = argparse_helpers._fill_line( # pylint: disable=protected-access - line, width, indent=" " - ) - - assert result.splitlines() == expect - - -@pytest.mark.kwparametrize( - dict( - text="lorem ipsum dolor sit amet", - expect=[" lorem ipsum", " dolor sit", " amet"], - ), - dict( - text="lorem\nipsum dolor sit amet", - expect=[" lorem", " ipsum dolor", " sit amet"], - ), -) -def test_newline_preserving_formatter(text, expect): - """``NewlinePreservingFormatter`` wraps lines and keeps newlines correctly""" - formatter = argparse_helpers.NewlinePreservingFormatter("dummy") - - result = formatter._fill_text( # pylint: disable=protected-access - text, width=15, indent=" " - ) - - assert result.splitlines() == expect - - -@pytest.mark.kwparametrize( - dict(const=10, initial=NOTSET, expect=DEBUG), - dict(const=10, initial=DEBUG, expect=INFO), - dict(const=10, initial=INFO, expect=WARNING), - dict(const=10, initial=WARNING, expect=ERROR), - dict(const=10, initial=ERROR, expect=CRITICAL), - dict(const=10, initial=CRITICAL, expect=CRITICAL), - dict(const=-10, initial=DEBUG, expect=DEBUG), - dict(const=-10, initial=INFO, expect=DEBUG), - dict(const=-10, initial=WARNING, expect=INFO), - dict(const=-10, initial=ERROR, expect=WARNING), - dict(const=-10, initial=CRITICAL, expect=ERROR), -) -def test_log_level_action(const, initial, expect): - """``LogLevelAction`` increments/decrements the log level value correctly""" - action = argparse_helpers.LogLevelAction([], "log_level", const) - parser = ArgumentParser() - namespace = Namespace() - namespace.log_level = initial - - action(parser, namespace, []) - - assert namespace.log_level == expect - - -@pytest.mark.kwparametrize( - dict(const=10, count=NOTSET, expect=WARNING), - dict(const=10, count=1, expect=ERROR), - dict(const=10, count=2, expect=CRITICAL), - dict(const=10, count=3, expect=CRITICAL), - dict(const=-10, count=NOTSET, expect=WARNING), - dict(const=-10, count=1, expect=INFO), - dict(const=-10, count=2, expect=DEBUG), - dict(const=-10, count=3, expect=DEBUG), -) -def test_argumentparser_log_level_action(const, count, expect): - """The log level action works correctly with an ``ArgumentParser``""" - parser = ArgumentParser() - parser.register("action", "log_level", argparse_helpers.LogLevelAction) - parser.add_argument("-l", dest="log_level", action="log_level", const=const) - - args = parser.parse_args(count * ["-l"]) - - assert args.log_level == expect From c75494277bd049541297b3b2168e90568801a6c9 Mon Sep 17 00:00:00 2001 From: Antti Kaihola <13725+akaihola@users.noreply.github.com> Date: Sun, 3 Mar 2024 18:24:55 +0200 Subject: [PATCH 26/30] Move `raises_if_exception()` to Darkgraylib --- src/darker/tests/helpers.py | 30 ++------------------ src/darker/tests/test_command_line.py | 3 +- src/darker/tests/test_config.py | 2 +- src/darker/tests/test_main_revision.py | 2 +- src/darker/tests/test_main_stdin_filename.py | 2 +- 5 files changed, 7 insertions(+), 32 deletions(-) diff --git a/src/darker/tests/helpers.py b/src/darker/tests/helpers.py index 3270e6072..00eb7c5e9 100644 --- a/src/darker/tests/helpers.py +++ b/src/darker/tests/helpers.py @@ -1,38 +1,12 @@ """Helper functions for unit tests""" -import re import sys -from contextlib import contextmanager, nullcontext +from contextlib import contextmanager from types import ModuleType -from typing import Any, ContextManager, Dict, Generator, List, Optional, Union +from typing import Dict, Generator, List, Optional from unittest.mock import patch import pytest -from _pytest.python_api import RaisesContext - - -def filter_dict(dct: Dict[str, Any], filter_key: str) -> Dict[str, Any]: - """Return only given keys with their values from a dictionary""" - return {key: value for key, value in dct.items() if key == filter_key} - - -def raises_if_exception(expect: Any) -> Union[RaisesContext[Any], ContextManager[None]]: - """Return a ``pytest.raises`` context manager only if expecting an exception - - If the expected value is not an exception, return a dummy context manager. - - """ - if (isinstance(expect, type) and issubclass(expect, BaseException)) or ( - isinstance(expect, tuple) - and all( - isinstance(item, type) and issubclass(item, BaseException) - for item in expect - ) - ): - return pytest.raises(expect) - if isinstance(expect, BaseException): - return pytest.raises(type(expect), match=re.escape(str(expect))) - return nullcontext() def matching_attrs(obj: BaseException, attrs: List[str]) -> Dict[str, int]: diff --git a/src/darker/tests/test_command_line.py b/src/darker/tests/test_command_line.py index f7b6f9cc9..a2b56fc28 100644 --- a/src/darker/tests/test_command_line.py +++ b/src/darker/tests/test_command_line.py @@ -18,10 +18,11 @@ from darker.__main__ import main from darker.command_line import make_argument_parser, parse_command_line from darker.config import Exclusions -from darker.tests.helpers import flynt_present, isort_present, raises_if_exception +from darker.tests.helpers import flynt_present, isort_present from darker.utils import joinlines from darkgraylib.config import ConfigurationError from darkgraylib.git import RevisionRange +from darkgraylib.testtools.helpers import raises_if_exception from darkgraylib.utils import TextDocument pytestmark = pytest.mark.usefixtures("find_project_root_cache_clear") diff --git a/src/darker/tests/test_config.py b/src/darker/tests/test_config.py index fc92ecb20..1470193ba 100644 --- a/src/darker/tests/test_config.py +++ b/src/darker/tests/test_config.py @@ -8,8 +8,8 @@ import pytest from darker.config import OutputMode -from darker.tests.helpers import raises_if_exception from darkgraylib.config import ConfigurationError +from darkgraylib.testtools.helpers import raises_if_exception @pytest.mark.kwparametrize( diff --git a/src/darker/tests/test_main_revision.py b/src/darker/tests/test_main_revision.py index 861764b4e..7d885f866 100644 --- a/src/darker/tests/test_main_revision.py +++ b/src/darker/tests/test_main_revision.py @@ -5,7 +5,7 @@ import pytest from darker.__main__ import main -from darker.tests.helpers import raises_if_exception +from darkgraylib.testtools.helpers import raises_if_exception # The following test is a bit dense, so some explanation is due. # diff --git a/src/darker/tests/test_main_stdin_filename.py b/src/darker/tests/test_main_stdin_filename.py index 7cfafc0d7..6f1080f57 100644 --- a/src/darker/tests/test_main_stdin_filename.py +++ b/src/darker/tests/test_main_stdin_filename.py @@ -10,9 +10,9 @@ import toml import darker.__main__ -from darker.tests.helpers import raises_if_exception from darkgraylib.config import ConfigurationError from darkgraylib.testtools.git_repo_plugin import GitRepoFixture +from darkgraylib.testtools.helpers import raises_if_exception pytestmark = pytest.mark.usefixtures("find_project_root_cache_clear") From a27b9cfb743709b0f556f2a9cf98a2b59767f664 Mon Sep 17 00:00:00 2001 From: Antti Kaihola <13725+akaihola@users.noreply.github.com> Date: Sun, 3 Mar 2024 19:37:35 +0200 Subject: [PATCH 27/30] Move `raises_or_matches()` to Darkgraylib --- src/darker/tests/helpers.py | 27 +-------------------------- src/darker/tests/test_black_diff.py | 5 ++--- src/darker/tests/test_git.py | 2 +- 3 files changed, 4 insertions(+), 30 deletions(-) diff --git a/src/darker/tests/helpers.py b/src/darker/tests/helpers.py index 00eb7c5e9..be10b29fa 100644 --- a/src/darker/tests/helpers.py +++ b/src/darker/tests/helpers.py @@ -3,34 +3,9 @@ import sys from contextlib import contextmanager from types import ModuleType -from typing import Dict, Generator, List, Optional +from typing import Generator, Optional from unittest.mock import patch -import pytest - - -def matching_attrs(obj: BaseException, attrs: List[str]) -> Dict[str, int]: - """Return object attributes whose name matches one in the given list""" - return {attname: getattr(obj, attname) for attname in dir(obj) if attname in attrs} - - -@contextmanager -def raises_or_matches(expect, match_exc_attrs): - """Helper for matching an expected value or an expected raised exception""" - if isinstance(expect, BaseException): - with pytest.raises(type(expect)) as exc_info: - # The lambda callback should never get called - yield lambda result: False - exception_attributes = matching_attrs(exc_info.value, match_exc_attrs) - expected_attributes = matching_attrs(expect, match_exc_attrs) - assert exception_attributes == expected_attributes - else: - - def check(result): - assert result == expect - - yield check - @contextmanager def _package_present( diff --git a/src/darker/tests/test_black_diff.py b/src/darker/tests/test_black_diff.py index efa2264cb..3cadf9c5f 100644 --- a/src/darker/tests/test_black_diff.py +++ b/src/darker/tests/test_black_diff.py @@ -6,12 +6,11 @@ import sys from dataclasses import dataclass, field from pathlib import Path -from typing import Dict, Iterable, Iterator, Optional, Pattern, TYPE_CHECKING +from typing import TYPE_CHECKING, Dict, Iterable, Iterator, Optional, Pattern from unittest.mock import ANY, Mock, call, patch import pytest import regex - from black import Mode, Report, TargetVersion from pathspec import PathSpec @@ -22,8 +21,8 @@ read_black_config, run_black, ) -from darker.tests.helpers import raises_or_matches from darkgraylib.config import ConfigurationError +from darkgraylib.testtools.helpers import raises_or_matches from darkgraylib.utils import TextDocument if sys.version_info >= (3, 11): diff --git a/src/darker/tests/test_git.py b/src/darker/tests/test_git.py index a0914ea3a..72b2594f5 100644 --- a/src/darker/tests/test_git.py +++ b/src/darker/tests/test_git.py @@ -15,10 +15,10 @@ import pytest from darker import git -from darker.tests.helpers import raises_or_matches from darker.utils import GIT_DATEFORMAT from darkgraylib.git import RevisionRange from darkgraylib.testtools.git_repo_plugin import GitRepoFixture +from darkgraylib.testtools.helpers import raises_or_matches from darkgraylib.utils import TextDocument From a3615b3ba8a5880afd0787703653b8eeaf176ccb Mon Sep 17 00:00:00 2001 From: Antti Kaihola <13725+akaihola@users.noreply.github.com> Date: Mon, 30 Jan 2023 22:23:36 +0200 Subject: [PATCH 28/30] Use `lru_cache` clear fixture at test module level --- src/darker/tests/test_import_sorting.py | 11 +++-------- src/darker/tests/test_main.py | 9 ++++----- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/src/darker/tests/test_import_sorting.py b/src/darker/tests/test_import_sorting.py index 9c6b99abb..bfefdd498 100644 --- a/src/darker/tests/test_import_sorting.py +++ b/src/darker/tests/test_import_sorting.py @@ -18,6 +18,8 @@ ORIGINAL_SOURCE = ("import sys", "import os", "", "print(42)") ISORTED_SOURCE = ("import os", "import sys", "", "print(42)") +pytestmark = pytest.mark.usefixtures("find_project_root_cache_clear") + @pytest.mark.parametrize("present", [True, False]) def test_import_sorting_importable_with_and_without_isort(present): @@ -134,14 +136,7 @@ def test_apply_isort_exclude(git_repo, encoding, newline, content, exclude, expe ), ), ) -def test_isort_config( - monkeypatch, - tmpdir, - find_project_root_cache_clear, - line_length, - settings_file, - expect, -): +def test_isort_config(monkeypatch, tmpdir, line_length, settings_file, expect): """``apply_isort()`` parses ``pyproject.toml``correctly""" monkeypatch.chdir(tmpdir) (tmpdir / "pyproject.toml").write( diff --git a/src/darker/tests/test_main.py b/src/darker/tests/test_main.py index 1e5b9ac2f..0d3ca7067 100644 --- a/src/darker/tests/test_main.py +++ b/src/darker/tests/test_main.py @@ -30,6 +30,8 @@ from darkgraylib.testtools.highlighting_helpers import BLUE, CYAN, RESET, WHITE, YELLOW from darkgraylib.utils import TextDocument +pytestmark = pytest.mark.usefixtures("find_project_root_cache_clear") + def randomword(length: int) -> str: """Create a random string of lowercase letters of a given length.""" @@ -56,7 +58,7 @@ def test_isort_option_without_isort(git_repo, caplog): @pytest.fixture -def run_isort(git_repo, monkeypatch, caplog, request, find_project_root_cache_clear): +def run_isort(git_repo, monkeypatch, caplog, request): """Fixture for running Darker with requested arguments and a patched `isort` Provides an `run_isort.isort_code` mock object which allows checking whether and how @@ -559,7 +561,6 @@ def test_main( git_repo, monkeypatch, capsys, - find_project_root_cache_clear, arguments, newline, pyproject_toml, @@ -662,9 +663,7 @@ def test_main_in_plain_directory(tmp_path, capsys): "encoding, text", [(b"utf-8", b"touch\xc3\xa9"), (b"iso-8859-1", b"touch\xe9")] ) @pytest.mark.parametrize("newline", [b"\n", b"\r\n"]) -def test_main_encoding( - git_repo, find_project_root_cache_clear, encoding, text, newline -): +def test_main_encoding(git_repo, encoding, text, newline): """Encoding and newline of the file is kept unchanged after reformatting""" paths = git_repo.add({"a.py": newline.decode("ascii")}, commit="Initial commit") edited = [b"# coding: ", encoding, newline, b's="', text, b'"', newline] From e230240d5c8bbd3d4a390c9fc67369314554adca Mon Sep 17 00:00:00 2001 From: Antti Kaihola <13725+akaihola@users.noreply.github.com> Date: Sun, 3 Mar 2024 19:39:45 +0200 Subject: [PATCH 29/30] Move some Git helpers to Darkgraylib --- src/darker/__main__.py | 12 +- src/darker/git.py | 256 +---------------- src/darker/tests/test_git.py | 517 +--------------------------------- src/darker/tests/test_main.py | 4 +- 4 files changed, 26 insertions(+), 763 deletions(-) diff --git a/src/darker/__main__.py b/src/darker/__main__.py index e0fbaff80..76c3702d6 100644 --- a/src/darker/__main__.py +++ b/src/darker/__main__.py @@ -24,13 +24,9 @@ from darker.exceptions import DependencyError, MissingPackageError from darker.fstring import apply_flynt, flynt from darker.git import ( - PRE_COMMIT_FROM_TO_REFS, - STDIN, - WORKTREE, EditedLinenumsDiffer, get_missing_at_revision, get_path_in_repo, - git_get_content_at_revision, git_get_modified_python_files, git_is_repository, ) @@ -40,7 +36,13 @@ from darker.verification import ASTVerifier, BinarySearch, NotEquivalentError from darkgraylib.black_compat import find_project_root from darkgraylib.config import show_config_if_debug -from darkgraylib.git import RevisionRange +from darkgraylib.git import ( + PRE_COMMIT_FROM_TO_REFS, + STDIN, + WORKTREE, + RevisionRange, + git_get_content_at_revision, +) from darkgraylib.highlighting import colorize, should_use_color from darkgraylib.log import setup_logging from darkgraylib.main import resolve_paths diff --git a/src/darker/git.py b/src/darker/git.py index 1fac38ac0..fff2eb1d6 100644 --- a/src/darker/git.py +++ b/src/darker/git.py @@ -1,35 +1,22 @@ """Helpers for listing modified files and getting unmodified content from Git""" import logging -import os -import re -import shlex -import sys -from contextlib import contextmanager from dataclasses import dataclass -from datetime import datetime from functools import lru_cache from pathlib import Path -from subprocess import DEVNULL, PIPE, CalledProcessError, check_output, run # nosec -from typing import ( - Dict, - Iterable, - Iterator, - List, - Match, - Optional, - Set, - Tuple, - Union, - cast, - overload, -) +from subprocess import DEVNULL, CalledProcessError, run # nosec +from typing import Iterable, List, Set from darker.diff import opcodes_to_edit_linenums from darker.multiline_strings import get_multiline_string_ranges -from darker.utils import GIT_DATEFORMAT from darkgraylib.diff import diff_and_get_opcodes -from darkgraylib.git import RevisionRange +from darkgraylib.git import ( + WORKTREE, + RevisionRange, + git_check_output_lines, + git_get_content_at_revision, + make_git_env, +) from darkgraylib.utils import TextDocument logger = logging.getLogger(__name__) @@ -39,55 +26,18 @@ # Handles these cases: # .. .. .. # ... ... ... -COMMIT_RANGE_RE = re.compile(r"(.*?)(\.{2,3})(.*)$") # A colon is an invalid character in tag/branch names. Use that in the special value for # - denoting the working tree as one of the "revisions" in revision ranges # - referring to the `PRE_COMMIT_FROM_REF` and `PRE_COMMIT_TO_REF` environment variables # for determining the revision range -WORKTREE = ":WORKTREE:" -STDIN = ":STDIN:" -PRE_COMMIT_FROM_TO_REFS = ":PRE-COMMIT:" - - -def git_get_version() -> Tuple[int, ...]: - """Return the Git version as a tuple of integers - - Ignores any suffixes to the dot-separated parts of the version string. - - :return: The version number of Git installed on the system - :raise: ``RuntimeError`` if unable to parse the Git version - - """ - output_lines = _git_check_output_lines(["--version"], Path(".")) - version_string = output_lines[0].rsplit(None, 1)[-1] - # The version string might be e.g. - # - "2.39.0.windows.1" - # - "2.36.2" - part_matches = [re.match(r"\d+", part) for part in version_string.split(".")][:3] - if all(part_matches): - return tuple( - int(match.group(0)) for match in cast(List[Match[str]], part_matches) - ) - raise RuntimeError(f"Unable to parse Git version: {output_lines!r}") - - -def git_rev_parse(revision: str, cwd: Path) -> str: - """Return the commit hash for the given revision - - :param revision: The revision to get the commit hash for - :param cwd: The root of the Git repository - :return: The commit hash for ``revision`` as parsed from Git output - - """ - return _git_check_output_lines(["rev-parse", revision], cwd)[0] def git_is_repository(path: Path) -> bool: """Return ``True`` if ``path`` is inside a Git working tree""" try: - lines = _git_check_output_lines( + lines = git_check_output_lines( ["rev-parse", "--is-inside-work-tree"], path, exit_on_error=False ) return lines[:1] == ["true"] @@ -99,52 +49,6 @@ def git_is_repository(path: Path) -> bool: return False -def git_get_mtime_at_commit(path: Path, revision: str, cwd: Path) -> str: - """Return the committer date of the given file at the given revision - - :param path: The relative path of the file in the Git repository - :param revision: The Git revision for which to get the file modification time - :param cwd: The root of the Git repository - - """ - cmd = ["log", "-1", "--format=%ct", revision, "--", path.as_posix()] - lines = _git_check_output_lines(cmd, cwd) - return datetime.utcfromtimestamp(int(lines[0])).strftime(GIT_DATEFORMAT) - - -def git_get_content_at_revision(path: Path, revision: str, cwd: Path) -> TextDocument: - """Get unmodified text lines of a file at a Git revision - - :param path: The relative path of the file in the Git repository - :param revision: The Git revision for which to get the file content, or ``WORKTREE`` - to get what's on disk right now. - :param cwd: The root of the Git repository - - """ - if path.is_absolute(): - raise ValueError( - f"the 'path' parameter must receive a relative path, got {path!r} instead" - ) - - if revision == WORKTREE: - abspath = cwd / path - return TextDocument.from_file(abspath) - cmd = ["show", f"{revision}:./{path.as_posix()}"] - try: - return TextDocument.from_bytes( - _git_check_output(cmd, cwd, exit_on_error=False), - mtime=git_get_mtime_at_commit(path, revision, cwd), - ) - except CalledProcessError as exc_info: - if exc_info.returncode != 128: - for error_line in exc_info.stderr.splitlines(): - logger.error(error_line) - raise - # The file didn't exist at the given revision. Act as if it was an empty - # file, so all current lines appear as edited. - return TextDocument() - - def get_path_in_repo(path: Path) -> Path: """Return the relative path to the file in the old revision @@ -172,79 +76,6 @@ def should_reformat_file(path: Path) -> bool: return path.exists() and get_path_in_repo(path).suffix == ".py" -@lru_cache(maxsize=1) -def _make_git_env() -> Dict[str, str]: - """Create custom minimal environment variables to use when invoking Git - - This makes sure that - - Git always runs in English - - ``$PATH`` is preserved (essential on NixOS) - - the environment is otherwise cleared - - """ - return {"LC_ALL": "C", "PATH": os.environ["PATH"]} - - -def _git_check_output_lines( - cmd: List[str], cwd: Path, exit_on_error: bool = True -) -> List[str]: - """Log command line, run Git, split stdout to lines, exit with 123 on error""" - return _git_check_output( - cmd, - cwd, - exit_on_error=exit_on_error, - encoding="utf-8", - ).splitlines() - - -@overload -def _git_check_output( - cmd: List[str], cwd: Path, *, exit_on_error: bool = ..., encoding: None = ... -) -> bytes: - ... - - -@overload -def _git_check_output( - cmd: List[str], cwd: Path, *, exit_on_error: bool = ..., encoding: str -) -> str: - ... - - -def _git_check_output( - cmd: List[str], - cwd: Path, - *, - exit_on_error: bool = True, - encoding: Optional[str] = None, -) -> Union[str, bytes]: - """Log command line, run Git, return stdout, exit with 123 on error""" - logger.debug("[%s]$ git %s", cwd, shlex.join(cmd)) - try: - return check_output( # nosec - ["git"] + cmd, - cwd=str(cwd), - encoding=encoding, - stderr=PIPE, - env=_make_git_env(), - ) - except CalledProcessError as exc_info: - if not exit_on_error: - raise - if exc_info.returncode != 128: - if encoding: - sys.stderr.write(exc_info.stderr) - else: - sys.stderr.buffer.write(exc_info.stderr) - raise - - # Bad revision or another Git failure. Follow Black's example and return the - # error status 123. - for error_line in exc_info.stderr.splitlines(): - logger.error(error_line) - sys.exit(123) - - def _git_exists_in_revision(path: Path, rev2: str, cwd: Path) -> bool: """Return ``True`` if the given path exists in the given Git revision @@ -266,7 +97,7 @@ def _git_exists_in_revision(path: Path, rev2: str, cwd: Path) -> bool: cwd=str(cwd), check=False, stderr=DEVNULL, - env=_make_git_env(), + env=make_git_env(), ) return result.returncode == 0 @@ -312,7 +143,7 @@ def _git_diff_name_only( ] if rev2 != WORKTREE: diff_cmd.insert(diff_cmd.index("--"), rev2) - lines = _git_check_output_lines(diff_cmd, cwd) + lines = git_check_output_lines(diff_cmd, cwd) return {Path(line) for line in lines} @@ -334,7 +165,7 @@ def _git_ls_files_others(relative_paths: Iterable[Path], cwd: Path) -> Set[Path] "--", *{path.as_posix() for path in relative_paths}, ] - lines = _git_check_output_lines(ls_files_cmd, cwd) + lines = git_check_output_lines(ls_files_cmd, cwd) return {Path(line) for line in lines} @@ -358,67 +189,6 @@ def git_get_modified_python_files( return {path for path in changed_paths if should_reformat_file(cwd / path)} -@contextmanager -def git_clone_local( - source_repository: Path, revision: str, destination: Path -) -> Iterator[Path]: - """Clone a local repository and check out the given revision - - :param source_repository: Path to the root of the local repository checkout - :param revision: The revision to check out, or ``HEAD`` - :param destination: Directory to create for the clone - :return: A context manager which yields the path to the clone - - """ - opts = [ - # By default, `add` refuses to create a new worktree when `` is - # a branch name and is already checked out by another worktree, or if - # `` is already assigned to some worktree but is missing (for - # instance, if `` was deleted manually). This option overrides these - # safeguards. To add a missing but locked worktree path, specify `--force` - # twice. - # `remove` refuses to remove an unclean worktree unless `--force` is used. - # To remove a locked worktree, specify `--force` twice. - # https://git-scm.com/docs/git-worktree#_options - "--force", - "--force", - str(destination), - ] - _ = _git_check_output( - ["worktree", "add", "--quiet", *opts, revision], cwd=source_repository - ) - yield destination - _ = _git_check_output(["worktree", "remove", *opts], cwd=source_repository) - - -def git_get_root(path: Path) -> Optional[Path]: - """Get the root directory of a local Git repository clone based on a path inside it - - :param path: A file or directory path inside the Git repository clone - :return: The root of the clone, or ``None`` if none could be found - :raises CalledProcessError: if Git exits with an unexpected error - - """ - try: - return Path( - _git_check_output( - ["rev-parse", "--show-toplevel"], - cwd=path if path.is_dir() else path.parent, - encoding="utf-8", - exit_on_error=False, - ).rstrip() - ) - except CalledProcessError as exc_info: - if exc_info.returncode == 128 and exc_info.stderr.splitlines()[0].startswith( - "fatal: not a git repository (or any " - ): - # The error string differs a bit in different Git versions, but up to the - # point above it's identical in recent versions. - return None - sys.stderr.write(exc_info.stderr) - raise - - def _revision_vs_lines( root: Path, path_in_repo: Path, rev1: str, content: TextDocument, context_lines: int ) -> List[int]: diff --git a/src/darker/tests/test_git.py b/src/darker/tests/test_git.py index 72b2594f5..8601faff3 100644 --- a/src/darker/tests/test_git.py +++ b/src/darker/tests/test_git.py @@ -4,187 +4,19 @@ # pylint: disable=too-many-lines,use-dict-literal import os -import re -from datetime import datetime, timedelta from pathlib import Path -from subprocess import DEVNULL, PIPE, CalledProcessError, check_call # nosec +from subprocess import DEVNULL, check_call # nosec from textwrap import dedent # nosec -from typing import List, Union -from unittest.mock import ANY, Mock, call, patch +from unittest.mock import patch import pytest from darker import git -from darker.utils import GIT_DATEFORMAT -from darkgraylib.git import RevisionRange +from darkgraylib.git import WORKTREE, RevisionRange from darkgraylib.testtools.git_repo_plugin import GitRepoFixture -from darkgraylib.testtools.helpers import raises_or_matches from darkgraylib.utils import TextDocument -def test_tmp_path_sanity(tmp_path): - """Make sure Pytest temporary directories aren't inside a Git repository""" - try: - result = git._git_check_output_lines( - ["rev-parse", "--absolute-git-dir"], tmp_path, exit_on_error=False - ) - except CalledProcessError as exc_info: - if exc_info.returncode != 128 or not exc_info.stderr.startswith( - "fatal: not a git repository" - ): - raise - else: - output = "\n".join(result) - raise AssertionError( - f"Temporary directory {tmp_path} for tests is not clean." - f" There is a Git directory in {output}" - ) - - -@pytest.mark.parametrize( - "revision_range, expect", - [ - ("", None), - ("..", ("", "..", "")), - ("...", ("", "...", "")), - ("a..", ("a", "..", "")), - ("a...", ("a", "...", "")), - ("a..b", ("a", "..", "b")), - ("a...b", ("a", "...", "b")), - ("..b", ("", "..", "b")), - ("...b", ("", "...", "b")), - ], -) -def test_commit_range_re(revision_range, expect): - """Test for ``COMMIT_RANGE_RE``""" - match = git.COMMIT_RANGE_RE.match(revision_range) - if expect is None: - assert match is None - else: - assert match is not None - assert match.groups() == expect - - -def test_worktree_symbol(): - """Test for the ``WORKTREE`` symbol""" - assert git.WORKTREE == ":WORKTREE:" - - -def test_git_get_mtime_at_commit(): - """darker.git.git_get_mtime_at_commit()""" - with patch.object(git, "_git_check_output_lines"): - git._git_check_output_lines.return_value = ["1609104839"] # type: ignore - - result = git.git_get_mtime_at_commit( - Path("dummy path"), "dummy revision", Path("dummy cwd") - ) - assert result == "2020-12-27 21:33:59.000000 +0000" - - -@pytest.mark.kwparametrize( - dict( - revision=":WORKTREE:", - expect_lines=("new content",), - expect_mtime=lambda: datetime(2001, 9, 9, 1, 46, 40), - ), - dict( - revision="HEAD", - expect_lines=("modified content",), - expect_mtime=datetime.utcnow, - ), - dict( - revision="HEAD^", - expect_lines=("original content",), - expect_mtime=datetime.utcnow, - ), - dict(revision="HEAD~2", expect_lines=(), expect_mtime=False), -) -def test_git_get_content_at_revision(git_repo, revision, expect_lines, expect_mtime): - """darker.git.git_get_content_at_revision()""" - git_repo.add({"my.txt": "original content"}, commit="Initial commit") - paths = git_repo.add({"my.txt": "modified content"}, commit="Initial commit") - paths["my.txt"].write_bytes(b"new content") - os.utime(paths["my.txt"], (1000000000, 1000000000)) - - result = git.git_get_content_at_revision( - Path("my.txt"), revision, cwd=Path(git_repo.root) - ) - - assert result.lines == expect_lines - if expect_mtime: - mtime_then = datetime.strptime(result.mtime, GIT_DATEFORMAT) - difference = expect_mtime() - mtime_then - assert timedelta(0) <= difference < timedelta(seconds=6) - else: - assert result.mtime == "" - assert result.encoding == "utf-8" - - -def git_call(cmd, encoding=None): - """Returns a mocked call to git""" - return call( - cmd.split(), - cwd=str(Path("/path")), - encoding=encoding, - stderr=PIPE, - env={"LC_ALL": "C", "PATH": os.environ["PATH"]}, - ) - - -@pytest.mark.kwparametrize( - dict( - revision=":WORKTREE:", - expect_textdocument_calls=[call.from_file(Path("/path/my.txt"))], - ), - dict( - revision="HEAD", - expect_git_calls=[ - git_call("git show HEAD:./my.txt"), - git_call("git log -1 --format=%ct HEAD -- my.txt", encoding="utf-8"), - ], - expect_textdocument_calls=[ - call.from_bytes(b"1627107028", mtime="2021-07-24 06:10:28.000000 +0000") - ], - ), - dict( - revision="HEAD^", - expect_git_calls=[ - git_call("git show HEAD^:./my.txt"), - git_call("git log -1 --format=%ct HEAD^ -- my.txt", encoding="utf-8"), - ], - expect_textdocument_calls=[ - call.from_bytes(b"1627107028", mtime="2021-07-24 06:10:28.000000 +0000") - ], - ), - dict( - revision="master", - expect_git_calls=[ - git_call("git show master:./my.txt"), - git_call("git log -1 --format=%ct master -- my.txt", encoding="utf-8"), - ], - expect_textdocument_calls=[ - call.from_bytes(b"1627107028", mtime="2021-07-24 06:10:28.000000 +0000") - ], - ), - expect_git_calls=[], -) -def test_git_get_content_at_revision_obtain_file_content( - revision, expect_git_calls, expect_textdocument_calls -): - """``git_get_content_at_revision`` calls Git or reads files based on revision""" - with patch("darker.git.check_output") as check_output, patch( - "darker.git.TextDocument" - ) as text_document_class: - # this dummy value acts both as a dummy Unix timestamp for the file as well as - # the contents of the file: - check_output.return_value = b"1627107028" - - git.git_get_content_at_revision(Path("my.txt"), revision, Path("/path")) - - assert check_output.call_args_list == expect_git_calls - assert text_document_class.method_calls == expect_textdocument_calls - - @pytest.mark.kwparametrize( dict(path="file.py", expect="file.py"), dict(path="subdir/file.py", expect="subdir/file.py"), @@ -224,185 +56,6 @@ def test_should_reformat_file(tmpdir, path, create, expect): assert result == expect -@pytest.mark.kwparametrize( - dict(cmd=[], exit_on_error=True, expect_template=CalledProcessError(1, "")), - dict( - cmd=["status", "-sb"], - exit_on_error=True, - expect_template=[ - "## branch", - "A add_index.py", - "D del_index.py", - " D del_worktree.py", - "A mod_index.py", - "?? add_worktree.py", - "?? mod_worktree.py", - ], - ), - dict( - cmd=["diff"], - exit_on_error=True, - expect_template=[ - "diff --git a/del_worktree.py b/del_worktree.py", - "deleted file mode 100644", - "index 94f3610..0000000", - "--- a/del_worktree.py", - "+++ /dev/null", - "@@ -1 +0,0 @@", - "-original", - "\\ No newline at end of file", - ], - ), - dict( - cmd=["merge-base", "master"], - exit_on_error=True, - expect_template=CalledProcessError(129, ""), - ), - dict( - cmd=["merge-base", "master", "HEAD"], - exit_on_error=True, - expect_template=[""], - ), - dict( - cmd=["show", "missing.file"], - exit_on_error=True, - expect_template=SystemExit(123), - ), - dict( - cmd=["show", "missing.file"], - exit_on_error=False, - expect_template=CalledProcessError(128, ""), - ), -) -def test_git_check_output_lines(branched_repo, cmd, exit_on_error, expect_template): - """Unit test for :func:`_git_check_output_lines`""" - if isinstance(expect_template, BaseException): - expect: Union[List[str], BaseException] = expect_template - else: - replacements = {"": branched_repo.get_hash("master^")} - expect = [replacements.get(line, line) for line in expect_template] - with raises_or_matches(expect, ["returncode", "code"]) as check: - - check(git._git_check_output_lines(cmd, branched_repo.root, exit_on_error)) - - -@pytest.mark.kwparametrize( - dict( - cmd=["show", "{initial}:/.file2"], - exit_on_error=True, - expect_exc=SystemExit, - expect_log=( - r"(?:^|\n)ERROR darker\.git:git\.py:\d+ fatal: " - r"[pP]ath '/\.file2' does not exist in '{initial}'\n" - ), - ), - dict( - cmd=["show", "{initial}:/.file2"], - exit_on_error=False, - expect_exc=CalledProcessError, - ), - dict( - cmd=["non-existing", "command"], - exit_on_error=True, - expect_exc=CalledProcessError, - expect_stderr="git: 'non-existing' is not a git command. See 'git --help'.\n", - ), - dict( - cmd=["non-existing", "command"], - exit_on_error=False, - expect_exc=CalledProcessError, - ), - expect_stderr="", - expect_log=r"$", -) -def test_git_check_output_lines_stderr_and_log( - git_repo, capfd, caplog, cmd, exit_on_error, expect_exc, expect_stderr, expect_log -): - """Git non-existing file error is logged and suppressed from stderr""" - git_repo.add({"file1": "file1"}, commit="Initial commit") - initial = git_repo.get_hash()[:7] - git_repo.add({"file2": "file2"}, commit="Second commit") - capfd.readouterr() # flush captured stdout and stderr - cmdline = [s.format(initial=initial) for s in cmd] - with pytest.raises(expect_exc): - - git._git_check_output_lines(cmdline, git_repo.root, exit_on_error) - - outerr = capfd.readouterr() - assert outerr.out == "" - assert outerr.err == expect_stderr - expect_log_re = expect_log.format(initial=initial) - assert re.search(expect_log_re, caplog.text), repr(caplog.text) - - -def test_git_get_content_at_revision_stderr(git_repo, capfd, caplog): - """No stderr or log output from ``git_get_content_at_revision`` for missing file""" - git_repo.add({"file1": "file1"}, commit="Initial commit") - initial = git_repo.get_hash()[:7] - git_repo.add({"file2": "file2"}, commit="Second commit") - capfd.readouterr() # flush captured stdout and stderr - - result = git.git_get_content_at_revision(Path("file2"), initial, git_repo.root) - - assert result == TextDocument() - outerr = capfd.readouterr() - assert outerr.out == "" - assert outerr.err == "" - assert [record for record in caplog.records if record.levelname != "DEBUG"] == [] - - -@pytest.fixture(scope="module") -def encodings_repo(tmp_path_factory): - """Create an example Git repository using various encodings for the same file""" - tmpdir = tmp_path_factory.mktemp("branched_repo") - git_repo = GitRepoFixture.create_repository(tmpdir) - # Commit without an encoding cookie, defaults to utf-8 - git_repo.add({"file.py": "darker = 'plus foncé'\n"}, commit="Default encoding") - git_repo.create_tag("default") - # Commit without an encoding cookie but with a utf-8 signature - content = "darker = 'plus foncé'\n".encode("utf-8-sig") - git_repo.add({"file.py": content}, commit="utf-8-sig") - git_repo.create_tag("utf-8-sig") - # Commit with an iso-8859-1 encoding cookie - content = "# coding: iso-8859-1\ndarker = 'plus foncé'\n".encode("iso-8859-1") - git_repo.add({"file.py": content}, commit="iso-8859-1") - git_repo.create_tag("iso-8859-1") - # Commit with a utf-8 encoding cookie - content = "# coding: utf-8\npython = 'パイソン'\n".encode() - git_repo.add({"file.py": content}, commit="utf-8") - git_repo.create_tag("utf-8") - # Current worktree content (not committed) with a shitfjs encoding cookie - content = "# coding: shiftjis\npython = 'パイソン'\n".encode("shiftjis") - git_repo.add({"file.py": content}) - return git_repo - - -@pytest.mark.kwparametrize( - dict(commit="default", encoding="utf-8", lines=("darker = 'plus foncé'",)), - dict(commit="utf-8-sig", encoding="utf-8-sig", lines=("darker = 'plus foncé'",)), - dict( - commit="iso-8859-1", - encoding="iso-8859-1", - lines=("# coding: iso-8859-1", "darker = 'plus foncé'"), - ), - dict( - commit="utf-8", encoding="utf-8", lines=("# coding: utf-8", "python = 'パイソン'") - ), - dict( - commit=":WORKTREE:", - encoding="shiftjis", - lines=("# coding: shiftjis", "python = 'パイソン'"), - ), -) -def test_git_get_content_at_revision_encoding(encodings_repo, commit, encoding, lines): - """Git file is loaded using its historical encoding""" - result = git.git_get_content_at_revision( - Path("file.py"), commit, encodings_repo.root - ) - assert result.encoding == encoding - assert result.lines == lines - - @pytest.mark.kwparametrize( dict(retval=0, expect=True), dict(retval=1, expect=False), @@ -483,7 +136,7 @@ def test_get_missing_at_revision_worktree(git_repo): paths["dir/b.py"].unlink() result = git.get_missing_at_revision( - {Path("dir"), Path("dir/a.py"), Path("dir/b.py")}, git.WORKTREE, git_repo.root + {Path("dir"), Path("dir/a.py"), Path("dir/b.py")}, WORKTREE, git_repo.root ) assert result == {Path("dir/a.py"), Path("dir/b.py")} @@ -577,66 +230,6 @@ def test_git_get_modified_python_files(git_repo, modify_paths, paths, expect): assert result == {Path(p) for p in expect} -@pytest.fixture(scope="module") -def branched_repo(tmp_path_factory): - """Create an example Git repository with a master branch and a feature branch - - The history created is:: - - . worktree - . index - * branch - | * master - |/ - * Initial commit - - """ - tmpdir = tmp_path_factory.mktemp("branched_repo") - git_repo = GitRepoFixture.create_repository(tmpdir) - git_repo.add( - { - "del_master.py": "original", - "del_branch.py": "original", - "del_index.py": "original", - "del_worktree.py": "original", - "mod_master.py": "original", - "mod_branch.py": "original", - "mod_both.py": "original", - "mod_same.py": "original", - "keep.py": "original", - }, - commit="Initial commit", - ) - branch_point = git_repo.get_hash() - git_repo.add( - { - "del_master.py": None, - "add_master.py": "master", - "mod_master.py": "master", - "mod_both.py": "master", - "mod_same.py": "same", - }, - commit="master", - ) - git_repo.create_branch("branch", branch_point) - git_repo.add( - { - "del_branch.py": None, - "mod_branch.py": "branch", - "mod_both.py": "branch", - "mod_same.py": "same", - }, - commit="branch", - ) - git_repo.add( - {"del_index.py": None, "add_index.py": "index", "mod_index.py": "index"} - ) - (git_repo.root / "del_worktree.py").unlink() - (git_repo.root / "add_worktree.py").write_bytes(b"worktree") - (git_repo.root / "mod_worktree.py").write_bytes(b"worktree") - return git_repo - - @pytest.mark.kwparametrize( dict( _description="from latest commit in branch to worktree and index", @@ -734,108 +327,6 @@ def test_git_get_modified_python_files_revision_range( assert {path.name for path in result} == expect -@pytest.mark.kwparametrize( - dict(branch="first", expect="first"), - dict(branch="second", expect="second"), - dict(branch="third", expect="third"), - dict(branch="HEAD", expect="third"), -) -def test_git_clone_local_branch(git_repo, tmp_path, branch, expect): - """``git_clone_local()`` checks out the specified branch""" - git_repo.add({"a.py": "first"}, commit="first") - git_repo.create_branch("first", "HEAD") - git_repo.create_branch("second", "HEAD") - git_repo.add({"a.py": "second"}, commit="second") - git_repo.create_branch("third", "HEAD") - git_repo.add({"a.py": "third"}, commit="third") - - with git.git_clone_local(git_repo.root, branch, tmp_path / "clone") as clone: - - assert (clone / "a.py").read_text() == expect - - -@pytest.mark.kwparametrize( - dict(branch="HEAD"), - dict(branch="mybranch"), -) -def test_git_clone_local_command(git_repo, tmp_path, branch): - """``git_clone_local()`` issues the correct Git command and options""" - git_repo.add({"a.py": "first"}, commit="first") - git_repo.create_branch("mybranch", "HEAD") - check_output = Mock(wraps=git.check_output) # type: ignore[attr-defined] - clone = tmp_path / "clone" - check_output_opts = dict( - cwd=str(git_repo.root), encoding=None, stderr=PIPE, env=ANY - ) - pre_call = call( - ["git", "worktree", "add", "--quiet", "--force", "--force", str(clone), branch], - **check_output_opts, - ) - post_call = call( - ["git", "worktree", "remove", "--force", "--force", str(clone)], - **check_output_opts, - ) - with patch.object(git, "check_output", check_output): - - with git.git_clone_local(git_repo.root, branch, clone) as result: - - assert result == clone - - check_output.assert_has_calls([pre_call]) - check_output.reset_mock() - check_output.assert_has_calls([post_call]) - - -@pytest.mark.parametrize( - "path", - [ - ".", - "root.py", - "subdir", - "subdir/sub.py", - "subdir/subsubdir", - "subdir/subsubdir/subsub.py", - ], -) -def test_git_get_root(git_repo, path): - """``git_get_root()`` returns repository root for any file or directory inside""" - git_repo.add( - { - "root.py": "root", - "subdir/sub.py": "sub", - "subdir/subsubdir/subsub.py": "subsub", - }, - commit="Initial commit", - ) - - root = git.git_get_root(git_repo.root / path) - - assert root == git_repo.root - - -@pytest.mark.parametrize( - "path", - [ - ".", - "root.py", - "subdir", - "subdir/sub.py", - "subdir/subsubdir", - "subdir/subsubdir/subsub.py", - ], -) -def test_git_get_root_not_found(tmp_path, path): - """``git_get_root()`` returns ``None`` for any file or directory outside of Git""" - (tmp_path / "subdir" / "subsubdir").mkdir(parents=True) - (tmp_path / "root.py").touch() - (tmp_path / "subdir" / "sub.py").touch() - (tmp_path / "subdir" / "subsubdir" / "subsub.py").touch() - - root = git.git_get_root(tmp_path / path) - - assert root is None - - edited_linenums_differ_cases = pytest.mark.kwparametrize( dict(context_lines=0, expect=[3, 7]), dict(context_lines=1, expect=[2, 3, 4, 6, 7, 8]), diff --git a/src/darker/tests/test_main.py b/src/darker/tests/test_main.py index 0d3ca7067..5a0c0c8f5 100644 --- a/src/darker/tests/test_main.py +++ b/src/darker/tests/test_main.py @@ -21,12 +21,12 @@ import darker.import_sorting from darker.config import Exclusions from darker.exceptions import MissingPackageError -from darker.git import WORKTREE, EditedLinenumsDiffer +from darker.git import EditedLinenumsDiffer from darker.tests.helpers import isort_present from darker.tests.test_fstring import FLYNTED_SOURCE, MODIFIED_SOURCE, ORIGINAL_SOURCE from darker.utils import joinlines from darker.verification import NotEquivalentError -from darkgraylib.git import RevisionRange +from darkgraylib.git import WORKTREE, RevisionRange from darkgraylib.testtools.highlighting_helpers import BLUE, CYAN, RESET, WHITE, YELLOW from darkgraylib.utils import TextDocument From 889844af1c57788c8a78fabeb42562a641d1fbed Mon Sep 17 00:00:00 2001 From: Antti Kaihola <13725+akaihola@users.noreply.github.com> Date: Sun, 3 Mar 2024 22:54:21 +0200 Subject: [PATCH 30/30] Move some `utils.py` helpers to Darkgraylib --- src/darker/__main__.py | 4 +- src/darker/chooser.py | 2 +- src/darker/diff.py | 3 +- src/darker/import_sorting.py | 4 +- src/darker/tests/test_command_line.py | 3 +- src/darker/tests/test_fstring.py | 3 +- src/darker/tests/test_import_sorting.py | 3 +- src/darker/tests/test_main.py | 3 +- src/darker/tests/test_utils.py | 379 +----------------------- src/darker/tests/test_verification.py | 3 +- src/darker/utils.py | 56 +--- src/darker/verification.py | 4 +- 12 files changed, 17 insertions(+), 450 deletions(-) diff --git a/src/darker/__main__.py b/src/darker/__main__.py index 76c3702d6..a894cfc39 100644 --- a/src/darker/__main__.py +++ b/src/darker/__main__.py @@ -32,7 +32,7 @@ ) from darker.help import get_extra_instruction from darker.import_sorting import apply_isort, isort -from darker.utils import GIT_DATEFORMAT, DiffChunk, debug_dump, glob_any +from darker.utils import debug_dump, glob_any from darker.verification import ASTVerifier, BinarySearch, NotEquivalentError from darkgraylib.black_compat import find_project_root from darkgraylib.config import show_config_if_debug @@ -46,7 +46,7 @@ from darkgraylib.highlighting import colorize, should_use_color from darkgraylib.log import setup_logging from darkgraylib.main import resolve_paths -from darkgraylib.utils import TextDocument +from darkgraylib.utils import GIT_DATEFORMAT, DiffChunk, TextDocument from graylint.linting import run_linters logger = logging.getLogger(__name__) diff --git a/src/darker/chooser.py b/src/darker/chooser.py index 0c9f0d6b2..d93a90626 100644 --- a/src/darker/chooser.py +++ b/src/darker/chooser.py @@ -32,7 +32,7 @@ import logging from typing import Generator, Iterable, List -from darker.utils import DiffChunk +from darkgraylib.utils import DiffChunk logger = logging.getLogger(__name__) diff --git a/src/darker/diff.py b/src/darker/diff.py index b3c2020f4..83e7d9abd 100644 --- a/src/darker/diff.py +++ b/src/darker/diff.py @@ -68,9 +68,8 @@ from typing import Generator, List, Sequence, Tuple from darker.multiline_strings import find_overlap -from darker.utils import DiffChunk from darkgraylib.diff import diff_and_get_opcodes, validate_opcodes -from darkgraylib.utils import TextDocument +from darkgraylib.utils import DiffChunk, TextDocument logger = logging.getLogger(__name__) diff --git a/src/darker/import_sorting.py b/src/darker/import_sorting.py index d01dd55cd..7239dd3ec 100644 --- a/src/darker/import_sorting.py +++ b/src/darker/import_sorting.py @@ -7,9 +7,9 @@ from darker.diff import diff_chunks from darker.exceptions import IncompatiblePackageError, MissingPackageError from darker.git import EditedLinenumsDiffer -from darker.utils import DiffChunk, glob_any +from darker.utils import glob_any from darkgraylib.black_compat import find_project_root -from darkgraylib.utils import TextDocument +from darkgraylib.utils import DiffChunk, TextDocument try: import isort diff --git a/src/darker/tests/test_command_line.py b/src/darker/tests/test_command_line.py index a2b56fc28..719e5a9fe 100644 --- a/src/darker/tests/test_command_line.py +++ b/src/darker/tests/test_command_line.py @@ -19,11 +19,10 @@ from darker.command_line import make_argument_parser, parse_command_line from darker.config import Exclusions from darker.tests.helpers import flynt_present, isort_present -from darker.utils import joinlines from darkgraylib.config import ConfigurationError from darkgraylib.git import RevisionRange from darkgraylib.testtools.helpers import raises_if_exception -from darkgraylib.utils import TextDocument +from darkgraylib.utils import TextDocument, joinlines pytestmark = pytest.mark.usefixtures("find_project_root_cache_clear") diff --git a/src/darker/tests/test_fstring.py b/src/darker/tests/test_fstring.py index c283942d2..749d44c1a 100644 --- a/src/darker/tests/test_fstring.py +++ b/src/darker/tests/test_fstring.py @@ -10,9 +10,8 @@ import darker.fstring from darker.git import EditedLinenumsDiffer from darker.tests.helpers import flynt_present -from darker.utils import joinlines from darkgraylib.git import RevisionRange -from darkgraylib.utils import TextDocument +from darkgraylib.utils import TextDocument, joinlines ORIGINAL_SOURCE = ("'{}'.format(x)", "#", "'{0}'.format(42)") MODIFIED_SOURCE = ("'{}'.format( x)", "#", "'{0}'.format( 42)") diff --git a/src/darker/tests/test_import_sorting.py b/src/darker/tests/test_import_sorting.py index bfefdd498..0203826e8 100644 --- a/src/darker/tests/test_import_sorting.py +++ b/src/darker/tests/test_import_sorting.py @@ -11,9 +11,8 @@ import darker.import_sorting from darker.git import EditedLinenumsDiffer from darker.tests.helpers import isort_present -from darker.utils import joinlines from darkgraylib.git import RevisionRange -from darkgraylib.utils import TextDocument +from darkgraylib.utils import TextDocument, joinlines ORIGINAL_SOURCE = ("import sys", "import os", "", "print(42)") ISORTED_SOURCE = ("import os", "import sys", "", "print(42)") diff --git a/src/darker/tests/test_main.py b/src/darker/tests/test_main.py index 5a0c0c8f5..9b5b62e82 100644 --- a/src/darker/tests/test_main.py +++ b/src/darker/tests/test_main.py @@ -24,11 +24,10 @@ from darker.git import EditedLinenumsDiffer from darker.tests.helpers import isort_present from darker.tests.test_fstring import FLYNTED_SOURCE, MODIFIED_SOURCE, ORIGINAL_SOURCE -from darker.utils import joinlines from darker.verification import NotEquivalentError from darkgraylib.git import WORKTREE, RevisionRange from darkgraylib.testtools.highlighting_helpers import BLUE, CYAN, RESET, WHITE, YELLOW -from darkgraylib.utils import TextDocument +from darkgraylib.utils import TextDocument, joinlines pytestmark = pytest.mark.usefixtures("find_project_root_cache_clear") diff --git a/src/darker/tests/test_utils.py b/src/darker/tests/test_utils.py index 655f1dca2..880a69dba 100644 --- a/src/darker/tests/test_utils.py +++ b/src/darker/tests/test_utils.py @@ -3,71 +3,9 @@ # pylint: disable=comparison-with-callable,redefined-outer-name,use-dict-literal import logging -import os -from pathlib import Path from textwrap import dedent -import pytest - -from darker.utils import ( - debug_dump, - detect_newline, - get_common_root, - get_path_ancestry, - joinlines, -) -from darkgraylib.utils import TextDocument - - -@pytest.fixture(params=[TextDocument.from_file, TextDocument.from_bytes]) -def textdocument_factory(request): - """Fixture for a factory function that creates a ``TextDocument`` - - The fixture can be parametrized with `(bytes) -> TextDocument` functions - that take the raw bytes of the document. - - By default, it is parametrized with the ``TextDocument.from_file()`` (for - which it creates a temporary file) and the ``TextDocument.from_bytes()`` - classmethods. - """ - if request.param == TextDocument.from_file: - - def factory(content): - tmp_path = request.getfixturevalue("tmp_path") - path = tmp_path / "test.py" - path.write_bytes(content) - return TextDocument.from_file(path) - - return factory - - return request.param - - -@pytest.fixture -def textdocument(request, textdocument_factory): - """Fixture for a ``TextDocument`` - - The fixture must be parametrized with the raw bytes of the document. - """ - return textdocument_factory(request.param) - - -@pytest.mark.kwparametrize( - dict(string="", expect="\n"), - dict(string="\n", expect="\n"), - dict(string="\r\n", expect="\r\n"), - dict(string="one line\n", expect="\n"), - dict(string="one line\r\n", expect="\r\n"), - dict(string="first line\nsecond line\n", expect="\n"), - dict(string="first line\r\nsecond line\r\n", expect="\r\n"), - dict(string="first unix\nthen windows\r\n", expect="\n"), - dict(string="first windows\r\nthen unix\n", expect="\r\n"), -) -def test_detect_newline(string, expect): - """``detect_newline()`` gives correct results""" - result = detect_newline(string) - - assert result == expect +from darker.utils import debug_dump def test_debug_dump(caplog, capsys): @@ -84,318 +22,3 @@ def test_debug_dump(caplog, capsys): """ ) ) - - -def test_joinlines(): - """``joinlines() concatenates and adds a newline after each given string item""" - result = joinlines(("a", "b", "c")) - assert result == "a\nb\nc\n" - - -def test_get_common_root_empty(): - """``get_common_root()`` raises a ``ValueError`` if ``paths`` argument is empty""" - with pytest.raises(ValueError): - - get_common_root([]) - - -def test_get_common_root(tmpdir): - """``get_common_root()`` traverses backwards correctly""" - tmpdir = Path(tmpdir) - path1 = tmpdir / "a" / "b" / "c" / "d" - path2 = tmpdir / "a" / "e" / ".." / "b" / "f" / "g" - path3 = tmpdir / "a" / "h" / ".." / "b" / "i" - result = get_common_root([path1, path2, path3]) - assert result == tmpdir / "a" / "b" - - -def test_get_common_root_of_directory(tmpdir): - """``get_common_root()`` returns a single directory itself""" - tmpdir = Path(tmpdir) - result = get_common_root([tmpdir]) - assert result == tmpdir - - -def test_get_path_ancestry_for_directory(tmpdir): - """``get_path_ancestry()`` includes a directory itself as the last item""" - tmpdir = Path(tmpdir) - result = list(get_path_ancestry(tmpdir)) - assert result[-1] == tmpdir - assert result[-2] == tmpdir.parent - - -def test_get_path_ancestry_for_file(tmpdir): - """``get_path_ancestry()`` includes a file's parent directory as the last item""" - tmpdir = Path(tmpdir) - dummy = tmpdir / "dummy" - dummy.write_text("dummy") - result = list(get_path_ancestry(dummy)) - assert result[-1] == tmpdir - assert result[-2] == tmpdir.parent - - -@pytest.mark.kwparametrize( - dict(textdocument=TextDocument(), expect="utf-8"), - dict(textdocument=TextDocument(encoding="utf-8"), expect="utf-8"), - dict(textdocument=TextDocument(encoding="utf-16"), expect="utf-16"), - dict(textdocument=TextDocument.from_str(""), expect="utf-8"), - dict(textdocument=TextDocument.from_str("", encoding="utf-8"), expect="utf-8"), - dict(textdocument=TextDocument.from_str("", encoding="utf-16"), expect="utf-16"), - dict(textdocument=TextDocument.from_lines([]), expect="utf-8"), - dict(textdocument=TextDocument.from_lines([], encoding="utf-8"), expect="utf-8"), - dict(textdocument=TextDocument.from_lines([], encoding="utf-16"), expect="utf-16"), -) -def test_textdocument_set_encoding(textdocument, expect): - """TextDocument.encoding is correct from each constructor""" - assert textdocument.encoding == expect - - -@pytest.mark.kwparametrize( - dict(doc=TextDocument(), expect=""), - dict(doc=TextDocument(lines=["zéro", "un"]), expect="zéro\nun\n"), - dict(doc=TextDocument(lines=["zéro", "un"], newline="\n"), expect="zéro\nun\n"), - dict( - doc=TextDocument(lines=["zéro", "un"], newline="\r\n"), expect="zéro\r\nun\r\n" - ), -) -def test_textdocument_string(doc, expect): - """TextDocument.string respects the newline setting""" - assert doc.string == expect - - -@pytest.mark.parametrize("newline", ["\n", "\r\n"]) -@pytest.mark.kwparametrize( - dict(textdocument=TextDocument(), expect=""), - dict(textdocument=TextDocument(lines=["zéro", "un"])), - dict(textdocument=TextDocument(string="zéro\nun\n")), - dict(textdocument=TextDocument(lines=["zéro", "un"], newline="\n")), - dict(textdocument=TextDocument(string="zéro\nun\n", newline="\n")), - dict(textdocument=TextDocument(lines=["zéro", "un"], newline="\r\n")), - dict(textdocument=TextDocument(string="zéro\r\nun\r\n", newline="\r\n")), - expect="zéro{newline}un{newline}", -) -def test_textdocument_string_with_newline(textdocument, newline, expect): - """TextDocument.string respects the newline setting""" - result = textdocument.string_with_newline(newline) - - expected = expect.format(newline=newline) - assert result == expected - - -@pytest.mark.kwparametrize( - dict(encoding="utf-8", newline="\n", expect=b"z\xc3\xa9ro\nun\n"), - dict(encoding="iso-8859-1", newline="\n", expect=b"z\xe9ro\nun\n"), - dict(encoding="utf-8", newline="\r\n", expect=b"z\xc3\xa9ro\r\nun\r\n"), - dict(encoding="iso-8859-1", newline="\r\n", expect=b"z\xe9ro\r\nun\r\n"), -) -def test_textdocument_encoded_string(encoding, newline, expect): - """TextDocument.encoded_string uses correct encoding and newline""" - textdocument = TextDocument( - lines=["zéro", "un"], encoding=encoding, newline=newline - ) - - assert textdocument.encoded_string == expect - - -@pytest.mark.kwparametrize( - dict(doc=TextDocument(), expect=()), - dict(doc=TextDocument(string="zéro\nun\n"), expect=("zéro", "un")), - dict(doc=TextDocument(string="zéro\nun\n", newline="\n"), expect=("zéro", "un")), - dict( - doc=TextDocument(string="zéro\r\nun\r\n", newline="\r\n"), expect=("zéro", "un") - ), -) -def test_textdocument_lines(doc, expect): - """TextDocument.lines is correct after parsing a string with different newlines""" - assert doc.lines == expect - - -@pytest.mark.kwparametrize( - dict( - textdocument=TextDocument.from_str(""), - expect_lines=(), - expect_encoding="utf-8", - expect_newline="\n", - expect_mtime="", - ), - dict( - textdocument=TextDocument.from_str("", encoding="utf-8"), - expect_lines=(), - expect_encoding="utf-8", - expect_newline="\n", - expect_mtime="", - ), - dict( - textdocument=TextDocument.from_str("", encoding="iso-8859-1"), - expect_lines=(), - expect_encoding="iso-8859-1", - expect_newline="\n", - expect_mtime="", - ), - dict( - textdocument=TextDocument.from_str("a\nb\n"), - expect_lines=("a", "b"), - expect_encoding="utf-8", - expect_newline="\n", - expect_mtime="", - ), - dict( - textdocument=TextDocument.from_str("a\r\nb\r\n"), - expect_lines=("a", "b"), - expect_encoding="utf-8", - expect_newline="\r\n", - expect_mtime="", - ), - dict( - textdocument=TextDocument.from_str("", mtime="my mtime"), - expect_lines=(), - expect_encoding="utf-8", - expect_newline="\n", - expect_mtime="my mtime", - ), -) -def test_textdocument_from_str( - textdocument, expect_lines, expect_encoding, expect_newline, expect_mtime -): - """TextDocument.from_str() gets correct content, encoding, newlines and mtime""" - assert textdocument.lines == expect_lines - assert textdocument.encoding == expect_encoding - assert textdocument.newline == expect_newline - assert textdocument.mtime == expect_mtime - - -@pytest.mark.kwparametrize( - dict(textdocument=b'print("touch\xc3\xa9")\n', expect="utf-8"), - dict(textdocument=b'\xef\xbb\xbfprint("touch\xc3\xa9")\n', expect="utf-8-sig"), - dict(textdocument=b'# coding: iso-8859-1\n"touch\xe9"\n', expect="iso-8859-1"), - indirect=["textdocument"], -) -def test_textdocument_detect_encoding(textdocument, expect): - """TextDocument.from_file/bytes() detects the file encoding correctly""" - assert textdocument.encoding == expect - - -@pytest.mark.kwparametrize( - dict(textdocument=b'print("unix")\n', expect="\n"), - dict(textdocument=b'print("windows")\r\n', expect="\r\n"), - indirect=["textdocument"], -) -def test_textdocument_detect_newline(textdocument, expect): - """TextDocument.from_file/bytes() detects the newline sequence correctly""" - assert textdocument.newline == expect - - -@pytest.mark.kwparametrize( - dict(doc1=TextDocument(lines=["foo"]), doc2=TextDocument(lines=[]), expect=False), - dict(doc1=TextDocument(lines=[]), doc2=TextDocument(lines=["foo"]), expect=False), - dict( - doc1=TextDocument(lines=["foo"]), doc2=TextDocument(lines=["bar"]), expect=False - ), - dict( - doc1=TextDocument(lines=["line1", "line2"]), - doc2=TextDocument(lines=["line1", "line2"]), - expect=True, - ), - dict( - doc1=TextDocument(lines=["line1", "line2"], encoding="utf-16", newline="\r\n"), - doc2=TextDocument(lines=["line1", "line2"]), - expect=True, - ), - dict(doc1=TextDocument(lines=["foo"]), doc2=TextDocument(""), expect=False), - dict(doc1=TextDocument(lines=[]), doc2=TextDocument("foo\n"), expect=False), - dict(doc1=TextDocument(lines=["foo"]), doc2=TextDocument("bar\n"), expect=False), - dict( - doc1=TextDocument(lines=["line1", "line2"]), - doc2=TextDocument("line1\nline2\n"), - expect=True, - ), - dict(doc1=TextDocument("foo\n"), doc2=TextDocument(lines=[]), expect=False), - dict(doc1=TextDocument(""), doc2=TextDocument(lines=["foo"]), expect=False), - dict(doc1=TextDocument("foo\n"), doc2=TextDocument(lines=["bar"]), expect=False), - dict( - doc1=TextDocument("line1\nline2\n"), - doc2=TextDocument(lines=["line1", "line2"]), - expect=True, - ), - dict(doc1=TextDocument("foo\n"), doc2=TextDocument(""), expect=False), - dict(doc1=TextDocument(""), doc2=TextDocument("foo\n"), expect=False), - dict(doc1=TextDocument("foo\n"), doc2=TextDocument("bar\n"), expect=False), - dict( - doc1=TextDocument("line1\nline2\n"), - doc2=TextDocument("line1\nline2\n"), - expect=True, - ), - dict( - doc1=TextDocument("line1\r\nline2\r\n"), - doc2=TextDocument("line1\nline2\n"), - expect=True, - ), - dict(doc1=TextDocument("foo"), doc2="line1\nline2\n", expect=NotImplemented), -) -def test_textdocument_eq(doc1, doc2, expect): - """TextDocument.__eq__()""" - result = doc1.__eq__(doc2) # pylint: disable=unnecessary-dunder-call - - assert result == expect - - -@pytest.mark.kwparametrize( - dict(document=TextDocument(""), expect="TextDocument([0 lines])"), - dict(document=TextDocument(lines=[]), expect="TextDocument([0 lines])"), - dict(document=TextDocument("One line\n"), expect="TextDocument([1 lines])"), - dict(document=TextDocument(lines=["One line"]), expect="TextDocument([1 lines])"), - dict(document=TextDocument("Two\nlines\n"), expect="TextDocument([2 lines])"), - dict( - document=TextDocument(lines=["Two", "lines"]), expect="TextDocument([2 lines])" - ), - dict( - document=TextDocument(mtime="some mtime"), - expect="TextDocument([0 lines], mtime='some mtime')", - ), - dict( - document=TextDocument(encoding="utf-8"), - expect="TextDocument([0 lines])", - ), - dict( - document=TextDocument(encoding="a non-default encoding"), - expect="TextDocument([0 lines], encoding='a non-default encoding')", - ), - dict( - document=TextDocument(newline="\n"), - expect="TextDocument([0 lines])", - ), - dict( - document=TextDocument(newline="a non-default newline"), - expect="TextDocument([0 lines], newline='a non-default newline')", - ), -) -def test_textdocument_repr(document, expect): - """TextDocument.__repr__()""" - result = repr(document) - - assert result == expect - - -@pytest.mark.kwparametrize( - dict(document=TextDocument(), expect=""), - dict(document=TextDocument(mtime=""), expect=""), - dict(document=TextDocument(mtime="dummy mtime"), expect="dummy mtime"), -) -def test_textdocument_mtime(document, expect): - """TextDocument.mtime""" - assert document.mtime == expect - - -def test_textdocument_from_file(tmp_path): - """TextDocument.from_file()""" - dummy_txt = tmp_path / "dummy.txt" - dummy_txt.write_bytes(b"# coding: iso-8859-1\r\ndummy\r\ncontent\r\n") - os.utime(dummy_txt, (1_000_000_000, 1_000_000_000)) - - document = TextDocument.from_file(dummy_txt) - - assert document.string == "# coding: iso-8859-1\r\ndummy\r\ncontent\r\n" - assert document.lines == ("# coding: iso-8859-1", "dummy", "content") - assert document.encoding == "iso-8859-1" - assert document.newline == "\r\n" - assert document.mtime == "2001-09-09 01:46:40.000000 +0000" diff --git a/src/darker/tests/test_verification.py b/src/darker/tests/test_verification.py index 70e209bf3..4cad7f42a 100644 --- a/src/darker/tests/test_verification.py +++ b/src/darker/tests/test_verification.py @@ -6,14 +6,13 @@ import pytest -from darker.utils import DiffChunk from darker.verification import ( ASTVerifier, BinarySearch, NotEquivalentError, verify_ast_unchanged, ) -from darkgraylib.utils import TextDocument +from darkgraylib.utils import DiffChunk, TextDocument @pytest.mark.kwparametrize( diff --git a/src/darker/utils.py b/src/darker/utils.py index 41e097017..acff93f36 100644 --- a/src/darker/utils.py +++ b/src/darker/utils.py @@ -1,29 +1,12 @@ """Miscellaneous utility functions""" import logging -import sys -from itertools import chain from pathlib import Path -from typing import Collection, Iterable, List, Tuple - -logger = logging.getLogger(__name__) - -TextLines = Tuple[str, ...] - - -WINDOWS = sys.platform.startswith("win") -GIT_DATEFORMAT = "%Y-%m-%d %H:%M:%S.%f +0000" +from typing import Collection, List +from darkgraylib.utils import DiffChunk -def detect_newline(string: str) -> str: - """Detect LF or CRLF newlines in a string by looking at the end of the first line""" - first_lf_pos = string.find("\n") - if first_lf_pos > 0 and string[first_lf_pos - 1] == "\r": - return "\r\n" - return "\n" - - -DiffChunk = Tuple[int, TextLines, TextLines] +logger = logging.getLogger(__name__) def debug_dump(black_chunks: List[DiffChunk], edited_linenums: List[int]) -> None: @@ -41,39 +24,6 @@ def debug_dump(black_chunks: List[DiffChunk], edited_linenums: List[int]) -> Non print(80 * "-") -def joinlines(lines: Iterable[str], newline: str = "\n") -> str: - """Join a list of lines back, adding a linefeed after each line - - This is the reverse of ``str.splitlines()``. - - """ - return "".join(f"{line}{newline}" for line in lines) - - -def get_path_ancestry(path: Path) -> Iterable[Path]: - """Return paths to directories leading to the given path - - :param path: The directory or file to get ancestor directories for - :return: A list of paths, starting from filesystem root and ending in the given - path (if it's a directory) or the parent of the given path (if it's a file) - - """ - reverse_parents = reversed(path.parents) - if path.is_dir(): - return chain(reverse_parents, [path]) - return reverse_parents - - -def get_common_root(paths: Iterable[Path]) -> Path: - """Find the deepest common parent directory of given paths""" - resolved_paths = [path.resolve() for path in paths] - parents = reversed(list(zip(*(get_path_ancestry(path) for path in resolved_paths)))) - for first_path, *other_paths in parents: - if all(path == first_path for path in other_paths): - return first_path - raise ValueError(f"Paths have no common parent Git root: {resolved_paths}") - - def glob_any(path: Path, patterns: Collection[str]) -> bool: """Return `True` if path matches any of the patterns diff --git a/src/darker/verification.py b/src/darker/verification.py index ebbc4ce61..1a89bac3b 100644 --- a/src/darker/verification.py +++ b/src/darker/verification.py @@ -4,8 +4,8 @@ from black import assert_equivalent, parse_ast, stringify_ast -from darker.utils import DiffChunk, debug_dump -from darkgraylib.utils import TextDocument +from darker.utils import debug_dump +from darkgraylib.utils import DiffChunk, TextDocument class NotEquivalentError(Exception):