diff --git a/.gitignore b/.gitignore index 872a6532..2079909a 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,4 @@ _build/ /sandbox/ /.ghtopdep_cache/ /worktrees/ +/.ruff_cache/ diff --git a/Makefile b/Makefile index 5fc1c7ae..40e4be7f 100644 --- a/Makefile +++ b/Makefile @@ -9,14 +9,12 @@ install: .PHONY: format format: - pyupgrade --py37-plus --exit-zero-even-if-changed `find $(sources) -name "*.py" -type f` - isort $(sources) black $(sources) + ruff --fix $(sources) .PHONY: lint lint: - flake8 $(sources) - isort $(sources) --check-only --df + ruff $(sources) black $(sources) --check --diff .PHONY: mypy diff --git a/docs/index.md b/docs/index.md index 48f679ea..e27f0fcc 100644 --- a/docs/index.md +++ b/docs/index.md @@ -16,14 +16,15 @@ from typing import Any, Callable, Set from pydantic import ( AliasChoices, + AmqpDsn, BaseModel, ConfigDict, + Field, ImportString, - RedisDsn, PostgresDsn, - AmqpDsn, - Field, + RedisDsn, ) + from pydantic_settings import BaseSettings @@ -38,7 +39,7 @@ class Settings(BaseSettings): redis_dsn: RedisDsn = Field( 'redis://user:pass@localhost:6379/1', - validation_alias=AliasChoices('service_redis_dsn', 'redis_url') + validation_alias=AliasChoices('service_redis_dsn', 'redis_url'), ) pg_dsn: PostgresDsn = 'postgres://user:pass@localhost:5432/foobar' amqp_dsn: AmqpDsn = 'amqp://user:pass@localhost:5672/' @@ -53,7 +54,8 @@ class Settings(BaseSettings): # export my_prefix_more_settings='{"foo": "x", "apple": 1}' more_settings: SubModel = SubModel() - model_config = ConfigDict(env_prefix = 'my_prefix_') # defaults to no prefix, i.e. "" + model_config = ConfigDict(env_prefix='my_prefix_') # defaults to no prefix, i.e. "" + print(Settings().model_dump()) """ @@ -63,7 +65,7 @@ print(Settings().model_dump()) 'redis_dsn': Url('redis://user:pass@localhost:6379/1'), 'pg_dsn': Url('postgres://user:pass@localhost:5432/foobar'), 'amqp_dsn': Url('amqp://user:pass@localhost:5672/'), - 'special_function': , + 'special_function': math.cos, 'domains': set(), 'more_settings': {'foo': 'bar', 'apple': 1}, } @@ -92,6 +94,7 @@ Case-sensitivity can be turned on through the `model_config`: ```py from pydantic import ConfigDict + from pydantic_settings import BaseSettings @@ -144,6 +147,7 @@ You could load a settings module thus: ```py from pydantic import BaseModel, ConfigDict + from pydantic_settings import BaseSettings @@ -169,12 +173,7 @@ print(Settings().model_dump()) """ { 'v0': '0', - 'sub_model': { - 'v1': 'json-1', - 'v2': b'nested-2', - 'v3': 3, - 'deep': {'v4': 'v4'}, - }, + 'sub_model': {'v1': 'json-1', 'v2': b'nested-2', 'v3': 3, 'deep': {'v4': 'v4'}}, } """ ``` @@ -196,11 +195,18 @@ import os from typing import Any, List, Tuple, Type from pydantic.fields import FieldInfo -from pydantic_settings import BaseSettings, EnvSettingsSource, PydanticBaseSettingsSource + +from pydantic_settings import ( + BaseSettings, + EnvSettingsSource, + PydanticBaseSettingsSource, +) class MyCustomSource(EnvSettingsSource): - def prepare_field_value(self, field_name: str, field: FieldInfo, value: Any, value_is_complex: bool) -> Any: + def prepare_field_value( + self, field_name: str, field: FieldInfo, value: Any, value_is_complex: bool + ) -> Any: if field_name == 'numbers': return [int(x) for x in value.split(',')] return json.loads(value) @@ -251,7 +257,7 @@ Once you have your `.env` file filled with variables, *pydantic* supports loadin **1.** setting `env_file` (and `env_file_encoding` if you don't want the default encoding of your OS) on `model_config` in a `BaseSettings` class: -```py +```py test="skip" lint="skip" class Settings(BaseSettings): ... @@ -261,7 +267,7 @@ class Settings(BaseSettings): **2.** instantiating a `BaseSettings` derived class with the `_env_file` keyword argument (and the `_env_file_encoding` if needed): -```py +```py test="skip" lint="skip" settings = Settings(_env_file='prod.env', _env_file_encoding='utf-8') ``` @@ -285,12 +291,18 @@ If you need to load multiple dotenv files, you can pass the file paths as a `lis Later files in the list/tuple will take priority over earlier files. ```py -from pydantic import BaseSettings +from pydantic import ConfigDict + +from pydantic_settings import BaseSettings + class Settings(BaseSettings): ... - model_config = ConfigDict(env_file=('.env', '.env.prod')) # `.env.prod` takes priority over `.env` + model_config = ConfigDict( + # `.env.prod` takes priority over `.env` + env_file=('.env', '.env.prod') + ) ``` You can also use the keyword argument override to tell Pydantic not to load any file at all (even if one is set in @@ -320,7 +332,7 @@ Once you have your secret files, *pydantic* supports loading it in two ways: **1.** setting `secrets_dir` on `model_config` in a `BaseSettings` class to the directory where your secret files are stored: -```py +```py test="skip" lint="skip" class Settings(BaseSettings): ... database_password: str @@ -330,7 +342,7 @@ class Settings(BaseSettings): **2.** instantiating a `BaseSettings` derived class with the `_secrets_dir` keyword argument: -```py +```py test="skip" lint="skip" settings = Settings(_secrets_dir='/var/run') ``` @@ -352,7 +364,7 @@ and using secrets in Docker see the official [Docker documentation](https://docs.docker.com/engine/reference/commandline/secret/). First, define your Settings -```py +```py test="skip" lint="skip" class Settings(BaseSettings): my_secret_data: str @@ -399,7 +411,9 @@ The order of the returned callables decides the priority of inputs; first item i ```py from typing import Tuple, Type + from pydantic import PostgresDsn + from pydantic_settings import BaseSettings, PydanticBaseSettingsSource @@ -436,6 +450,7 @@ from typing import Any, Dict, Tuple, Type from pydantic import ConfigDict from pydantic.fields import FieldInfo + from pydantic_settings import BaseSettings, PydanticBaseSettingsSource @@ -448,22 +463,31 @@ class JsonConfigSettingsSource(PydanticBaseSettingsSource): when reading `config.json` """ - def get_field_value(self, field: FieldInfo, field_name: str) -> Tuple[Any, str, bool]: + def get_field_value( + self, field: FieldInfo, field_name: str + ) -> Tuple[Any, str, bool]: encoding = self.config.get('env_file_encoding') - file_content_json = json.loads(Path('config.json').read_text(encoding)) + file_content_json = json.loads( + Path('tests/example_test_config.json').read_text(encoding) + ) fiel_value = file_content_json.get(field_name) return fiel_value, field_name, False - - def prepare_field_value(self, field_name: str, field: FieldInfo, value: Any, value_is_complex: bool) -> Any: + def prepare_field_value( + self, field_name: str, field: FieldInfo, value: Any, value_is_complex: bool + ) -> Any: return value def __call__(self) -> Dict[str, Any]: d: Dict[str, Any] = {} for field_name, field in self.settings_cls.model_fields.items(): - field_value, field_key, value_is_complex = self.get_field_value(field, field_name) - field_value = self.prepare_field_value(field_name, field, field_value, value_is_complex) + field_value, field_key, value_is_complex = self.get_field_value( + field, field_name + ) + field_value = self.prepare_field_value( + field_name, field, field_value, value_is_complex + ) if field_value is not None: d[field_key] = field_value @@ -493,7 +517,7 @@ class Settings(BaseSettings): print(Settings()) -#> foobar='spam' +#> foobar='test' ``` ### Removing sources @@ -503,6 +527,8 @@ You might also want to disable a source: ```py from typing import Tuple, Type +from pydantic import ValidationError + from pydantic_settings import BaseSettings, PydanticBaseSettingsSource @@ -522,6 +548,14 @@ class Settings(BaseSettings): return env_settings, file_secret_settings -print(Settings(my_api_key='this is ignored')) -# requires: `MY_API_KEY` env variable to be set, e.g. `export MY_API_KEY=xxx` +try: + Settings(my_api_key='this is ignored') +except ValidationError as exc_info: + print(exc_info) + """ + 1 validation error for Settings + my_api_key + Field required [type=missing, input_value={}, input_type=dict] + For further information visit https://errors.pydantic.dev/2/v/missing + """ ``` diff --git a/pydantic_settings/sources.py b/pydantic_settings/sources.py index a280b902..00258362 100644 --- a/pydantic_settings/sources.py +++ b/pydantic_settings/sources.py @@ -470,7 +470,7 @@ def _read_env_files(self, case_sensitive: bool) -> Mapping[str, str | None]: def __call__(self) -> dict[str, Any]: data: dict[str, Any] = super().__call__() - data_lower_keys: List[str] = [] + data_lower_keys: list[str] = [] if not self.settings_cls.model_config.get('case_sensitive', False): data_lower_keys = [x.lower() for x in data.keys()] diff --git a/pyproject.toml b/pyproject.toml index 22c95ffa..446137ef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,13 +59,6 @@ filterwarnings = [ 'ignore:This is a placeholder until pydantic-settings.*:UserWarning', ] -[tool.flake8] -max_line_length = 120 -max_complexity = 14 -inline_quotes = 'single' -multiline_quotes = 'double' -ignore = ['E203', 'W503'] - [tool.coverage.run] source = ['pydantic_settings'] branch = true @@ -86,21 +79,20 @@ source = [ 'pydantic_settings/', ] +[tool.ruff] +line-length = 120 +extend-select = ['Q', 'RUF100', 'C90', 'UP', 'I'] +flake8-quotes = {inline-quotes = 'single', multiline-quotes = 'double'} +mccabe = { max-complexity = 14 } +isort = { known-first-party = ['pydantic_settings', 'tests'] } +target-version = 'py37' + [tool.black] color = true line-length = 120 target-version = ['py310'] skip-string-normalization = true -[tool.isort] -line_length = 120 -known_first_party = 'pydantic_settings' -known_third_party = 'pydantic' -multi_line_output = 3 -include_trailing_comma = true -force_grid_wrap = 0 -combine_as_imports = true - [tool.mypy] python_version = '3.10' show_error_codes = true diff --git a/requirements/linting.in b/requirements/linting.in index 541205ed..64c9833f 100644 --- a/requirements/linting.in +++ b/requirements/linting.in @@ -1,8 +1,5 @@ black -flake8 -flake8-quotes -flake8-pyproject -isort +ruff pyupgrade mypy pre-commit diff --git a/requirements/linting.txt b/requirements/linting.txt index e4117f61..21b53172 100644 --- a/requirements/linting.txt +++ b/requirements/linting.txt @@ -1,6 +1,6 @@ # -# This file is autogenerated by pip-compile with python 3.10 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: # # pip-compile --output-file=requirements/linting.txt requirements/linting.in # @@ -14,21 +14,8 @@ distlib==0.3.6 # via virtualenv filelock==3.12.0 # via virtualenv -flake8==6.0.0 - # via - # -r requirements/linting.in - # flake8-pyproject - # flake8-quotes -flake8-pyproject==1.2.3 - # via -r requirements/linting.in -flake8-quotes==3.3.2 - # via -r requirements/linting.in identify==2.5.24 # via pre-commit -isort==5.12.0 - # via -r requirements/linting.in -mccabe==0.7.0 - # via flake8 mypy==1.3.0 # via -r requirements/linting.in mypy-extensions==1.0.0 @@ -47,21 +34,14 @@ platformdirs==3.5.1 # virtualenv pre-commit==3.3.1 # via -r requirements/linting.in -pycodestyle==2.10.0 - # via flake8 -pyflakes==3.0.1 - # via flake8 pyupgrade==3.4.0 # via -r requirements/linting.in pyyaml==6.0 # via pre-commit +ruff==0.0.265 + # via -r requirements/linting.in tokenize-rt==5.0.0 # via pyupgrade -tomli==2.0.1 - # via - # black - # flake8-pyproject - # mypy typing-extensions==4.5.0 # via mypy virtualenv==20.23.0 diff --git a/requirements/testing.in b/requirements/testing.in index b7f150a6..15cba286 100644 --- a/requirements/testing.in +++ b/requirements/testing.in @@ -1,4 +1,5 @@ coverage[toml] pytest pytest-mock -pytest-sugar +pytest-examples +pytest-pretty diff --git a/requirements/testing.txt b/requirements/testing.txt index 47fc5de1..9cc46d6e 100644 --- a/requirements/testing.txt +++ b/requirements/testing.txt @@ -1,41 +1,48 @@ # -# This file is autogenerated by pip-compile with python 3.7 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: # # pip-compile --output-file=requirements/testing.txt requirements/testing.in # +black==23.3.0 + # via pytest-examples +click==8.1.3 + # via black coverage[toml]==7.2.5 # via -r requirements/testing.in -exceptiongroup==1.1.1 - # via pytest -importlib-metadata==6.6.0 - # via - # pluggy - # pytest iniconfig==2.0.0 # via pytest +markdown-it-py==2.2.0 + # via rich +mdurl==0.1.2 + # via markdown-it-py +mypy-extensions==1.0.0 + # via black packaging==23.1 # via + # black # pytest - # pytest-sugar +pathspec==0.11.1 + # via black +platformdirs==3.5.1 + # via black pluggy==1.0.0 # via pytest +pygments==2.15.1 + # via rich pytest==7.3.1 # via # -r requirements/testing.in + # pytest-examples # pytest-mock - # pytest-sugar + # pytest-pretty +pytest-examples==0.0.9 + # via -r requirements/testing.in pytest-mock==3.10.0 # via -r requirements/testing.in -pytest-sugar==0.9.7 +pytest-pretty==1.2.0 # via -r requirements/testing.in -termcolor==2.3.0 - # via pytest-sugar -tomli==2.0.1 - # via - # coverage - # pytest -typing-extensions==4.5.0 - # via importlib-metadata -zipp==3.15.0 - # via importlib-metadata +rich==13.3.5 + # via pytest-pretty +ruff==0.0.265 + # via pytest-examples diff --git a/tests/conftest.py b/tests/conftest.py index ac900259..aec508f5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -27,3 +27,23 @@ def env(): yield setenv setenv.clear() + + +@pytest.fixture +def docs_test_env(): + setenv = SetEnv() + + # envs for basic usage example + setenv.set('my_auth_key', 'xxx') + setenv.set('my_api_key', 'xxx') + + # envs for parsing environment variable values example + setenv.set('V0', '0') + setenv.set('SUB_MODEL', '{"v1": "json-1", "v2": "json-2"}') + setenv.set('SUB_MODEL__V2', 'nested-2') + setenv.set('SUB_MODEL__V3', '3') + setenv.set('SUB_MODEL__DEEP__V4', 'v4') + + yield setenv + + setenv.clear() diff --git a/tests/example_test_config.json b/tests/example_test_config.json new file mode 100644 index 00000000..69f783a7 --- /dev/null +++ b/tests/example_test_config.json @@ -0,0 +1 @@ +{"foobar": "test"} diff --git a/tests/test_docs.py b/tests/test_docs.py new file mode 100644 index 00000000..e0e3478e --- /dev/null +++ b/tests/test_docs.py @@ -0,0 +1,117 @@ +from __future__ import annotations as _annotations + +import platform +import re +import sys +from pathlib import Path + +import pytest +from pytest_examples import CodeExample, EvalExample, find_examples +from pytest_examples.config import ExamplesConfig +from pytest_examples.lint import black_format + +DOCS_ROOT = Path(__file__).parent.parent / 'docs' + + +def skip_docs_tests(): + if sys.platform not in {'linux', 'darwin'}: + return 'not in linux or macos' + + if platform.python_implementation() != 'CPython': + return 'not cpython' + + +class GroupModuleGlobals: + def __init__(self) -> None: + self.name = None + self.module_dict: dict[str, str] = {} + + def get(self, name: str | None): + if name is not None and name == self.name: + return self.module_dict + + def set(self, name: str | None, module_dict: dict[str, str]): + self.name = name + if self.name is None: + self.module_dict = None + else: + self.module_dict = module_dict + + +group_globals = GroupModuleGlobals() + +skip_reason = skip_docs_tests() + + +def print_callback(print_statement: str) -> str: + # make error display uniform + s = re.sub(r'(https://errors.pydantic.dev)/.+?/', r'\1/2/', print_statement) + # hack until https://github.com/pydantic/pytest-examples/issues/11 is fixed + if '' in s: + # avoid function repr breaking black formatting + s = re.sub('', 'math.cos', s) + return black_format(s, ExamplesConfig()).rstrip('\n') + return s + + +@pytest.mark.filterwarnings('ignore:(parse_obj_as|schema_json_of|schema_of) is deprecated.*:DeprecationWarning') +@pytest.mark.skipif(bool(skip_reason), reason=skip_reason or 'not skipping') +@pytest.mark.parametrize('example', find_examples(str(DOCS_ROOT), skip=sys.platform == 'win32'), ids=str) +def test_docs_examples( # noqa C901 + example: CodeExample, eval_example: EvalExample, tmp_path: Path, mocker, docs_test_env +): + eval_example.print_callback = print_callback + + prefix_settings = example.prefix_settings() + test_settings = prefix_settings.get('test') + lint_settings = prefix_settings.get('lint') + if test_settings == 'skip' and lint_settings == 'skip': + pytest.skip('both test and lint skipped') + + requires_settings = prefix_settings.get('requires') + if requires_settings: + major, minor = map(int, requires_settings.split('.')) + if sys.version_info < (major, minor): + pytest.skip(f'requires python {requires_settings}') + + group_name = prefix_settings.get('group') + + if '# ignore-above' in example.source: + eval_example.set_config(ruff_ignore=['E402']) + if group_name: + eval_example.set_config(ruff_ignore=['F821']) + + # eval_example.set_config(line_length=120) + if lint_settings != 'skip': + if eval_example.update_examples: + eval_example.format(example) + else: + eval_example.lint(example) + + if test_settings == 'skip': + return + + group_name = prefix_settings.get('group') + d = group_globals.get(group_name) + + xfail = None + if test_settings and test_settings.startswith('xfail'): + xfail = test_settings[5:].lstrip(' -') + + rewrite_assertions = prefix_settings.get('rewrite_assert', 'true') == 'true' + + try: + if test_settings == 'no-print-intercept': + d2 = eval_example.run(example, module_globals=d, rewrite_assertions=rewrite_assertions) + elif eval_example.update_examples: + d2 = eval_example.run_print_update(example, module_globals=d, rewrite_assertions=rewrite_assertions) + else: + d2 = eval_example.run_print_check(example, module_globals=d, rewrite_assertions=rewrite_assertions) + except BaseException as e: # run_print_check raises a BaseException + if xfail: + pytest.xfail(f'{xfail}, {type(e).__name__}: {e}') + raise + else: + if xfail: + pytest.fail('expected xfail') + group_globals.set(group_name, d2) diff --git a/tests/test_settings.py b/tests/test_settings.py index af32a240..b4ea8fe6 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -17,6 +17,8 @@ HttpUrl, SecretStr, ValidationError, +) +from pydantic import ( dataclasses as pydantic_dataclasses, ) from pydantic.fields import FieldInfo