Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: support reading/loading multiple .env files #424

Closed
wants to merge 29 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
71f1730
accept multiple files in dotenv run
duarte-pompeu Sep 10, 2022
5f7d1f8
allow multiple files for other cli targets
duarte-pompeu Sep 10, 2022
6718b16
fix behavior or adapt tests
duarte-pompeu Sep 10, 2022
b4f335b
improve readability
duarte-pompeu Sep 10, 2022
0660ded
remove f-string
duarte-pompeu Sep 10, 2022
d46b1e6
improve formatting
duarte-pompeu Sep 10, 2022
bc474ca
restore code to test for warning
duarte-pompeu Sep 10, 2022
2b3a28b
format code
duarte-pompeu Oct 1, 2022
f730c6d
add flag to allow suppressing warning in get_key
duarte-pompeu Oct 1, 2022
d6c1385
fix warning and test in cli
duarte-pompeu Oct 1, 2022
630c0bd
Merge branch 'main' into multiple-files
duarte-pompeu Dec 4, 2023
c585a7b
fix lint warnings
duarte-pompeu Dec 4, 2023
9b1b4af
fix some problems
duarte-pompeu Dec 4, 2023
a2d9365
fix tests
duarte-pompeu Dec 4, 2023
f5ed3e9
fix type hint
duarte-pompeu Dec 4, 2023
49c34a5
add some tests
duarte-pompeu Dec 4, 2023
437a921
remove support for unsetting multiple files
duarte-pompeu Dec 8, 2023
a10e462
improve backward compatibility
duarte-pompeu Dec 8, 2023
186dc81
remove unused logger
duarte-pompeu Dec 8, 2023
dde1654
remove support for set on multiple files
duarte-pompeu Dec 8, 2023
ec23819
print errors to stderr
duarte-pompeu Dec 8, 2023
febc153
improve params for test_lit_multi_file
duarte-pompeu Dec 8, 2023
c52dd9e
test get for multiple files
duarte-pompeu Dec 8, 2023
9e3df30
test run with multiple envs
duarte-pompeu Dec 8, 2023
6090d64
update docs
duarte-pompeu Dec 8, 2023
d61c1ed
remove unecessary param
duarte-pompeu Dec 8, 2023
46f1b75
remove format param for test_list_multi_file
duarte-pompeu Dec 8, 2023
363aab5
document loading order of multiple .envs
duarte-pompeu Dec 28, 2023
1d33b6a
revert change in description
duarte-pompeu Dec 28, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,18 @@ $ dotenv list --format=json
$ dotenv run -- python foo.py
```

The CLI interface also supports reading from multiple `.env` files.
Each file overrides the previous: if there are duplicate keys, the last file determines the final value.

```shell
$ echo -e "a=1\nb=2\n" > .env1
$ echo -e "b=20\nc=30\n" > .env2
$ python -m dotenv -f .env1 -f .env2 list
a=1
b=20
c=30
```
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@duarte-pompeu Consider document the precedence/loading order values from the of the files. I didn't see a test but based on the this example it seems the .env2 overrides the previous value from .env1.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I agree: it's important to document this. I chose an overriding behavior, as my use case would be:

  • have a big file with all values
  • have smaller .env files with specific values to override in certain situations

I tried to improve it on 363aab5


Run `dotenv --help` for more information about the options and subcommands.

## File format
Expand Down
72 changes: 49 additions & 23 deletions src/dotenv/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,14 +26,15 @@ def enumerate_env():
try:
cwd = os.getcwd()
except FileNotFoundError:
return None
return []
path = os.path.join(cwd, '.env')
return path
return [path]


@click.group()
@click.option('-f', '--file', default=enumerate_env(),
type=click.Path(file_okay=True),
multiple=True,
help="Location of the .env file, defaults to .env file in current working directory.")
@click.option('-q', '--quote', default='always',
type=click.Choice(['always', 'never', 'auto']),
Expand All @@ -43,9 +44,9 @@ def enumerate_env():
help="Whether to write the dot file as an executable bash script.")
@click.version_option(version=__version__)
@click.pass_context
def cli(ctx: click.Context, file: Any, quote: Any, export: Any) -> None:
def cli(ctx: click.Context, file: List[Any], quote: Any, export: Any) -> None:
"""This script is used to set, get or unset values from a .env file."""
ctx.obj = {'QUOTE': quote, 'EXPORT': export, 'FILE': file}
ctx.obj = {'QUOTE': quote, 'EXPORT': export, 'FILES': file}


@contextmanager
Expand All @@ -72,10 +73,13 @@ def stream_file(path: os.PathLike) -> Iterator[IO[str]]:
"which displays name=value without quotes.")
def list(ctx: click.Context, format: bool) -> None:
"""Display all the stored key/value."""
file = ctx.obj['FILE']
files = ctx.obj['FILES']

with stream_file(file) as stream:
values = dotenv_values(stream=stream)
values = {}
for file in files:
with stream_file(file) as stream:
file_values = dotenv_values(stream=stream)
values.update(file_values)

if format == 'json':
click.echo(json.dumps(values, indent=2, sort_keys=True))
Expand All @@ -95,9 +99,16 @@ def list(ctx: click.Context, format: bool) -> None:
@click.argument('value', required=True)
def set(ctx: click.Context, key: Any, value: Any) -> None:
"""Store the given key/value."""
file = ctx.obj['FILE']
files = ctx.obj['FILES']
quote = ctx.obj['QUOTE']
export = ctx.obj['EXPORT']

if len(files) > 1:
click.echo(f"Set is not supported for multiple files: {[str(f) for f in files]}.", err=True)
exit(1)

file = files[0]

success, key, value = set_key(file, key, value, quote, export)
if success:
click.echo(f'{key}={value}')
Expand All @@ -110,10 +121,13 @@ def set(ctx: click.Context, key: Any, value: Any) -> None:
@click.argument('key', required=True)
def get(ctx: click.Context, key: Any) -> None:
"""Retrieve the value for the given key."""
file = ctx.obj['FILE']
files = ctx.obj['FILES']
values = {}

with stream_file(file) as stream:
values = dotenv_values(stream=stream)
for file in files:
with stream_file(file) as stream:
file_values = dotenv_values(stream=stream)
values.update(file_values)

stored_value = values.get(key)
if stored_value:
Expand All @@ -127,8 +141,15 @@ def get(ctx: click.Context, key: Any) -> None:
@click.argument('key', required=True)
def unset(ctx: click.Context, key: Any) -> None:
"""Removes the given key."""
file = ctx.obj['FILE']
files = ctx.obj['FILES']
quote = ctx.obj['QUOTE']

if len(files) > 1:
click.echo(f"Unset is not supported for multiple files: {[str(f) for f in files]}.", err=True)
exit(1)

file = files[0]

success, key = unset_key(file, key, quote)
if success:
click.echo(f"Successfully removed {key}")
Expand All @@ -146,17 +167,22 @@ def unset(ctx: click.Context, key: Any) -> None:
@click.argument('commandline', nargs=-1, type=click.UNPROCESSED)
def run(ctx: click.Context, override: bool, commandline: List[str]) -> None:
"""Run command with environment variables present."""
file = ctx.obj['FILE']
if not os.path.isfile(file):
raise click.BadParameter(
f'Invalid value for \'-f\' "{file}" does not exist.',
ctx=ctx
)
dotenv_as_dict = {
k: v
for (k, v) in dotenv_values(file).items()
if v is not None and (override or k not in os.environ)
}

files = ctx.obj['FILES']

dotenv_as_dict = {}
for file in files:
if not os.path.isfile(file):
raise click.BadParameter(
f'Invalid value for \'-f\' "{file}" does not exist.',
ctx=ctx
)
file_dotenv_as_dict = {
k: v
for (k, v) in dotenv_values(file).items()
if v is not None and (override or k not in os.environ)
}
dotenv_as_dict.update(file_dotenv_as_dict)

if not commandline:
click.echo('No command given.')
Expand Down
7 changes: 7 additions & 0 deletions tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,3 +14,10 @@ def dotenv_path(tmp_path):
path = tmp_path / '.env'
path.write_bytes(b'')
yield path


@pytest.fixture
def extra_dotenv_path(tmp_path):
path = tmp_path / '.env_extra'
path.write_bytes(b'')
yield path
80 changes: 76 additions & 4 deletions tests/test_cli.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import os
import sh
from pathlib import Path
from typing import Optional
from typing import Dict, List, Optional

import pytest

Expand All @@ -25,7 +25,7 @@
("export", "x='a b c'", '''export x='a b c'\n'''),
)
)
def test_list(cli, dotenv_path, format: Optional[str], content: str, expected: str):
def test_list_single_file(cli, dotenv_path, format: Optional[str], content: str, expected: str):
dotenv_path.write_text(content + '\n')

args = ['--file', dotenv_path, 'list']
Expand All @@ -37,6 +37,25 @@ def test_list(cli, dotenv_path, format: Optional[str], content: str, expected: s
assert (result.exit_code, result.output) == (0, expected)


@pytest.mark.parametrize(
"contents,expected",
(
(["x='1'", "y='2'"], '''x=1\ny=2\n'''),
(["x='1'", "x='2'"], '''x=2\n'''),
(["x='1'\ny='2'", "y='20'\nz='30'"], '''x=1\ny=20\nz=30\n'''),
)
)
def test_list_multi_file(cli, dotenv_path, extra_dotenv_path, contents: List[str], expected: str):
dotenv_path.write_text(contents[0] + '\n')
extra_dotenv_path.write_text(contents[1] + '\n')

args = ['--file', dotenv_path, '--file', extra_dotenv_path, 'list']
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I omitted tests with multiple formatting because that seems to me like a separate concern - if the variables are loaded correctly, the formatting behavior shouldn't change.

Do you agree?

Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You are right. This is fine.


result = cli.invoke(dotenv_cli, args)

assert (result.exit_code, result.output) == (0, expected)


def test_list_non_existent_file(cli):
result = cli.invoke(dotenv_cli, ['--file', 'nx_file', 'list'])

Expand All @@ -57,17 +76,40 @@ def test_list_no_file(cli):
assert (result.exit_code, result.output) == (1, "")


def test_get_existing_value(cli, dotenv_path):
def test_get_existing_value_single_file(cli, dotenv_path):
dotenv_path.write_text("a=b")

result = cli.invoke(dotenv_cli, ['--file', dotenv_path, 'get', 'a'])

assert (result.exit_code, result.output) == (0, "b\n")


@pytest.mark.parametrize(
"contents,expected_values",
(
(["a=1", "b=2"], {"a": "1", "b": "2"}),
(["b=2", "a=1"], {"a": "1", "b": "2"}),
(["a=1", "a=2"], {"a": "2"}),
)
)
def test_get_existing_value_multi_file(
cli,
dotenv_path,
extra_dotenv_path,
contents: List[str],
expected_values: Dict[str, str]
):
dotenv_path.write_text(contents[0])
extra_dotenv_path.write_text(contents[1])

for key, value in expected_values.items():
result = cli.invoke(dotenv_cli, ['--file', dotenv_path, '--file', extra_dotenv_path, 'get', key])

assert (result.exit_code, result.output) == (0, f"{value}\n")


def test_get_non_existent_value(cli, dotenv_path):
result = cli.invoke(dotenv_cli, ['--file', dotenv_path, 'get', 'a'])

assert (result.exit_code, result.output) == (1, "")


Expand Down Expand Up @@ -101,6 +143,12 @@ def test_unset_non_existent_value(cli, dotenv_path):
assert dotenv_path.read_text() == ""


def test_unset_multi_file_not_allowed(cli, dotenv_path, extra_dotenv_path):
result = cli.invoke(dotenv_cli, ['--file', dotenv_path, '--file', extra_dotenv_path, 'unset', 'a'])
assert result.exit_code == 1
assert result.output == f"Unset is not supported for multiple files: ['{dotenv_path}', '{extra_dotenv_path}'].\n"


@pytest.mark.parametrize(
"quote_mode,variable,value,expected",
(
Expand Down Expand Up @@ -151,6 +199,12 @@ def test_set_no_file(cli):
assert "Missing argument" in result.output


def test_set_multi_file_not_allowed(cli, dotenv_path, extra_dotenv_path):
result = cli.invoke(dotenv_cli, ['--file', dotenv_path, '--file', extra_dotenv_path, 'set', 'a', 'b'])
assert result.exit_code == 1
assert result.output == f"Set is not supported for multiple files: ['{dotenv_path}', '{extra_dotenv_path}'].\n"


def test_get_default_path(tmp_path):
with sh.pushd(tmp_path):
(tmp_path / ".env").write_text("a=b")
Expand Down Expand Up @@ -208,6 +262,24 @@ def test_run_with_other_env(dotenv_path):
assert result == "b\n"


@pytest.mark.parametrize(
"contents,expected_values",
(
(["a=1", "b=2"], {"a": "1", "b": "2"}),
(["b=2", "a=1"], {"a": "1", "b": "2"}),
(["a=1", "a=2"], {"a": "2"}),
)
)
def test_run_with_multi_envs(dotenv_path, extra_dotenv_path, contents: List[str], expected_values: Dict[str, str]):
dotenv_path.write_text(contents[0])
extra_dotenv_path.write_text(contents[1])

for key, value in expected_values.items():
result = sh.dotenv("--file", dotenv_path, '--file', extra_dotenv_path, "run", "printenv", key)

assert result == f"{value}\n"


def test_run_without_cmd(cli):
result = cli.invoke(dotenv_cli, ['run'])

Expand Down