Add docker-style file variable support (fixes joke2k#189) (joke2k#328)

README.rst

@@ -113,6 +113,8 @@ a concrete example on using with a django project.
 - Fill ``os.environ`` with .env file variables
 - Variables casting
 - Url variables exploded to django specific package settings
+- Optional support for Docker-style file based config variables (use
+  ``environ.FileAwareEnv`` instead of ``environ.Env``)
 
 .. -project-information-
 

docs/tips.rst

@@ -3,6 +3,42 @@ Tips
 ====
 
 
+Docker-style file based variables
+=================================
+
+Docker (swarm) and Kubernetes are two widely used platforms that store their
+secrets in tmpfs inside containers as individual files, providing a secure way
+to be able to share configuration data between containers.
+
+Use ``environ.FileAwareEnv`` rather than ``environ.Env`` to first look for
+environment variables with ``_FILE`` appended. If found, their contents will be
+read from the file system and used instead.
+
+For example, given an app with the following in its settings module:
+
+.. code-block:: python
+
+   import environ
+
+   env = environ.FileAwareEnv()
+   SECRET_KEY = env("SECRET_KEY")
+
+the example ``docker-compose.yml`` for would contain:
+
+.. code-block:: yaml
+
+   secrets:
+     secret_key:
+       external: true
+
+   services:
+     app:
+       secrets:
+         - secret_key
+       environment:
+         - SECRET_KEY_FILE=/run/secrets/secret_key
+
+
 Using unsafe characters in URLs
 ===============================
 
@@ -14,6 +50,7 @@ In order to use unsafe characters you have to encode with ``urllib.parse.encode`
 
 See https://perishablepress.com/stop-using-unsafe-characters-in-urls/ for reference.
 
+
 Smart Casting
 =============
 

environ/environ.py

@@ -27,6 +27,7 @@
 )
 
 from .compat import DJANGO_POSTGRES, ImproperlyConfigured, json, REDIS_DRIVER
+from .fileaware_mapping import FileAwareMapping
 
 try:
     from os import PathLike
@@ -786,6 +787,24 @@ def read_env(cls, env_file=None, **overrides):
             cls.ENVIRON.setdefault(key, value)
 
 
+class FileAwareEnv(Env):
+    """
+    First look for environment variables with ``_FILE`` appended. If found,
+    their contents will be read from the file system and used instead.
+
+    Use as a drop-in replacement for the standard ``environ.Env``:
+
+    .. code-block:: python
+
+        python env = environ.FileAwareEnv()
+
+    For example, if a ``SECRET_KEY_FILE`` environment variable was set,
+    ``env("SECRET_KEY")`` would find the related variable, returning the file
+    contents rather than ever looking up a ``SECRET_KEY`` environment variable.
+    """
+    ENVIRON = FileAwareMapping()
+
+
 class Path:
     """Inspired to Django Two-scoops, handling File Paths in Settings."""
 

environ/fileaware_mapping.py

@@ -0,0 +1,82 @@
+import os
+from collections.abc import MutableMapping
+
+
+class FileAwareMapping(MutableMapping):
+    """
+    A mapping that wraps os.environ, first checking for the existance of a key
+    appended with ``_FILE`` whenever reading a value. If a matching file key is
+    found then the value is instead read from the file system at this location.
+
+    By default, values read from the file system are cached so future lookups
+    do not hit the disk again.
+
+    A ``_FILE`` key has higher precidence than a value is set directly in the
+    environment, and an exception is raised if the file can not be found.
+    """
+
+    def __init__(self, env=None, cache=True):
+        """
+        Initialize the mapping.
+
+        :param env:
+            where to read environment variables from (defaults to
+            ``os.environ``)
+        :param cache:
+            cache environment variables read from the file system (defaults to
+            ``True``)
+        """
+        self.env = env if env is not None else os.environ
+        self.cache = cache
+        self.files_cache = {}
+
+    def __getitem__(self, key):
+        if self.cache and key in self.files_cache:
+            return self.files_cache[key]
+        key_file = self.env.get(key + "_FILE")
+        if key_file:
+            with open(key_file) as f:
+                value = f.read()
+            if self.cache:
+                self.files_cache[key] = value
+            return value
+        return self.env[key]
+
+    def __iter__(self):
+        """
+        Iterate all keys, also always including the shortened key if ``_FILE``
+        keys are found.
+        """
+        for key in self.env:
+            yield key
+            if key.endswith("_FILE"):
+                no_file_key = key[:-5]
+                if no_file_key and no_file_key not in self.env:
+                    yield no_file_key
+
+    def __len__(self):
+        """
+        Return the length of the file, also always counting shortened keys for
+        any ``_FILE`` key found.
+        """
+        return len(tuple(iter(self)))
+
+    def __setitem__(self, key, value):
+        self.env[key] = value
+        if self.cache and key.endswith("_FILE"):
+            no_file_key = key[:-5]
+            if no_file_key and no_file_key in self.files_cache:
+                del self.files_cache[no_file_key]
+
+    def __delitem__(self, key):
+        file_key = key + "_FILE"
+        if file_key in self.env:
+            del self[file_key]
+            if key in self.env:
+                del self.env[key]
+            return
+        if self.cache and key.endswith("_FILE"):
+            no_file_key = key[:-5]
+            if no_file_key and no_file_key in self.files_cache:
+                del self.files_cache[no_file_key]
+        del self.env[key]

tests/test_fileaware.py

@@ -0,0 +1,180 @@
+import os
+import tempfile
+from contextlib import contextmanager
+
+import environ
+import pytest
+
+
+@contextmanager
+def make_temp_file(text):
+    with tempfile.NamedTemporaryFile("w", delete=False) as f:
+        f.write(text)
+        f.close()
+    try:
+        yield f.name
+    finally:
+        if os.path.exists(f.name):
+            os.unlink(f.name)
+
+
+@pytest.fixture
+def tmp_f():
+    with make_temp_file(text="fish") as f_name:
+        yield f_name
+
+
+def test_mapping(tmp_f):
+    env = environ.FileAwareMapping(env={"ANIMAL_FILE": tmp_f})
+    assert env["ANIMAL"] == "fish"
+
+
+def test_precidence(tmp_f):
+    env = environ.FileAwareMapping(
+        env={
+            "ANIMAL_FILE": tmp_f,
+            "ANIMAL": "cat",
+        }
+    )
+    assert env["ANIMAL"] == "fish"
+
+
+def test_missing_file_raises_exception():
+    env = environ.FileAwareMapping(env={"ANIMAL_FILE": "non-existant-file"})
+    with pytest.raises(FileNotFoundError):
+        env["ANIMAL"]
+
+
+def test_iter():
+    env = environ.FileAwareMapping(
+        env={
+            "ANIMAL_FILE": "some-file",
+            "VEGETABLE": "leek",
+            "VEGETABLE_FILE": "some-vegetable-file",
+        }
+    )
+    keys = set(env)
+    assert keys == {"ANIMAL_FILE", "ANIMAL", "VEGETABLE", "VEGETABLE_FILE"}
+    assert "ANIMAL" in keys
+
+
+def test_len():
+    env = environ.FileAwareMapping(
+        env={
+            "ANIMAL_FILE": "some-file",
+            "VEGETABLE": "leek",
+            "VEGETABLE_FILE": "some-vegetable-file",
+        }
+    )
+    assert len(env) == 4
+
+
+def test_cache(tmp_f):
+    env = environ.FileAwareMapping(env={"ANIMAL_FILE": tmp_f})
+    assert env["ANIMAL"] == "fish"
+
+    with open(tmp_f, "w") as f:
+        f.write("cat")
+    assert env["ANIMAL"] == "fish"
+
+    os.unlink(tmp_f)
+    assert not os.path.exists(env["ANIMAL_FILE"])
+    assert env["ANIMAL"] == "fish"
+
+
+def test_no_cache(tmp_f):
+    env = environ.FileAwareMapping(
+        cache=False,
+        env={"ANIMAL_FILE": tmp_f},
+    )
+    assert env["ANIMAL"] == "fish"
+
+    with open(tmp_f, "w") as f:
+        f.write("cat")
+    assert env["ANIMAL"] == "cat"
+
+    os.unlink(tmp_f)
+    assert not os.path.exists(env["ANIMAL_FILE"])
+    with pytest.raises(FileNotFoundError):
+        assert env["ANIMAL"]
+
+
+def test_setdefault(tmp_f):
+    env = environ.FileAwareMapping(env={"ANIMAL_FILE": tmp_f})
+    assert env.setdefault("FRUIT", "apple") == "apple"
+    assert env.setdefault("ANIMAL", "cat") == "fish"
+    assert env.env == {"ANIMAL_FILE": tmp_f, "FRUIT": "apple"}
+
+
+class TestDelItem:
+    def test_del_key(self):
+        env = environ.FileAwareMapping(env={"FRUIT": "apple"})
+        del env["FRUIT"]
+        with pytest.raises(KeyError):
+            env["FRUIT"]
+
+    def test_del_key_with_file_key(self):
+        env = environ.FileAwareMapping(env={"ANIMAL_FILE": "some-file"})
+        del env["ANIMAL"]
+        with pytest.raises(KeyError):
+            env["ANIMAL"]
+
+    def test_del_shadowed_key_with_file_key(self):
+        env = environ.FileAwareMapping(
+            env={"ANIMAL_FILE": "some-file", "ANIMAL": "cat"}
+        )
+        del env["ANIMAL"]
+        with pytest.raises(KeyError):
+            env["ANIMAL"]
+
+    def test_del_file_key(self):
+        env = environ.FileAwareMapping(
+            env={
+                "ANIMAL_FILE": "some-file",
+                "ANIMAL": "fish",
+            }
+        )
+        del env["ANIMAL_FILE"]
+        assert env["ANIMAL"] == "fish"
+
+    def test_del_file_key_clears_cache(self, tmp_f):
+        env = environ.FileAwareMapping(
+            env={
+                "ANIMAL_FILE": tmp_f,
+                "ANIMAL": "cat",
+            }
+        )
+        assert env["ANIMAL"] == "fish"
+        del env["ANIMAL_FILE"]
+        assert env["ANIMAL"] == "cat"
+
+
+class TestSetItem:
+    def test_set_key(self):
+        env = environ.FileAwareMapping(env={"FRUIT": "apple"})
+        env["FRUIT"] = "banana"
+        assert env["FRUIT"] == "banana"
+
+    def test_cant_override_key_with_file_key(self, tmp_f):
+        env = environ.FileAwareMapping(
+            env={
+                "FRUIT": "apple",
+                "FRUIT_FILE": tmp_f,
+            }
+        )
+        with open(tmp_f, "w") as f:
+            f.write("banana")
+        env["FRUIT"] = "cucumber"
+        assert env["FRUIT"] == "banana"
+
+    def test_set_file_key(self, tmp_f):
+        env = environ.FileAwareMapping(env={"ANIMAL": "cat"})
+        env["ANIMAL_FILE"] = tmp_f
+        assert env["ANIMAL"] == "fish"
+
+    def test_change_file_key_clears_cache(self, tmp_f):
+        env = environ.FileAwareMapping(env={"ANIMAL_FILE": tmp_f})
+        assert env["ANIMAL"] == "fish"
+        with make_temp_file(text="cat") as new_tmp_f:
+            env["ANIMAL_FILE"] = new_tmp_f
+            assert env["ANIMAL"] == "cat"