Merge pull request #12 from joshourisman/unescaped-cloud-sql

Unescaped cloud sql

README.md

@@ -2,7 +2,9 @@
 
 ## Use pydantic settings management to simplify configuration of Django settings.
 
-Very much a work in progress, but reads the standard DJANGO_SETTINGS_MODULE environment variable (defaulting to pydantic_settings.Settings) to load a sub-class of pydantic_settings.Settings. All settings (that have been defined in pydantic_settings.Settings) can be overridden with environment variables. A special DatabaseSettings class is used to allow multiple databases to be configured simply with DSNs. In theory, django-pydantic-settings should be compatible with any version of Django that runs on Python 3.6+ (which means Django 1.11 and on), but is only tested against officially supported versions (currently 2.2, 3.0, and 3.1).
+Very much a work in progress, but reads the standard DJANGO_SETTINGS_MODULE environment variable (defaulting to pydantic_settings.settings.PydanticSettings) to load a sub-class of pydantic_settings.Settings. All settings (that have been defined in pydantic_settings.Settings) can be overridden with environment variables. A special DatabaseSettings class is used to allow multiple databases to be configured simply with DSNs. In theory, django-pydantic-settings should be compatible with any version of Django that runs on Python 3.6.1+ (which means Django 1.11 and on), but is only tested against officially supported versions (currently 2.2, 3.0, and 3.1).
+
+Note: as of django-pydantic-settings 0.4.0, Pydantic 1.8+ is required, which means Python 3.6.1+ is also required. If you need to use Python 3.6, you'll need to stick with django-pydantic-settings <0.4.0.
 
 ## Installation & Setup
 
@@ -62,7 +64,7 @@ The other setting worth thinking about is `SECRET_KEY`. By default, `SECRET_KEY`
 
 ## Database configuration
 
-By defining multiple `DatabaseDsn` attributes of the `DatabaseSettings` class, you can easily configure one or more database connections with environment variables. DSNs are parsed using dj-database-url.
+By defining multiple `DatabaseDsn` attributes of the `DatabaseSettings` class, you can easily configure one or more database connections with environment variables. DSNs are parsed using dj-database-url. In order to support Google Cloud SQL database connections from within Google Cloud Run, the DatabaseDsn type will detect and automatically escape DSN strings of the form `postgres://username:password@/cloudsql/project:region:instance/database` so that they can be properly handled by dj-database-url.
 
 ```python
 class DatabaseSettings(BaseSettings):
@@ -71,38 +73,27 @@ class DatabaseSettings(BaseSettings):
 ```
 
 ```python
-❯ DATABASE_URL=sqlite:///foo SECONDARY_DATABASE_URL=sqlite:///bar ./settings_test/manage.py shell
-Python 3.9.1 (default, Jan  8 2021, 17:17:43)
-[Clang 12.0.0 (clang-1200.0.32.28)] on darwin
+❯ DATABASE_URL=postgres://username:password@/cloudsql/project:region:instance/database SECONDARY_DATABASE_URL=sqlite:///foo poetry run python settings_test/manage.py shell
+Python 3.9.1 (default, Jan 12 2021, 16:45:25) 
+[GCC 8.3.0] on linux
 Type "help", "copyright", "credits" or "license" for more information.
 (InteractiveConsole)
+>>> from rich import print
 >>> from django.conf import settings
-...
->>> pp.pprint(settings.DATABASES)
-{   'default': {   'ATOMIC_REQUESTS': False,
-                   'AUTOCOMMIT': True,
-                   'CONN_MAX_AGE': 0,
-                   'ENGINE': 'django.db.backends.sqlite3',
-                   'HOST': '',
-                   'NAME': 'foo',
-                   'OPTIONS': {},
-                   'PASSWORD': '',
-                   'PORT': '',
-                   'TEST': {   'CHARSET': None,
-                               'COLLATION': None,
-                               'MIGRATE': True,
-                               'MIRROR': None,
-                               'NAME': None},
-                   'TIME_ZONE': None,
-                   'USER': ''},
-    'secondary': {   'CONN_MAX_AGE': 0,
-                     'ENGINE': 'django.db.backends.sqlite3',
-                     'HOST': '',
-                     'NAME': 'bar',
-                     'PASSWORD': '',
-                     'PORT': '',
-                     'USER': ''}}
->>>
+>>> print(settings.DATABASES)
+{
+    'default': {
+        'NAME': 'database',
+        'USER': 'username',
+        'PASSWORD': 'password',
+        'HOST': '/cloudsql/project:region:instance',
+        'PORT': '',
+        'CONN_MAX_AGE': 0,
+        'ENGINE': 'django.db.backends.postgresql_psycopg2'
+    },
+    'secondary': {'NAME': 'foo', 'USER': '', 'PASSWORD': '', 'HOST': '', 'PORT': '', 'CONN_MAX_AGE': 0, 'ENGINE': 'django.db.backends.sqlite3'}
+}
+>>> 
 ```
 
 ## Sentry configuration

pydantic_settings/database.py

@@ -1,6 +1,23 @@
-from typing import Dict, Optional, Tuple
+import re
+from typing import Dict, Optional, Pattern, Tuple, cast
+from urllib.parse import quote_plus
 
 from pydantic import AnyUrl
+from pydantic.validators import constr_length_validator, str_validator
+
+_cloud_sql_regex_cache = None
+
+
+def cloud_sql_regex() -> Pattern[str]:
+    global _cloud_sql_regex_cache
+    if _cloud_sql_regex_cache is None:
+        _cloud_sql_regex_cache = re.compile(
+            r"(?:(?P<scheme>[a-z][a-z0-9+\-.]+)://)?"  # scheme https://tools.ietf.org/html/rfc3986#appendix-A
+            r"(?:(?P<user>[^\s:/]*)(?::(?P<password>[^\s/]*))?@)?"  # user info
+            r"(?P<path>/[^\s?#]*)?",  # path
+            re.IGNORECASE,
+        )
+    return _cloud_sql_regex_cache
 
 
 class DatabaseDsn(AnyUrl):
@@ -45,6 +62,28 @@ def __init__(
         "redshift",
     }
 
+    @classmethod
+    def validate(cls, value, field, config):
+        if value.__class__ == cls:
+            return value
+
+        value = str_validator(value)
+        if cls.strip_whitespace:
+            value = value.strip()
+
+        url: str = cast(str, constr_length_validator(value, field, config))
+
+        if "/cloudsql/" in url:
+            m = cloud_sql_regex().match(url)
+            if m:
+                parts = m.groupdict()
+                socket, path = parts["path"].rsplit("/", 1)
+                escaped_socket = quote_plus(socket)
+                escaped_dsn = value.replace(socket, escaped_socket)
+                return super().validate(escaped_dsn, field, config)
+
+        return super().validate(value, field, config)
+
     @classmethod
     def validate_host(
         cls, parts: Dict[str, str]

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "django-pydantic-settings"
-version = "0.3.0"
+version = "0.4.0"
 description = "Manage Django settings with Pydantic."
 authors = ["Josh Ourisman <[email protected]>"]
 license = "MIT"
@@ -10,11 +10,11 @@ repository = "https://github.com/joshourisman/django-pydantic-settings"
 packages = [{ include = "pydantic_settings" }]
 
 [tool.poetry.dependencies]
-python = "^3.6"
+python = "^3.6.1"
 Django = ">=1.11"
 dj-database-url = "^0.5.0"
 sentry-sdk = { version = "*", optional = true }
-pydantic = { version = "^1.7.3", extras = ["email"] }
+pydantic = { version = "^1.8", extras = ["email"] }
 typing-extensions = { version = "^3.7.4", python = '<2.8' }
 
 [tool.poetry.extras]

settings_test/tests/test_env.py

@@ -102,7 +102,7 @@ def test_http(monkeypatch):
     assert response.json()["success"] is True
 
 
-def test_gcp_cloudsql_socket(monkeypatch):
+def test_escaped_gcp_cloudsql_socket(monkeypatch):
     monkeypatch.setenv("DJANGO_BASE_DIR", "settings_test")
     monkeypatch.setenv(
         "DATABASE_URL",
@@ -120,3 +120,23 @@ def test_gcp_cloudsql_socket(monkeypatch):
     assert default["PASSWORD"] == "password"
     assert default["HOST"] == "/cloudsql/project:region:instance"
     assert default["ENGINE"] == "django.db.backends.postgresql_psycopg2"
+
+
+def test_unescaped_gcp_cloudsql_socket(monkeypatch):
+    monkeypatch.setenv("DJANGO_BASE_DIR", "settings_test")
+    monkeypatch.setenv(
+        "DATABASE_URL",
+        "postgres://username:password@/cloudsql/project:region:instance/database",
+    )
+
+    settings._wrapped = empty
+    SetUp().configure()
+
+    assert "default" in settings.DATABASES
+
+    default = settings.DATABASES["default"]
+    assert default["NAME"] == "database"
+    assert default["USER"] == "username"
+    assert default["PASSWORD"] == "password"
+    assert default["HOST"] == "/cloudsql/project:region:instance"
+    assert default["ENGINE"] == "django.db.backends.postgresql_psycopg2"