From 3f724ecacbd832898753c1742d8cc9a30a0555e8 Mon Sep 17 00:00:00 2001 From: Phil Salant Date: Fri, 30 Sep 2022 18:50:25 -0600 Subject: [PATCH 1/7] Correctly `isort` `versioneer` in files --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 8acba256a68..1cbbbd871ac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,6 +80,7 @@ exclude = ''' ## isort ## ########### [tool.isort] +known_first_party = ["versioneer"] profile = "black" line_length = 88 src_paths = ["src", "tests", "noxfiles"] From 9f5d78af82b8fc37ad0e558336de07c7d44add40 Mon Sep 17 00:00:00 2001 From: Phil Salant Date: Fri, 30 Sep 2022 18:52:15 -0600 Subject: [PATCH 2/7] Make `optional-requirements.txt` a source of truth --- setup.py | 38 ++++++++++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/setup.py b/setup.py index 9c584f2b204..b925ebd2647 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,5 @@ import pathlib +from typing import List from setuptools import find_packages, setup @@ -13,18 +14,35 @@ install_requires = open("requirements.txt").read().strip().split("\n") dev_requires = open("dev-requirements.txt").read().strip().split("\n") +optional_requires = open("optional-requirements.txt").read().strip().split("\n") + + +def optional_requirements(dependency_names: List[str]) -> List[str]: + requirements: List[str] = [] + + for dependency in dependency_names: + for optional_dependency in optional_requires: + if dependency in optional_dependency: + requirements.append(optional_dependency) + break + + if len(requirements) == len(dependency_names): + return requirements + + raise ModuleNotFoundError + # Human-Readable Extras -# Add these to `optional-requirements.txt` as well for Docker caching -aws = ["boto3~=1.24.46"] -bigquery = ["sqlalchemy-bigquery==1.4.4"] -mongo = ["pymongo==3.12.0"] -mssql = ["pyodbc==4.0.34"] -mysql = ["pymysql==1.0.2"] -okta = ["okta==2.5.0"] -redis = ["redis==3.5.3", "fastapi-caching[redis]"] -redshift = ["sqlalchemy-redshift==0.8.11"] -snowflake = ["snowflake-sqlalchemy==1.4.1"] +# Versions are read from corresponding lines in `optional-requirements.txt` +aws = optional_requirements(["boto3"]) +bigquery = optional_requirements(["sqlalchemy-bigquery"]) +mongo = optional_requirements(["pymongo"]) +mssql = optional_requirements(["pyodbc"]) +mysql = optional_requirements(["PyMySQL"]) +okta = optional_requirements(["okta"]) +redis = optional_requirements(["redis", "fastapi-caching[redis]"]) +redshift = optional_requirements(["sqlalchemy-redshift"]) +snowflake = optional_requirements(["snowflake-sqlalchemy"]) extras = { "aws": aws, From 4ca839d799d1410c036ff73362025ef255d61133 Mon Sep 17 00:00:00 2001 From: Phil Salant Date: Fri, 30 Sep 2022 18:57:31 -0600 Subject: [PATCH 3/7] Update `CHANGELOG.md` --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8eb9c2c2362..3bbf22ee778 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,10 @@ The types of changes are: * Changed behavior of adding privacy declarations to decouple the actions of the "add" and "next" buttons [#1086](https://github.com/ethyca/fides/pull/1086) * Moved system related UI components from the `config-wizard` directory to the `system` directory [#1097](https://github.com/ethyca/fides/pull/1097) +### Developer Experience + +* Optional dependencies may have their version defined only once, in `optional-requirements.txt` [#1171](https://github.com/ethyca/fides/pull/1171) + ### Fixed * Fixed the "help" link in the UI header [#1078](https://github.com/ethyca/fides/pull/1078) From dcc907b54984a5b39a1f2b63f9e6ce9323020e70 Mon Sep 17 00:00:00 2001 From: Phil Salant Date: Fri, 30 Sep 2022 19:04:21 -0600 Subject: [PATCH 4/7] Include `optional_requirements.txt` in the package --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST.in b/MANIFEST.in index 91d47156966..1f62a87c7da 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,6 +2,7 @@ include LICENSE include README.md include requirements.txt include dev-requirements.txt +include optional-requirements.txt include versioneer.py include src/fides/api/ctl/alembic.ini include src/fides/_version.py From 47afe0c9def569490af00926cccc3262913b0128 Mon Sep 17 00:00:00 2001 From: Phil Salant Date: Mon, 3 Oct 2022 11:10:45 -0600 Subject: [PATCH 5/7] Specify an encoding when opening requirements files Quiets the pylint warning --- setup.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index b925ebd2647..efa90dc894a 100644 --- a/setup.py +++ b/setup.py @@ -6,15 +6,17 @@ import versioneer here = pathlib.Path(__file__).parent.resolve() -long_description = open("README.md").read() +long_description = open("README.md", encoding="utf-8").read() ################## ## Requirements ## ################## -install_requires = open("requirements.txt").read().strip().split("\n") -dev_requires = open("dev-requirements.txt").read().strip().split("\n") -optional_requires = open("optional-requirements.txt").read().strip().split("\n") +install_requires = open("requirements.txt", encoding="utf-8").read().strip().split("\n") +dev_requires = open("dev-requirements.txt", encoding="utf-8").read().strip().split("\n") +optional_requires = ( + open("optional-requirements.txt", encoding="utf-8").read().strip().split("\n") +) def optional_requirements(dependency_names: List[str]) -> List[str]: From e8b2ec8fbcede2417a8a8ddda42d3507fd4bd263 Mon Sep 17 00:00:00 2001 From: Phil Salant Date: Mon, 3 Oct 2022 11:11:23 -0600 Subject: [PATCH 6/7] Remove superfluous variables --- setup.py | 28 +++++++++------------------- 1 file changed, 9 insertions(+), 19 deletions(-) diff --git a/setup.py b/setup.py index efa90dc894a..1b2ee607fb3 100644 --- a/setup.py +++ b/setup.py @@ -36,26 +36,16 @@ def optional_requirements(dependency_names: List[str]) -> List[str]: # Human-Readable Extras # Versions are read from corresponding lines in `optional-requirements.txt` -aws = optional_requirements(["boto3"]) -bigquery = optional_requirements(["sqlalchemy-bigquery"]) -mongo = optional_requirements(["pymongo"]) -mssql = optional_requirements(["pyodbc"]) -mysql = optional_requirements(["PyMySQL"]) -okta = optional_requirements(["okta"]) -redis = optional_requirements(["redis", "fastapi-caching[redis]"]) -redshift = optional_requirements(["sqlalchemy-redshift"]) -snowflake = optional_requirements(["snowflake-sqlalchemy"]) - extras = { - "aws": aws, - "bigquery": bigquery, - "mongo": mongo, - "mssql": mssql, - "mysql": mysql, - "okta": okta, - "redis": redis, - "redshift": redshift, - "snowflake": snowflake, + "aws": optional_requirements(["boto3"]), + "bigquery": optional_requirements(["sqlalchemy-bigquery"]), + "mongo": optional_requirements(["pymongo"]), + "mssql": optional_requirements(["pyodbc"]), + "mysql": optional_requirements(["PyMySQL"]), + "okta": optional_requirements(["okta"]), + "redis": optional_requirements(["redis", "fastapi-caching[redis]"]), + "redshift": optional_requirements(["sqlalchemy-redshift"]), + "snowflake": optional_requirements(["snowflake-sqlalchemy"]), } dangerous_extras = ["mssql"] # These extras break on certain platforms extras["all"] = sum( From 8f539a1d418ba47b03a9c0317a7f7b7a057da12d Mon Sep 17 00:00:00 2001 From: Phil Salant Date: Mon, 3 Oct 2022 11:19:23 -0600 Subject: [PATCH 7/7] Match dependency names more strictly, add docstring --- setup.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1b2ee607fb3..ce14e02ffde 100644 --- a/setup.py +++ b/setup.py @@ -20,11 +20,18 @@ def optional_requirements(dependency_names: List[str]) -> List[str]: + """ + Matches the provided dependency names to lines in `optional-requirements.txt`, + and returns the full dependency string for each one. + + Prevents the need to store version numbers in two places. + """ + requirements: List[str] = [] for dependency in dependency_names: for optional_dependency in optional_requires: - if dependency in optional_dependency: + if optional_dependency.startswith(dependency): requirements.append(optional_dependency) break