Move fideslib files into fides (#1859)

.fides/fides.toml

@@ -49,3 +49,8 @@ task_retry_delay = 1
 
 [admin_ui]
 enabled = true
+
+[celery]
+event_queue_prefix = "fides_worker"
+task_default_queue = "fides"
+task_always_eager = true

.github/workflows/backend_checks.yml

@@ -84,6 +84,7 @@ jobs:
           - "pytest_ctl(not-external)"
           - "pytest_ops(unit)"
           - "pytest_ops(integration)"
+          - "pytest_lib"
           - "check_fides_annotations"
           - "fides_db_scan"
           - "docs_check"

.gitignore

@@ -1,5 +1,7 @@
 # Source for the following rules: https://raw.githubusercontent.com/github/gitignore/master/Python.gitignore
 
+# Files to keep that would otherwise get ignored
+
 # frontend
 ui-build/
 
@@ -115,8 +117,8 @@ develop-eggs/
 dist/
 downloads/
 eggs/
-lib/
 lib64/
+/lib/
 parts/
 sdist/
 var/

CHANGELOG.md

@@ -46,6 +46,8 @@ The types of changes are:
 
 ### Changed
 
+* The `celery.toml` is no longer used, instead it is a subsection of the `fides.toml` file [#1990](https://github.com/ethyca/fides/pull/1990)
+* The `fideslib` module has been merged into `fides`, code redundancies have been removed [#1859](https://github.com/ethyca/fides/pull/1859)
 * Update sample project landing page copy to be version-agnostic [#1958](https://github.com/ethyca/fides/pull/1958)
 * `get` and `ls` CLI commands now return valid `fides` object YAML [#1991](https://github.com/ethyca/fides/pull/1991)
 

docs/fides/docs/installation/configuration.md

@@ -159,6 +159,11 @@ task_retry_delay = 1
 [admin_ui]
 enabled = true
 
+[celery]
+event_queue_prefix = "fides_worker"
+task_default_queue = "fides"
+task_always_eager = true
+
 ```
 
 ## Configuration variable reference
@@ -257,6 +262,10 @@ The credentials section uses custom keys which can be referenced in certain comm
 | `my_okta.orgUrl` | String | Sets the `orgUrl` for `my_okta` credentials |
 | `my_okta.token` | String | Sets the `token` for `my_okta` credentials |
 
+#### Celery
+
+The celery section is a place for user-defined key-value overrides that get passed directly to Celery and are not validated by `fides`.
+
 #### Admin UI
 
 | Name | Type | Default | Description |

noxfiles/ci_nox.py

@@ -119,14 +119,14 @@ def check_install(session: nox.Session) -> None:
 
 @nox.session()
 def check_fides_annotations(session: nox.Session) -> None:
-    """Run a fidesctl evaluation."""
+    """Run a fides evaluation."""
     run_command = (*RUN_NO_DEPS, "fides", "--local", *(WITH_TEST_CONFIG), "evaluate")
     session.run(*run_command, external=True)
 
 
 @nox.session()
 def fides_db_scan(session: nox.Session) -> None:
-    """Scan the fidesctl application database to check for dataset discrepancies."""
+    """Scan the fides application database to check for dataset discrepancies."""
     session.notify("teardown")
     session.run(*START_APP, external=True)
     run_command = (
@@ -163,6 +163,19 @@ def minimal_config_startup(session: nox.Session) -> None:
 
 
 # Pytest
+@nox.session()
+def pytest_lib(session: nox.Session) -> None:
+    """Runs ctl tests."""
+    session.notify("teardown")
+    session.run(*START_APP, external=True)
+    run_command = (
+        *RUN_NO_DEPS,
+        "pytest",
+        "tests/lib/",
+    )
+    session.run(*run_command, external=True)
+
+
 @nox.session()
 @nox.parametrize(
     "mark",
@@ -174,7 +187,7 @@ def minimal_config_startup(session: nox.Session) -> None:
     ],
 )
 def pytest_ctl(session: nox.Session, mark: str) -> None:
-    """Runs fidesctl tests."""
+    """Runs ctl tests."""
     session.notify("teardown")
     if mark == "external":
         start_command = (

pyproject.toml

@@ -41,7 +41,6 @@ module = [
   "dask.*",
   "deepdiff.*",
   "fideslang.*",
-  "fideslib.*",
   "fideslog.*",
   "firebase_admin.*",
   "google.api_core.*",

requirements.txt

@@ -12,7 +12,6 @@ fastapi[all]==0.82.0
 fastapi-caching[redis]==0.3.0
 fastapi-pagination[sqlalchemy]~= 0.10.0
 fideslang==1.3.1
-fideslib==3.1.5
 fideslog==1.2.10
 firebase-admin==5.3.0
 GitPython==3.1.27

scripts/create_test_data.py

@@ -4,10 +4,6 @@
 from datetime import datetime, timedelta
 from uuid import uuid4
 
-from fideslib.db.session import get_db_session
-from fideslib.models.audit_log import AuditLog, AuditLogAction
-from fideslib.models.client import ClientDetail
-from fideslib.models.fides_user import FidesUser
 from sqlalchemy import orm
 
 from fides.api.ctl.database.database import init_db
@@ -33,6 +29,10 @@
 )
 from fides.api.ops.util.data_category import DataCategory
 from fides.ctl.core.config import get_config
+from fides.lib.db.session import get_db_session
+from fides.lib.models.audit_log import AuditLog, AuditLogAction
+from fides.lib.models.client import ClientDetail
+from fides.lib.models.fides_user import FidesUser
 
 CONFIG = get_config()
 

src/fides/api/ctl/database/crud.py

@@ -4,7 +4,6 @@
 """
 from typing import Dict, List, Tuple
 
-from fideslib.db.base import Base
 from loguru import logger as log
 from sqlalchemy import column
 from sqlalchemy import delete as _delete
@@ -16,6 +15,7 @@
 
 from fides.api.ctl.database.session import async_session
 from fides.api.ctl.utils import errors
+from fides.lib.db.base import Base  # type: ignore[attr-defined]
 
 
 # CRUD Functions

src/fides/api/ctl/database/database.py

@@ -6,14 +6,14 @@
 from alembic import command, script
 from alembic.config import Config
 from alembic.runtime import migration
-from fideslib.db.base import Base
 from loguru import logger as log
 from sqlalchemy.orm import Session
 from sqlalchemy_utils.functions import create_database, database_exists
 
 from fides.api.ctl.utils.errors import get_full_exception_name
 from fides.ctl.core.config import get_config
 from fides.ctl.core.utils import get_db_engine
+from fides.lib.db.base import Base  # type: ignore[attr-defined]
 
 from .seed import load_default_resources
 

src/fides/api/ctl/database/seed.py

@@ -4,11 +4,6 @@
 from typing import List
 
 from fideslang import DEFAULT_TAXONOMY
-from fideslib.exceptions import KeyOrNameAlreadyExists
-from fideslib.models.client import ClientDetail
-from fideslib.models.fides_user import FidesUser
-from fideslib.models.fides_user_permissions import FidesUserPermissions
-from fideslib.utils.text import to_snake_case
 from loguru import logger as log
 
 from fides.api.ctl.database.session import sync_session
@@ -28,6 +23,11 @@
     StorageType,
 )
 from fides.ctl.core.config import get_config
+from fides.lib.exceptions import KeyOrNameAlreadyExists
+from fides.lib.models.client import ClientDetail
+from fides.lib.models.fides_user import FidesUser
+from fides.lib.models.fides_user_permissions import FidesUserPermissions
+from fides.lib.utils.text import to_snake_case
 
 from .crud import create_resource, list_resource
 

src/fides/api/ctl/migrations/versions/1f61c765cd1c_merge_alembic_heads.py

@@ -5,13 +5,12 @@
 Create Date: 2022-12-02 17:59:08.490577
 
 """
-from alembic import op
 import sqlalchemy as sa
-
+from alembic import op
 
 # revision identifiers, used by Alembic.
-revision = '1f61c765cd1c'
-down_revision = ('8f84fad4e00b', 'b72541d79f10')
+revision = "1f61c765cd1c"
+down_revision = ("8f84fad4e00b", "b72541d79f10")
 branch_labels = None
 depends_on = None
 

src/fides/api/ctl/migrations/versions/58933b5cc6e8_merge_failed_dsr_and_twilio.py

@@ -5,9 +5,8 @@
 Create Date: 2022-11-14 21:26:49.027809
 
 """
-from alembic import op
 import sqlalchemy as sa
-
+from alembic import op
 
 # revision identifiers, used by Alembic.
 revision = "58933b5cc6e8"

src/fides/api/ctl/migrations/versions/7abe778b7082_update_fideslang_data_categories.py

@@ -9,11 +9,11 @@
 """
 import logging
 
-from fideslib.db.session import get_db_session
 from sqlalchemy.exc import ProgrammingError
 
 from fides.api.ops.db.base import DatasetConfig
 from fides.ctl.core.config import get_config
+from fides.lib.db.session import get_db_session
 
 CONFIG = get_config()
 

src/fides/api/ctl/migrations/versions/8f84fad4e00b_add_error_message_tracking.py

@@ -5,9 +5,8 @@
 Create Date: 2022-11-15 01:38:28.531640
 
 """
-from alembic import op
 import sqlalchemy as sa
-
+from alembic import op
 
 # revision identifiers, used by Alembic.
 revision = "8f84fad4e00b"

src/fides/api/ctl/routes/util.py

@@ -1,14 +1,14 @@
 from typing import Dict, List
 
 from fideslang import FidesModelType
-from fideslib.db.base import Base
 
 from fides.api.ctl.database.crud import get_resource, list_resource
 from fides.api.ctl.sql_models import (  # type: ignore[attr-defined]
     models_with_default_field,
 )
 from fides.api.ctl.utils import errors
 from fides.api.ctl.utils.api_router import APIRouter
+from fides.lib.db.base import Base  # type: ignore[attr-defined]
 
 API_PREFIX = "/api/v1"
 

src/fides/api/ctl/sql_models.py

@@ -6,8 +6,6 @@
 
 from typing import Dict
 
-from fideslib.db.base import Base, ClientDetail, FidesUser, FidesUserPermissions
-from fideslib.db.base_class import FidesBase as FideslibBase
 from sqlalchemy import (
     ARRAY,
     BOOLEAN,
@@ -25,6 +23,13 @@
 from sqlalchemy.sql.sqltypes import DateTime
 
 from fides.ctl.core.config import FidesConfig, get_config
+from fides.lib.db.base import (  # type: ignore[attr-defined]
+    Base,
+    ClientDetail,
+    FidesUser,
+    FidesUserPermissions,
+)
+from fides.lib.db.base_class import FidesBase as FideslibBase
 
 CONFIG: FidesConfig = get_config()
 

src/fides/api/main.py

@@ -9,10 +9,6 @@
 
 from fastapi import FastAPI, HTTPException, Request, Response, status
 from fastapi.responses import FileResponse
-from fideslib.oauth.api.deps import get_config as lib_get_config
-from fideslib.oauth.api.deps import get_db as lib_get_db
-from fideslib.oauth.api.deps import verify_oauth_client as lib_verify_oauth_client
-from fideslib.oauth.api.routes.user_endpoints import router as user_router
 from fideslog.sdk.python.event import AnalyticsEvent
 from loguru import logger as log
 from redis.exceptions import RedisError, ResponseError
@@ -44,7 +40,6 @@
     send_analytics_event,
 )
 from fides.api.ops.api.deps import get_api_session
-from fides.api.ops.api.deps import get_db as get_ctl_db
 from fides.api.ops.api.v1.api import api_router
 from fides.api.ops.api.v1.exception_handlers import ExceptionHandlers
 from fides.api.ops.common_exceptions import (
@@ -60,12 +55,11 @@
 from fides.api.ops.tasks.scheduled.scheduler import scheduler
 from fides.api.ops.util.cache import get_cache
 from fides.api.ops.util.logger import get_fides_log_record_factory
-from fides.api.ops.util.oauth_util import verify_oauth_client
-from fides.ctl.core.config import FidesConfig
-from fides.ctl.core.config import get_config as get_ctl_config
-from fides.ctl.core.config.utils import check_required_webserver_config_values
+from fides.ctl.core.config import FidesConfig, get_config
+from fides.ctl.core.config.helpers import check_required_webserver_config_values
+from fides.lib.oauth.api.routes.user_endpoints import router as user_router
 
-CONFIG: FidesConfig = get_ctl_config()
+CONFIG: FidesConfig = get_config()
 
 logging.basicConfig(level=CONFIG.logging.level)
 logging.setLogRecordFactory(get_fides_log_record_factory())
@@ -187,9 +181,6 @@ def configure_routes() -> None:
 
 # Configure the routes here so we can generate the openapi json file
 configure_routes()
-app.dependency_overrides[lib_get_config] = get_ctl_config
-app.dependency_overrides[lib_get_db] = get_ctl_db
-app.dependency_overrides[lib_verify_oauth_client] = verify_oauth_client
 
 for handler in ExceptionHandlers.get_handlers():
     app.add_exception_handler(FunctionalityNotConfigured, handler)

src/fides/api/ops/api/deps.py

@@ -1,12 +1,12 @@
 from typing import Generator
 
-from fideslib.db.session import get_db_engine, get_db_session
 from sqlalchemy.orm import Session
 
 from fides.api.ops.common_exceptions import FunctionalityNotConfigured
 from fides.api.ops.util.cache import get_cache as get_redis_connection
 from fides.ctl.core.config import FidesConfig
 from fides.ctl.core.config import get_config as get_app_config
+from fides.lib.db.session import get_db_engine, get_db_session
 
 _engine = None
 CONFIG = get_app_config()

src/fides/api/ops/api/v1/endpoints/connection_endpoints.py

@@ -8,7 +8,6 @@
 from fastapi_pagination import Page, Params
 from fastapi_pagination.bases import AbstractPage
 from fastapi_pagination.ext.sqlalchemy import paginate
-from fideslib.exceptions import KeyOrNameAlreadyExists
 from pydantic import ValidationError, conlist
 from sqlalchemy import or_
 from sqlalchemy.orm import Session
@@ -73,6 +72,7 @@
 from fides.api.ops.util.api_router import APIRouter
 from fides.api.ops.util.logger import Pii
 from fides.api.ops.util.oauth_util import verify_oauth_client
+from fides.lib.exceptions import KeyOrNameAlreadyExists
 
 router = APIRouter(tags=["Connections"], prefix=V1_URL_PREFIX)
 

src/fides/api/ops/api/v1/endpoints/drp_endpoints.py

@@ -82,7 +82,7 @@ async def create_drp_privacy_request(
     policy: Optional[Policy] = Policy.get_by(
         db=db,
         field="drp_action",
-        value=data.exercise[0],
+        value=data.exercise[0],  # type: ignore[arg-type]
     )
 
     if not policy: