diff --git a/CHANGELOG.md b/CHANGELOG.md index 09bc5e1c8f0..931ec0b8b70 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,6 @@ The types of changes are: - `Fixed` for any bug fixes. - `Security` in case of vulnerabilities. - ## [Unreleased](https://github.com/ethyca/fides/compare/2.15.0...main) ### Added @@ -24,6 +23,10 @@ The types of changes are: - Support for acknowledge button for notice-only Privacy Notices and to disable toggling them off [#3546](https://github.com/ethyca/fides/pull/3546) - HTML format for privacy request storage destinations [#3427](https://github.com/ethyca/fides/pull/3427) +### Changed + +- Removed `pyodbc` in favor of `pymssql` for handling SQL Server connections [#3435](https://github.com/ethyca/fides/pull/3435) + ### Fixed - Fix race condition with consent modal link rendering [#3521](https://github.com/ethyca/fides/pull/3521) @@ -36,7 +39,6 @@ The types of changes are: - Optimize GitHub workflows used for docker image publishing [#3526](https://github.com/ethyca/fides/pull/3526) - ## [2.15.0](https://github.com/ethyca/fides/compare/2.14.1...2.15.0) ### Added diff --git a/Dockerfile b/Dockerfile index 09208864f2e..e880aa91df1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,9 @@ # If you update this, also update `DEFAULT_PYTHON_VERSION` in the GitHub workflow files ARG PYTHON_VERSION="3.10.11" - ######################### ## Compile Python Deps ## ######################### FROM python:${PYTHON_VERSION}-slim-bullseye as compile_image -ARG TARGETPLATFORM # Install auxiliary software RUN apt-get update && \ @@ -17,6 +15,26 @@ RUN apt-get update && \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* + +# Install FreeTDS (used for PyMSSQL) +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + libssl-dev \ + libffi-dev \ + libxslt-dev \ + libkrb5-dev \ + unixodbc \ + unixodbc-dev \ + freetds-dev \ + freetds-bin \ + python-dev \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +# Install Python Dependencies +COPY dev-requirements.txt . +RUN pip install --user -U pip --no-cache-dir install -r dev-requirements.txt + # Activate a Python venv RUN python3 -m venv /opt/fides ENV PATH="/opt/fides/bin:${PATH}" @@ -24,9 +42,6 @@ ENV PATH="/opt/fides/bin:${PATH}" # Install Python Dependencies RUN pip --no-cache-dir --disable-pip-version-check install --upgrade pip setuptools wheel -COPY dangerous-requirements.txt . -RUN if [ $TARGETPLATFORM != linux/arm64 ] ; then pip install --no-cache-dir install -r dangerous-requirements.txt ; fi - COPY requirements.txt . RUN pip install --no-cache-dir install -r requirements.txt @@ -37,38 +52,20 @@ RUN pip install --no-cache-dir install -r dev-requirements.txt ## Backend Base ## ################## FROM python:${PYTHON_VERSION}-slim-bullseye as backend -ARG TARGETPLATFORM -# Loads compiled requirements and adds the to the path -COPY --from=compile_image /opt/fides /opt/fides -ENV PATH=/opt/fides/bin:$PATH - -# These are all required for MSSQL -RUN : \ - && apt-get update \ - && apt-get install \ - -y --no-install-recommends \ - apt-transport-https \ +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ curl \ git \ - gnupg \ - unixodbc-dev \ + freetds-dev \ + freetds-bin \ + python-dev \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* -# SQL Server (MS SQL) -# https://docs.microsoft.com/en-us/sql/connect/odbc/linux-mac/installing-the-microsoft-odbc-driver-for-sql-server?view=sql-server-ver15 -RUN curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - -RUN curl https://packages.microsoft.com/config/debian/11/prod.list | tee /etc/apt/sources.list.d/msprod.list -ENV ACCEPT_EULA=y DEBIAN_FRONTEND=noninteractive -RUN if [ "$TARGETPLATFORM" != "linux/arm64" ] ; \ - then apt-get update \ - && apt-get install \ - -y --no-install-recommends \ - mssql-tools \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* ; \ - fi +# Loads compiled requirements and adds the to the path +COPY --from=compile_image /opt/fides /opt/fides +ENV PATH=/opt/fides/bin:$PATH # General Application Setup ## COPY . /fides diff --git a/dangerous-requirements.txt b/dangerous-requirements.txt deleted file mode 100644 index f4fd4d1218d..00000000000 --- a/dangerous-requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -# This file exists for caching optional dependencies within the Docker build -# These requirements are dangerous on certain platforms - -# MSSQL -pyodbc==4.0.34 diff --git a/docs/fides/docs/development/overview.md b/docs/fides/docs/development/overview.md index 4740c1396d4..e61b0d47106 100644 --- a/docs/fides/docs/development/overview.md +++ b/docs/fides/docs/development/overview.md @@ -30,7 +30,6 @@ The primary requirements for contributing to Fides are `Docker` and `Python`. Th * CPUs: 4 * Memory:8GB * Disk Limit: 200GB - 2. There are known issues around connecting to MSSQL for Apple M1 users. M1 users that wish to install `pyodbc` locally, please reference the workaround [here](https://github.com/mkleehammer/pyodbc/issues/846). Now that those are installed, the final step is to install the Python dev requirements for the Fides project. We recommend doing this in a virtual environment or using [pipx](https://pypa.github.io/pipx/), but specific details are outside the scope of this guide. diff --git a/noxfiles/docker_nox.py b/noxfiles/docker_nox.py index b389c84e050..f516fbe9ccf 100644 --- a/noxfiles/docker_nox.py +++ b/noxfiles/docker_nox.py @@ -1,5 +1,4 @@ """Contains the nox sessions for docker-related tasks.""" -import platform from multiprocessing import Pool from subprocess import run from typing import Callable, Dict, List, Tuple @@ -19,11 +18,6 @@ ) from git_nox import get_current_tag, recognized_tag -DOCKER_PLATFORM_MAP = { - "amd64": "linux/amd64", - "arm64": "linux/arm64", - "x86_64": "linux/amd64", -} DOCKER_PLATFORMS = "linux/amd64,linux/arm64" @@ -83,18 +77,6 @@ def get_current_image() -> str: return f"{IMAGE}:{get_current_tag()}" -def get_platform(posargs: List[str]) -> str: - """ - Calculate the CPU platform or get it from the - positional arguments. - """ - if "amd64" in posargs: - return DOCKER_PLATFORM_MAP["amd64"] - if "arm64" in posargs: - return DOCKER_PLATFORM_MAP["arm64"] - return DOCKER_PLATFORM_MAP[platform.machine().lower()] - - @nox.session() @nox.parametrize( "image", @@ -117,7 +99,6 @@ def build(session: nox.Session, image: str, machine_type: str = "") -> None: prod = Build the fides webserver/CLI and tag it as the current application version. test = Build the fides webserver/CLI the same as `prod`, but tag it as `local`. """ - build_platform = get_platform(session.posargs) # This check needs to be here so it has access to the session to throw an error if image == "prod": @@ -153,8 +134,6 @@ def build(session: nox.Session, image: str, machine_type: str = "") -> None: "docker", "build", "--target=prod_pc", - "--platform", - build_platform, "--tag", privacy_center_image_tag, ".", @@ -166,8 +145,6 @@ def build(session: nox.Session, image: str, machine_type: str = "") -> None: "docker", "build", "clients/sample-app", - "--platform", - build_platform, "--tag", sample_app_image_tag, external=True, @@ -180,8 +157,6 @@ def build(session: nox.Session, image: str, machine_type: str = "") -> None: "docker", "build", f"--target={target}", - "--platform", - build_platform, "--tag", tag(), ".", diff --git a/requirements.txt b/requirements.txt index a090a27ea28..4bf337d1a0e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -35,6 +35,7 @@ pydantic<1.10.2 pydash==6.0.2 PyJWT==2.4.0 pymongo==3.13.0 +pymssql==2.2.7 PyMySQL==1.0.2 python-jose[cryptography]==3.3.0 pyyaml>=5,<6 diff --git a/scripts/mssql_discover.py b/scripts/mssql_discover.py index e1f773e5d2d..6c5cdd5ebad 100644 --- a/scripts/mssql_discover.py +++ b/scripts/mssql_discover.py @@ -4,7 +4,7 @@ import sqlalchemy -MASTER_MSSQL_URL = f"mssql+pyodbc://{USER}:{PASS}@{IP}:{PORT}/{DB}?driver=ODBC+Driver+17+for+SQL+Server" +MASTER_MSSQL_URL = f"mssql+pymssql://{USER}:{PASS}@{IP}:{PORT}/{DB}" SUPPORTED_DATA_TYPES = set( diff --git a/setup.py b/setup.py index e5c813503af..ff48fcd556e 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,4 @@ import pathlib -from typing import List from setuptools import find_packages, setup @@ -14,45 +13,6 @@ install_requires = open("requirements.txt", encoding="utf-8").read().strip().split("\n") dev_requires = open("dev-requirements.txt", encoding="utf-8").read().strip().split("\n") -dangerous_requires = ( - open("dangerous-requirements.txt", encoding="utf-8").read().strip().split("\n") -) - - -def optional_requirements( - dependency_names: List[str], requires: List[str] = dangerous_requires -) -> List[str]: - """ - Matches the provided dependency names to lines in `optional-requirements.txt`, - and returns the full dependency string for each one. - - Prevents the need to store version numbers in two places. - """ - - requirements: List[str] = [] - - for dependency in dependency_names: - for optional_dependency in requires: - if optional_dependency.startswith(dependency): - requirements.append(optional_dependency) - break - - if len(requirements) == len(dependency_names): - return requirements - - raise ModuleNotFoundError - - -# Human-Readable Extras -# Versions are read from corresponding lines in `optional-requirements.txt` -extras = { - "mssql": optional_requirements(["pyodbc"], dangerous_requires), -} -dangerous_extras = ["mssql"] # These extras break on certain platforms -extras["all"] = sum( - [value for key, value in extras.items() if key not in dangerous_extras], [] -) - ################### ## Package Setup ## @@ -75,7 +35,6 @@ def optional_requirements( license="Apache License 2.0", install_requires=install_requires, dev_requires=dev_requires, - extras_require=extras, classifiers=[ "License :: OSI Approved :: Apache Software License", "Programming Language :: Python :: 3 :: Only", diff --git a/src/fides/api/api/v1/endpoints/connection_endpoints.py b/src/fides/api/api/v1/endpoints/connection_endpoints.py index 33ff20f75ef..25df07822f3 100644 --- a/src/fides/api/api/v1/endpoints/connection_endpoints.py +++ b/src/fides/api/api/v1/endpoints/connection_endpoints.py @@ -29,7 +29,6 @@ V1_URL_PREFIX, ) from fides.api.common_exceptions import ClientUnsuccessfulException, ConnectionException -from fides.api.ctl.sql_models import Dataset as CtlDataset # type: ignore[attr-defined] from fides.api.models.connectionconfig import ( ConnectionConfig, ConnectionTestStatus, @@ -49,6 +48,7 @@ TestStatusMessage, ) from fides.api.service.connectors import get_connector +from fides.api.models.sql_models import Dataset as CtlDataset # type: ignore[attr-defined] from fides.api.util.api_router import APIRouter from fides.api.util.connection_util import ( patch_connection_configs, diff --git a/src/fides/api/api/v1/endpoints/dataset_endpoints.py b/src/fides/api/api/v1/endpoints/dataset_endpoints.py index 6421e8cd7cd..8355c676a76 100644 --- a/src/fides/api/api/v1/endpoints/dataset_endpoints.py +++ b/src/fides/api/api/v1/endpoints/dataset_endpoints.py @@ -52,7 +52,9 @@ convert_dataset_to_graph, to_graph_field, ) -from fides.api.models.sql_models import Dataset as CtlDataset # type: ignore[attr-defined] +from fides.api.models.sql_models import ( # type: ignore[attr-defined] + Dataset as CtlDataset, +) from fides.api.oauth.utils import verify_oauth_client from fides.api.schemas.api import BulkUpdateFailed from fides.api.schemas.dataset import ( diff --git a/src/fides/api/api/v1/endpoints/utils.py b/src/fides/api/api/v1/endpoints/utils.py index 62e7037ad5c..ae5689e0872 100644 --- a/src/fides/api/api/v1/endpoints/utils.py +++ b/src/fides/api/api/v1/endpoints/utils.py @@ -24,7 +24,9 @@ ) from fides.api.db.base import Base # type: ignore[attr-defined] from fides.api.db.crud import get_resource, list_resource -from fides.api.models.sql_models import models_with_default_field # type: ignore[attr-defined] +from fides.api.models.sql_models import ( # type: ignore[attr-defined] + models_with_default_field, +) from fides.api.util import errors from fides.core.config import CONFIG diff --git a/src/fides/api/ctl/migrations/versions/c1885270b3cc_add_html_format_for_storageconfig.py b/src/fides/api/migrations/versions/c1885270b3cc_add_html_format_for_storageconfig.py similarity index 100% rename from src/fides/api/ctl/migrations/versions/c1885270b3cc_add_html_format_for_storageconfig.py rename to src/fides/api/migrations/versions/c1885270b3cc_add_html_format_for_storageconfig.py diff --git a/src/fides/api/models/datasetconfig.py b/src/fides/api/models/datasetconfig.py index 3d0289b5752..286cebd70ed 100644 --- a/src/fides/api/models/datasetconfig.py +++ b/src/fides/api/models/datasetconfig.py @@ -19,7 +19,9 @@ ) from fides.api.graph.data_type import parse_data_type_string from fides.api.models.connectionconfig import ConnectionConfig, ConnectionType -from fides.api.models.sql_models import Dataset as CtlDataset # type: ignore[attr-defined] +from fides.api.models.sql_models import ( # type: ignore[attr-defined] + Dataset as CtlDataset, +) from fides.api.util.saas_util import merge_datasets diff --git a/src/fides/api/schemas/connection_configuration/connection_secrets_mssql.py b/src/fides/api/schemas/connection_configuration/connection_secrets_mssql.py index fd5da346aaa..90e7ac8f25e 100644 --- a/src/fides/api/schemas/connection_configuration/connection_secrets_mssql.py +++ b/src/fides/api/schemas/connection_configuration/connection_secrets_mssql.py @@ -12,7 +12,7 @@ class MicrosoftSQLServerSchema(ConnectionConfigSecretsSchema): """Schema to validate the secrets needed to connect to a MS SQL Database connection string takes the format: - mssql+pyodbc://[username]:[password]@[host]:[port]/[dbname]?driver=ODBC+Driver+17+for+SQL+Server + mssql+pymssql://[username]:[password]@[host]:[port]/[dbname] """ diff --git a/src/fides/api/schemas/privacy_request.py b/src/fides/api/schemas/privacy_request.py index b3ecd8d028c..d9fe732b95b 100644 --- a/src/fides/api/schemas/privacy_request.py +++ b/src/fides/api/schemas/privacy_request.py @@ -14,7 +14,8 @@ ) from fides.api.schemas.api import BulkResponse, BulkUpdateFailed from fides.api.schemas.base_class import FidesSchema -from fides.api.schemas.policy import ActionType, PolicyResponse as PolicySchema +from fides.api.schemas.policy import ActionType +from fides.api.schemas.policy import PolicyResponse as PolicySchema from fides.api.schemas.redis_cache import Identity from fides.api.schemas.user import PrivacyRequestReviewer from fides.api.util.encryption.aes_gcm_encryption_scheme import verify_encryption_key diff --git a/src/fides/api/service/connectors/sql_connector.py b/src/fides/api/service/connectors/sql_connector.py index 51c9635b2c1..7208ef3326f 100644 --- a/src/fides/api/service/connectors/sql_connector.py +++ b/src/fides/api/service/connectors/sql_connector.py @@ -497,20 +497,19 @@ class MicrosoftSQLServerConnector(SQLConnector): def build_uri(self) -> URL: """ Build URI of format - mssql+pyodbc://[username]:[password]@[host]:[port]/[dbname]?driver=ODBC+Driver+17+for+SQL+Server + mssql+pymssql://[username]:[password]@[host]:[port]/[dbname] Returns URL obj, since SQLAlchemy's create_engine method accepts either a URL obj or a string """ config = self.secrets_schema(**self.configuration.secrets or {}) url = URL.create( - "mssql+pyodbc", + "mssql+pymssql", username=config.username, password=config.password, host=config.host, port=config.port, database=config.dbname, - query={"driver": "ODBC Driver 17 for SQL Server"}, ) return url diff --git a/src/fides/api/util/data_category.py b/src/fides/api/util/data_category.py index ae2e8d57577..ccb0e606702 100644 --- a/src/fides/api/util/data_category.py +++ b/src/fides/api/util/data_category.py @@ -6,7 +6,9 @@ from sqlalchemy.orm import Session from fides.api import common_exceptions -from fides.api.models.sql_models import DataCategory as DataCategoryDbModel # type: ignore[attr-defined] +from fides.api.models.sql_models import ( # type: ignore[attr-defined] + DataCategory as DataCategoryDbModel, +) def generate_fides_data_categories() -> Type[EnumType]: diff --git a/src/fides/core/utils.py b/src/fides/core/utils.py index 66f6875d5cb..7899691bea5 100644 --- a/src/fides/core/utils.py +++ b/src/fides/core/utils.py @@ -39,10 +39,11 @@ def get_db_engine(connection_string: str) -> Engine: """ Use SQLAlchemy to create a DB engine. """ + + # Pymssql doesn't support this arg + connect_args = {"connect_timeout": 10} if "pymssql" not in connection_string else {} try: - engine = sqlalchemy.create_engine( - connection_string, connect_args={"connect_timeout": 10} - ) + engine = sqlalchemy.create_engine(connection_string, connect_args=connect_args) except Exception as err: raise Exception("Failed to create engine!") from err diff --git a/tests/ctl/core/test_dataset.py b/tests/ctl/core/test_dataset.py index f8981c5701c..01006f79dd4 100644 --- a/tests/ctl/core/test_dataset.py +++ b/tests/ctl/core/test_dataset.py @@ -4,6 +4,7 @@ from urllib.parse import quote_plus from uuid import uuid4 +import pymssql import pytest import sqlalchemy from fideslang.manifests import write_manifest @@ -365,9 +366,9 @@ def test_unsupported_dialect_error() -> None: MYSQL_URL = "mysql+pymysql://mysql_user:mysql_pw@mysql-test:3306/mysql_example" -MSSQL_URL_TEMPLATE = "mssql+pyodbc://sa:SQLserver1@sqlserver-test:1433/{}?driver=ODBC+Driver+17+for+SQL+Server" +MSSQL_URL_TEMPLATE = "mssql+pymssql://sa:SQLserver1@sqlserver-test:1433/{}" MSSQL_URL = MSSQL_URL_TEMPLATE.format("sqlserver_example") -MASTER_MSSQL_URL = MSSQL_URL_TEMPLATE.format("master") + "&autocommit=True" +MASTER_MSSQL_URL = MSSQL_URL_TEMPLATE.format("master") # External databases require credentials passed through environment variables SNOWFLAKE_URL_TEMPLATE = "snowflake://FIDESCTL:{}@ZOA73785/FIDESCTL_TEST" @@ -409,6 +410,9 @@ def test_unsupported_dialect_error() -> None: "mssql": { "url": MSSQL_URL, "setup_url": MASTER_MSSQL_URL, + "server": "sqlserver-test:1433", + "username": "sa", + "password": "SQLserver1", "init_script_path": "tests/ctl/data/example_sql/sqlserver_example.sql", "is_external": False, "expected_collection": { @@ -461,9 +465,26 @@ def database_setup(self) -> Generator: queries = [ query for query in query_file.read().splitlines() if query != "" ] - print(queries) - for query in queries: - engine.execute(sqlalchemy.sql.text(query)) + + try: + if "pymssql" not in database_parameters.get("setup_url", ""): + for query in queries: + engine.execute(sqlalchemy.sql.text(query)) + else: + # This special MSSQL case is required due to how autocommit is activated + with pymssql.connect( + database_parameters["server"], + database_parameters["username"], + database_parameters["password"], + autocommit=True, + ) as connection: + for query in queries: + with connection.cursor() as cursor: + cursor.execute(query) + except: + print(f"> FAILED DB SETUP: {database_parameters.get('setup_url')}") + # We don't want to error all tests if a single setup fails + pass yield def test_get_db_tables(self, request: Dict, database_type: str) -> None: diff --git a/tests/ops/api/v1/endpoints/test_connection_config_endpoints.py b/tests/ops/api/v1/endpoints/test_connection_config_endpoints.py index 565dbb89d62..ae7a18978a9 100644 --- a/tests/ops/api/v1/endpoints/test_connection_config_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_connection_config_endpoints.py @@ -16,7 +16,6 @@ CONNECTION_READ, STORAGE_DELETE, ) -from fides.api.ctl.sql_models import Dataset from fides.api.api.v1.urn_registry import CONNECTIONS, SAAS_CONFIG, V1_URL_PREFIX from fides.api.models.client import ClientDetail from fides.api.models.connectionconfig import ( @@ -28,6 +27,7 @@ from fides.api.models.manual_webhook import AccessManualWebhook from fides.api.models.privacy_request import PrivacyRequestStatus from fides.api.oauth.roles import APPROVER, OWNER, VIEWER +from fides.api.models.sql_models import Dataset from tests.fixtures.application_fixtures import integration_secrets from tests.fixtures.saas.connection_template_fixtures import instantiate_connector diff --git a/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py b/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py index 735a32cc3e8..86e8e569ef6 100644 --- a/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py @@ -827,7 +827,7 @@ def test_get_connection_secret_schema_mssql( ) assert resp.json() == { "title": "MicrosoftSQLServerSchema", - "description": "Schema to validate the secrets needed to connect to a MS SQL Database\n\nconnection string takes the format:\nmssql+pyodbc://[username]:[password]@[host]:[port]/[dbname]?driver=ODBC+Driver+17+for+SQL+Server", + "description": "Schema to validate the secrets needed to connect to a MS SQL Database\n\nconnection string takes the format:\nmssql+pymssql://[username]:[password]@[host]:[port]/[dbname]", "type": "object", "properties": { "url": {"title": "URL", "sensitive": True, "type": "string"}, diff --git a/tests/ops/api/v1/endpoints/test_dataset_endpoints.py b/tests/ops/api/v1/endpoints/test_dataset_endpoints.py index 43dafbd9e43..3fee8de0e51 100644 --- a/tests/ops/api/v1/endpoints/test_dataset_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_dataset_endpoints.py @@ -15,20 +15,20 @@ from starlette.testclient import TestClient from fides.api.api.v1.scope_registry import ( + CTL_DATASET_READ, DATASET_CREATE_OR_UPDATE, DATASET_DELETE, DATASET_READ, - CTL_DATASET_READ, ) from fides.api.api.v1.urn_registry import ( + CONNECTION_DATASETS, DATASET_BY_KEY, DATASET_CONFIGS, DATASET_VALIDATE, DATASETCONFIG_BY_KEY, - CONNECTION_DATASETS, + DATASETS, V1_URL_PREFIX, YAML_DATASETS, - DATASETS, ) from fides.api.models.connectionconfig import ConnectionConfig from fides.api.models.datasetconfig import DatasetConfig diff --git a/tests/ops/integration_tests/setup_scripts/mssql_setup.py b/tests/ops/integration_tests/setup_scripts/mssql_setup.py index 5499634c8ff..97c4b7223e1 100644 --- a/tests/ops/integration_tests/setup_scripts/mssql_setup.py +++ b/tests/ops/integration_tests/setup_scripts/mssql_setup.py @@ -1,10 +1,16 @@ from time import sleep +import pymssql import sqlalchemy from sqlalchemy.exc import SQLAlchemyError -MSSQL_URL_TEMPLATE = "mssql+pyodbc://sa:Mssql_pw1@mssql_example:1433/{}?driver=ODBC+Driver+17+for+SQL+Server" -MASTER_MSSQL_URL = MSSQL_URL_TEMPLATE.format("master") + "&autocommit=True" +MSSQL_URL_TEMPLATE = "mssql+pymssql://sa:Mssql_pw1@mssql_example:1433/{}" +MASTER_MSSQL_URL = MSSQL_URL_TEMPLATE.format("master") + +SERVER = "mssql_example" +USER = "sa" +PASSWORD = "Mssql_pw1" +DATABASE = "master" def mssql_setup(): @@ -26,13 +32,21 @@ def mssql_setup(): break except SQLAlchemyError: try_number += 1 - print(f"Error connecting, retrying. Try number {try_number}") + print( + f"Error connecting with URL: {MASTER_MSSQL_URL}\nRetrying...Try number {try_number}" + ) sleep(1) with open("./docker/sample_data/mssql_example.sql", "r") as query_file: queries = [query for query in query_file.read().splitlines() if query != ""] - for query in queries: - engine.execute(sqlalchemy.sql.text(query)) + + # This must be done within a direct connection to enable autocommit + with pymssql.connect( + SERVER, USER, PASSWORD, DATABASE, autocommit=True + ) as connection: + for query in queries: + with connection.cursor() as cursor: + cursor.execute(query) if __name__ == "__main__": diff --git a/tests/ops/integration_tests/test_connection_configuration_integration.py b/tests/ops/integration_tests/test_connection_configuration_integration.py index ee46fb734ca..8fce28b5be7 100644 --- a/tests/ops/integration_tests/test_connection_configuration_integration.py +++ b/tests/ops/integration_tests/test_connection_configuration_integration.py @@ -829,7 +829,7 @@ def test_mssql_db_connection_incorrect_secrets( == f"Secrets updated for ConnectionConfig with key: {connection_config_mssql.key}." ) assert body["test_status"] == "failed" - assert "Connection error." == body["failure_reason"] + assert "Operational Error connecting to mssql db." == body["failure_reason"] db.refresh(connection_config_mssql) assert connection_config_mssql.secrets == { @@ -895,7 +895,7 @@ def test_mssql_db_connection_connect_with_url( connection_config_mssql, ) -> None: payload = { - "url": "mssql+pyodbc://sa:Mssql_pw1@mssql_example:1433/mssql_example?driver=ODBC+Driver+17+for+SQL+Server" + "url": "mssql+pymssql://sa:Mssql_pw1@mssql_example:1433/mssql_example" } auth_header = generate_auth_header(scopes=[CONNECTION_CREATE_OR_UPDATE]) @@ -989,7 +989,7 @@ def test_connection_configuration_test_failed_response( assert connection_config_mssql.last_test_timestamp is not None assert connection_config_mssql.last_test_succeeded is False assert body["test_status"] == "failed" - assert "Connection error." == body["failure_reason"] + assert "Operational Error connecting to mssql db." == body["failure_reason"] assert ( body["msg"] == f"Test completed for ConnectionConfig with key: {connection_config_mssql.key}." diff --git a/tests/ops/integration_tests/test_integration_mssql_example.py b/tests/ops/integration_tests/test_integration_mssql_example.py index 44e6b8945f1..3f0a5934b34 100644 --- a/tests/ops/integration_tests/test_integration_mssql_example.py +++ b/tests/ops/integration_tests/test_integration_mssql_example.py @@ -1,7 +1,7 @@ import pytest from sqlalchemy import func, select, table -MSSQL_URL_TEMPLATE = "mssql+pyodbc://sa:Mssql_pw1@mssql_example:1433/{}?driver=ODBC+Driver+17+for+SQL+Server" +MSSQL_URL_TEMPLATE = "mssql+pymssql://sa:Mssql_pw1@mssql_example:1433/{}" MSSQL_URL = MSSQL_URL_TEMPLATE.format("mssql_example") diff --git a/tests/ops/service/storage_uploader_service_test.py b/tests/ops/service/test_storage_uploader_service.py similarity index 100% rename from tests/ops/service/storage_uploader_service_test.py rename to tests/ops/service/test_storage_uploader_service.py