Google Cloud SQL MySQL Connector

clients/admin-ui/cypress/fixtures/connectors/connection_types.json

@@ -30,6 +30,12 @@
       "human_readable": "MySQL",
       "encoded_icon": null
     },
+    {
+      "identifier": "google_cloud_mysql",
+      "type": "database",
+      "human_readable": "Google Cloud for MySQL",
+      "encoded_icon": null
+    },
     {
       "identifier": "postgres",
       "type": "database",

src/fides/api/alembic/migrations/versions/d7449292621d_add_google_cloud_sql_mysql_to_.py

@@ -0,0 +1,47 @@
+"""add google_cloud_mysql to connectiontype
+
+Revision ID: d7449292621d
+Revises: 4b2eade4353c
+Create Date: 2024-06-03 17:29:38.233347
+
+"""
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision = 'd7449292621d'
+down_revision = '4b2eade4353c'
+branch_labels = None
+depends_on = None
+
+
+
+def upgrade():
+    # Add google_cloud_mysql to ConnectionType enum
+    op.execute("alter type connectiontype rename to connectiontype_old")
+    op.execute(
+        "create type connectiontype as enum('postgres', 'mongodb', 'mysql', 'https', 'snowflake', 'redshift', 'mssql', 'mariadb', 'bigquery', 'saas', 'manual', 'email', 'manual_webhook', 'timescale', 'fides', 'sovrn', 'google_cloud_mysql')"
+    )
+    op.execute(
+        (
+            "alter table connectionconfig alter column connection_type type connectiontype using "
+            "connection_type::text::connectiontype"
+        )
+    )
+    op.execute("drop type connectiontype_old")
+
+
+
+def downgrade():
+    # Remove google_cloud_mysql from the ConnectionType enum
+    op.execute("alter type connectiontype rename to connectiontype_old")
+    op.execute(
+        "create type connectiontype as enum('postgres', 'mongodb', 'mysql', 'https', 'snowflake', 'redshift', 'mssql', 'mariadb', 'bigquery', 'saas', 'manual', 'email', 'manual_webhook', 'timescale', 'fides', 'sovrn')"
+    )
+    op.execute(
+        (
+            "alter table connectionconfig alter column connection_type type connectiontype using "
+            "connection_type::text::connectiontype"
+        )
+    )
+    op.execute("drop type connectiontype_old")

src/fides/api/models/connectionconfig.py

@@ -37,6 +37,7 @@ class ConnectionType(enum.Enum):
     postgres = "postgres"
     mongodb = "mongodb"
     mysql = "mysql"
+    google_cloud_mysql = "google_cloud_mysql"
     https = "https"
     saas = "saas"
     redshift = "redshift"
@@ -66,9 +67,10 @@ def human_readable(self) -> str:
             ConnectionType.fides.value: "Fides Connector",
             ConnectionType.generic_consent_email.value: "Generic Consent Email",
             ConnectionType.generic_erasure_email.value: "Generic Erasure Email",
+            ConnectionType.google_cloud_mysql.value: "Google Cloud for MySQL",
             ConnectionType.https.value: "Policy Webhook",
-            ConnectionType.manual.value: "Manual Connector",
             ConnectionType.manual_webhook.value: "Manual Process",
+            ConnectionType.manual.value: "Manual Connector",
             ConnectionType.mariadb.value: "MariaDB",
             ConnectionType.mongodb.value: "MongoDB",
             ConnectionType.mssql.value: "Microsoft SQL Server",

src/fides/api/schemas/connection_configuration/__init__.py

@@ -15,23 +15,15 @@
 )
 from fides.api.schemas.connection_configuration.connection_secrets_bigquery import (
     BigQueryDocsSchema as BigQueryDocsSchema,
-)
-from fides.api.schemas.connection_configuration.connection_secrets_bigquery import (
     BigQuerySchema as BigQuerySchema,
 )
 from fides.api.schemas.connection_configuration.connection_secrets_dynamodb import (
     DynamoDBDocsSchema as DynamoDBDocsSchema,
-)
-from fides.api.schemas.connection_configuration.connection_secrets_dynamodb import (
     DynamoDBSchema as DynamoDBSchema,
 )
 from fides.api.schemas.connection_configuration.connection_secrets_email import (
     EmailDocsSchema as EmailDocsSchema,
-)
-from fides.api.schemas.connection_configuration.connection_secrets_email import (
     EmailSchema as EmailSchema,
-)
-from fides.api.schemas.connection_configuration.connection_secrets_email import (
     ExtendedEmailSchema,
 )
 from fides.api.schemas.connection_configuration.connection_secrets_fides import (
@@ -40,56 +32,42 @@
 )
 from fides.api.schemas.connection_configuration.connection_secrets_manual_webhook import (
     ManualWebhookSchema as ManualWebhookSchema,
-)
-from fides.api.schemas.connection_configuration.connection_secrets_manual_webhook import (
     ManualWebhookSchemaforDocs as ManualWebhookSchemaforDocs,
 )
 from fides.api.schemas.connection_configuration.connection_secrets_mariadb import (
     MariaDBDocsSchema as MariaDBDocsSchema,
-)
-from fides.api.schemas.connection_configuration.connection_secrets_mariadb import (
     MariaDBSchema as MariaDBSchema,
 )
 from fides.api.schemas.connection_configuration.connection_secrets_mongodb import (
     MongoDBDocsSchema as MongoDBDocsSchema,
-)
-from fides.api.schemas.connection_configuration.connection_secrets_mongodb import (
     MongoDBSchema as MongoDBSchema,
 )
 from fides.api.schemas.connection_configuration.connection_secrets_mssql import (
     MicrosoftSQLServerSchema as MicrosoftSQLServerSchema,
-)
-from fides.api.schemas.connection_configuration.connection_secrets_mssql import (
     MSSQLDocsSchema as MSSQLDocsSchema,
 )
 from fides.api.schemas.connection_configuration.connection_secrets_mysql import (
     MySQLDocsSchema as MySQLDocsSchema,
-)
-from fides.api.schemas.connection_configuration.connection_secrets_mysql import (
     MySQLSchema as MySQLSchema,
 )
-from fides.api.schemas.connection_configuration.connection_secrets_postgres import (
-    PostgreSQLDocsSchema as PostgreSQLDocsSchema,
+from fides.api.schemas.connection_configuration.connection_secrets_google_cloud_mysql import (
+    GoogleCloudMySQLDocsSchema as GoogleCloudMySQLDocsSchema,
+    GoogleCloudMySQLSchema as GoogleCloudMySQLSchema,
 )
 from fides.api.schemas.connection_configuration.connection_secrets_postgres import (
+    PostgreSQLDocsSchema as PostgreSQLDocsSchema,
     PostgreSQLSchema as PostgreSQLSchema,
 )
 from fides.api.schemas.connection_configuration.connection_secrets_redshift import (
     RedshiftDocsSchema as RedshiftDocsSchema,
-)
-from fides.api.schemas.connection_configuration.connection_secrets_redshift import (
     RedshiftSchema as RedshiftSchema,
 )
 from fides.api.schemas.connection_configuration.connection_secrets_saas import (
     SaaSSchema as SaaSSchema,
-)
-from fides.api.schemas.connection_configuration.connection_secrets_saas import (
     SaaSSchemaFactory as SaaSSchemaFactory,
 )
 from fides.api.schemas.connection_configuration.connection_secrets_snowflake import (
     SnowflakeDocsSchema as SnowflakeDocsSchema,
-)
-from fides.api.schemas.connection_configuration.connection_secrets_snowflake import (
     SnowflakeSchema as SnowflakeSchema,
 )
 from fides.api.schemas.connection_configuration.connection_secrets_sovrn import (
@@ -114,6 +92,7 @@
     ConnectionType.fides.value: FidesConnectorSchema,
     ConnectionType.generic_consent_email.value: ExtendedEmailSchema,
     ConnectionType.generic_erasure_email.value: EmailSchema,
+    ConnectionType.google_cloud_mysql.value: GoogleCloudMySQLSchema,
     ConnectionType.https.value: HttpsSchema,
     ConnectionType.manual_webhook.value: ManualWebhookSchema,
     ConnectionType.mariadb.value: MariaDBSchema,
@@ -159,6 +138,7 @@ def get_connection_secrets_schema(
     MongoDBDocsSchema,
     PostgreSQLDocsSchema,
     MySQLDocsSchema,
+    GoogleCloudMySQLDocsSchema,
     RedshiftDocsSchema,
     SnowflakeDocsSchema,
     MSSQLDocsSchema,

src/fides/api/schemas/connection_configuration/connection_secrets_google_cloud_mysql.py

@@ -0,0 +1,48 @@
+from typing import List, Optional
+
+from pydantic import Field
+
+from fides.api.schemas.base_class import NoValidationSchema
+from fides.api.schemas.connection_configuration.connection_secrets import (
+    ConnectionConfigSecretsSchema,
+)
+
+
+class GoogleCloudMySQLSchema(ConnectionConfigSecretsSchema):
+    """Schema to validate the secrets needed to connect to a Google Cloud MySQL Database"""
+
+    host: str = Field(
+        title="Host",
+        description="The hostname or IP address of the server where the database is running.",
+    )
+    port: int = Field(
+        3306,
+        title="Port",
+        description="The network port number on which the server is listening for incoming connections (default: 3306).",
+    )
+    username: Optional[str] = Field(
+        None,
+        title="Username",
+        description="The user account used to authenticate and access the database.",
+    )
+    password: Optional[str] = Field(
+        None,
+        title="Password",
+        description="The password used to authenticate and access the database.",
+        sensitive=True,
+    )
+    dbname: str = Field(
+        description="The name of the specific database within the database server that you want to connect to.",
+        title="Database",
+    )
+    ssh_required: bool = Field(
+        False,
+        title="SSH required",
+        description="Indicates whether an SSH tunnel is required for the connection. Enable this option if your Google Cloud MySQL server is behind a firewall and requires SSH tunneling for remote connections.",
+    )
+
+    _required_components: List[str] = ["host", "dbname"]
+
+
+class GoogleCloudMySQLDocsSchema(GoogleCloudMySQLSchema, NoValidationSchema):
+    """Google Cloud MySQL Secrets Schema for API Docs"""

src/fides/api/service/connectors/__init__.py

@@ -33,21 +33,12 @@
 from fides.api.service.connectors.saas_connector import SaaSConnector as SaaSConnector
 from fides.api.service.connectors.sql_connector import (
     BigQueryConnector as BigQueryConnector,
-)
-from fides.api.service.connectors.sql_connector import (
+    GoogleCloudMySQLConnector as GoogleCloudMySQLConnector,
     MariaDBConnector as MariaDBConnector,
-)
-from fides.api.service.connectors.sql_connector import (
     MicrosoftSQLServerConnector as MicrosoftSQLServerConnector,
-)
-from fides.api.service.connectors.sql_connector import MySQLConnector as MySQLConnector
-from fides.api.service.connectors.sql_connector import (
+    MySQLConnector as MySQLConnector,
     PostgreSQLConnector as PostgreSQLConnector,
-)
-from fides.api.service.connectors.sql_connector import (
     RedshiftConnector as RedshiftConnector,
-)
-from fides.api.service.connectors.sql_connector import (
     SnowflakeConnector as SnowflakeConnector,
 )
 from fides.api.service.connectors.timescale_connector import (
@@ -61,6 +52,7 @@
     ConnectionType.fides.value: FidesConnector,
     ConnectionType.generic_consent_email.value: GenericConsentEmailConnector,
     ConnectionType.generic_erasure_email.value: GenericErasureEmailConnector,
+    ConnectionType.google_cloud_mysql.value: GoogleCloudMySQLConnector,
     ConnectionType.https.value: HTTPSConnector,
     ConnectionType.manual_webhook.value: ManualWebhookConnector,
     ConnectionType.mariadb.value: MariaDBConnector,

src/fides/api/service/connectors/sql_connector.py

@@ -577,3 +577,7 @@ def cursor_result_to_rows(results: LegacyCursorResult) -> List[Row]:
         Convert SQLAlchemy results to a list of dictionaries
         """
         return SQLConnector.default_cursor_result_to_rows(results)
+
+
+class GoogleCloudMySQLConnector(MySQLConnector):
+    """Connector specific to Google Cloud for MySQL"""

tests/fixtures/application_fixtures.py

@@ -137,6 +137,13 @@
         "username": pydash.get(integration_config, "mysql_example.user"),
         "password": pydash.get(integration_config, "mysql_example.password"),
     },
+    "google_cloud_mysql": {
+        "host": pydash.get(integration_config, "google_cloud_mysql.server"),
+        "port": pydash.get(integration_config, "google_cloud_mysql.port"),
+        "dbname": pydash.get(integration_config, "google_cloud_mysql.db"),
+        "username": pydash.get(integration_config, "google_cloud_mysql.user"),
+        "password": pydash.get(integration_config, "google_cloud_mysql.password"),
+    },
     "mssql_example": {
         "host": pydash.get(integration_config, "mssql_example.server"),
         "port": pydash.get(integration_config, "mssql_example.port"),

tests/fixtures/mysql_fixtures.py

@@ -1,3 +1,4 @@
+import os
 from typing import Dict, Generator, List
 from uuid import uuid4
 
@@ -23,6 +24,17 @@ def mysql_example_secrets():
     return integration_secrets["mysql_example"]
 
 
+@pytest.fixture(scope="function")
+def google_cloud_mysql_secrets():
+    secrets = integration_secrets.get("google_cloud_mysql", {})
+    secrets["host"] = secrets.get("host") or os.environ.get("GOOGLE_CLOUD_MYSQL_TEST_HOST")
+    secrets["dbname"] = secrets.get("dbname") or os.environ.get("GOOGLE_CLOUD_MYSQL_TEST_DBNAME")
+    secrets["username"] = secrets.get("username") or os.environ.get("GOOGLE_CLOUD_MYSQL_TEST_USERNAME")
+    secrets["password"] = secrets.get("password") or os.environ.get("GOOGLE_CLOUD_MYSQL_TEST_PASSWORD")
+    secrets["port"] = secrets.get("port") or os.environ.get("GOOGLE_CLOUD_MYSQL_TEST_PORT")
+    return secrets
+
+
 @pytest.fixture(scope="function")
 def dataset_config_mysql(
     connection_config: ConnectionConfig,
@@ -112,6 +124,22 @@ def connection_config_mysql(db: Session) -> Generator:
     connection_config.delete(db)
 
 
+@pytest.fixture(scope="function")
+def connection_config_google_cloud_mysql(db: Session, google_cloud_mysql_secrets) -> Generator:
+    connection_config = ConnectionConfig.create(
+        db=db,
+        data={
+            "name": str(uuid4()),
+            "key": "my_google_cloud_mysql_db_1",
+            "connection_type": ConnectionType.google_cloud_mysql,
+            "access": AccessLevel.write,
+            "secrets": google_cloud_mysql_secrets,
+        },
+    )
+    yield connection_config
+    connection_config.delete(db)
+
+
 @pytest.fixture(scope="function")
 def mysql_integration_session_cls(connection_config_mysql):
     example_postgres_uri = MySQLConnector(connection_config_mysql).build_uri()

tests/ops/integration_test_config.toml

@@ -20,6 +20,13 @@ password="mysql_pw"
 db="mysql_example"
 port=3306
 
+[google_cloud_mysql]
+server=""
+user=""
+password=""
+db=""
+port=""
+
 [redshift]
 host=""
 port=""