add cloudsql cicd workflow

retrieval_service/datastore/providers/postgres_test.py

@@ -26,7 +26,7 @@
 from .test_data import query_embedding1, query_embedding2, query_embedding3
 from .utils import get_env_var
 
-pytestmark = pytest.mark.asyncio(scope="module")
+pytestmark = [pytest.mark.asyncio(scope="module"), pytest.mark.postgres]
 
 
 @pytest.fixture(scope="module")

retrieval_service/pyproject.toml

@@ -8,3 +8,8 @@ warn_unused_configs = true
 [[tool.mypy.overrides]]
 module = ["pgvector.asyncpg"]
 ignore_missing_imports = true
+
+[tool.pytest.ini_options]
+markers = [
+    "postgres",
+]

retrieval_service/sql-proxy/Dockerfile

@@ -0,0 +1 @@
+FROM gcr.io/gcp-runtimes/ubuntu_20_0_4

retrieval_service/sql-proxy/cloudsql.tests.cloudbuild.yaml

@@ -0,0 +1,161 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+steps:
+  - id: "docker-build"
+    name: "gcr.io/cloud-builders/docker"
+    args: ["build", "-t", "${_IMAGE_NAME}", "retrieval_service/sql-proxy/."]
+
+  - id: "docker-push"
+    name: "gcr.io/cloud-builders/docker"
+    args: ["push", "${_IMAGE_NAME}"]
+
+  - id: "docker-layer"
+    name: "gcr.io/cloud-builders/docker"
+    entrypoint: /bin/bash
+    args:
+      - '-c'
+      - |
+        echo "FROM $_IMAGE_NAME
+        COPY --from=gcr.io/cloud-sql-connectors/cloud-sql-proxy /cloud-sql-proxy /cloudsql/cloud-sql-proxy" > Dockerfile-proxy;
+
+        docker build -f Dockerfile-proxy -t ${_IMAGE_NAME}-proxy .
+
+  - id: Install dependencies
+    name: python:3.11
+    dir: retrieval_service
+    entrypoint: pip
+    args:
+      [
+        "install",
+        "-r",
+        "requirements.txt",
+        "-r",
+        "requirements-test.txt",
+        "--user",
+      ]
+
+  - id: Create database
+    name: ${_IMAGE_NAME}-proxy
+    dir: retrieval_service/sql-proxy
+    entrypoint: /bin/bash
+    secretEnv:
+      - CLOUDSQLUSER # Use built-in env vars for database connection
+      - CLOUDSQLPASSWORD
+    args:
+      - "-c"
+      - |
+        /cloudsql/cloud-sql-proxy --port ${_DATABASE_PORT} ${_INSTANCE_CONNECTION_NAME} & sleep 2;
+        psql -h ${_DATABASE_HOST} -c "CREATE DATABASE ${_DATABASE_NAME};" || echo "Database '${_DATABASE_NAME}' already exists."
+        psql -h ${_DATABASE_HOST} -d ${_DATABASE_NAME} -c 'CREATE EXTENSION vector;' || echo "Extension 'vector' already exists."
+
+  - id: Initialize data
+    name: python:3.11
+    dir: retrieval_service
+    secretEnv:
+      - CLOUDSQLUSER
+      - CLOUDSQLPASSWORD
+    entrypoint: /bin/bash
+    args:
+      - "-c"
+      - |
+        # Create config
+        cp example-config.yml config.yml
+        sed -i "/127.0.0.1/d" config.yml
+        sed -i "s/postgres/cloudsql-postgres/g" config.yml
+        sed -i "s/my_database/${_DATABASE_NAME}/g" config.yml
+        sed -i "s/my-user/$$CLOUDSQLUSER/g" config.yml
+        sed -i "s/my-password/$$CLOUDSQLPASSWORD/g" config.yml
+        echo -e '\n  project: "'$PROJECT_ID'"' >> config.yml
+        echo '  region: "'${_CLOUDSQL_REGION}'"' >> config.yml
+        echo '  instance: "'${_CLOUDSQL_INSTANCE}'"' >> config.yml
+        # Run script
+        python run_database_init.py
+
+  - id: Run Alloy DB integration tests
+    name: python:3.11
+    dir: retrieval_service
+    env: # Set env var expected by tests
+      - "DB_NAME=${_DATABASE_NAME}"
+      - "DB_HOST=${_DATABASE_HOST}"
+      - "DB_PROJECT=$PROJECT_ID"
+      - "DB_REGION=${_CLOUDSQL_REGION}"
+      - "DB_INSTANCE=${_CLOUDSQL_INSTANCE}"
+    secretEnv:
+      - CLOUDSQLUSER
+      - CLOUDSQLPASSWORD
+    entrypoint: /bin/bash
+    args:
+      - "-c"
+      - |
+        # Set env var expected by tests
+        export DB_USER=$$CLOUDSQLUSER
+        export DB_PASS=$$CLOUDSQLPASSWORD
+        python -m pytest -m "not postgres"
+
+  - id: Run database export
+    name: python:3.11
+    dir: retrieval_service
+    entrypoint: /bin/bash
+    args:
+      - "-c"
+      - |
+        # Run script
+        python run_database_export.py
+        cd ../data
+        diff --strip-trailing-cr -Z airport_dataset.csv airport_dataset.csv.new || (echo "airport dataset export fail" && exit 1)
+        diff --strip-trailing-cr -Z amenity_dataset.csv amenity_dataset.csv.new || (echo "amenity dataset export fail" && exit 1)
+        diff --strip-trailing-cr -Z flights_dataset.csv flights_dataset.csv.new || (echo "flight dataset export fail" && exit 1)
+
+  - id: Clean exported files
+    name: python:3.11
+    dir: data
+    entrypoint: /bin/bash
+    args:
+      - "-c"
+      - |
+        rm airport_dataset.csv.new amenity_dataset.csv.new flights_dataset.csv.new
+
+
+  - id: Clean database
+    name: postgres
+    entrypoint: /bin/bash
+    secretEnv:
+      - CLOUDSQLUSER
+      - CLOUDSQLPASSWORD
+    args:
+      - "-c"
+      - |
+        psql -h ${_DATABASE_HOST} -c "DROP DATABASE IF EXISTS ${_DATABASE_NAME};"
+
+substitutions:
+  _DATABASE_NAME: test_${SHORT_SHA}
+  _DATABASE_USER: postgres
+  _DATABASE_HOST: 127.0.0.1
+  _CLOUDSQL_REGION: "us-central1"
+  _CLOUDSQL_INSTANCE: "my-cloudsql-instance"
+  _INSTANCE_CONNECTION_NAME: ${PROJECT_ID}:us-central1:myinstance
+  _DATABASE_PORT: "5432"
+  _IMAGE_NAME: gcr.io/${PROJECT_ID}/sample-sql-proxy
+
+availableSecrets:
+  secretManager:
+    - versionName: projects/$PROJECT_ID/secrets/cloudsql_pass/versions/latest
+      env: CLOUDSQLPASSWORD
+    - versionName: projects/$PROJECT_ID/secrets/cloudsql_user/versions/latest
+      env: CLOUDSQLUSER
+
+options:
+  substitutionOption: 'ALLOW_LOOSE'
+  dynamic_substitutions: true