From 8bc8a637e61134c454a517a6f3b5abd1f3dd017f Mon Sep 17 00:00:00 2001 From: Andrew Heard Date: Fri, 7 Jun 2024 11:56:53 -0400 Subject: [PATCH 1/7] [Vertex AI] Add environment variable to control integration tests --- .github/workflows/vertexai.yml | 3 +- .../Tests/Integration/IntegrationTests.swift | 93 +++++++++---------- Package.swift | 5 +- 3 files changed, 48 insertions(+), 53 deletions(-) diff --git a/.github/workflows/vertexai.yml b/.github/workflows/vertexai.yml index 8429178df3a..f0af71d2d5b 100644 --- a/.github/workflows/vertexai.yml +++ b/.github/workflows/vertexai.yml @@ -43,13 +43,14 @@ jobs: spm-integration: strategy: matrix: - target: [macOS] + target: [iOS] os: [macos-14] include: - os: macos-14 xcode: Xcode_15.2 runs-on: ${{ matrix.os }} env: + VERTEXAI_RUN_INTEGRATION_TESTS: 1 FIREBASECI_USE_LATEST_GOOGLEAPPMEASUREMENT: 1 plist_secret: ${{ secrets.GHASecretsGPGPassphrase1 }} steps: diff --git a/FirebaseVertexAI/Tests/Integration/IntegrationTests.swift b/FirebaseVertexAI/Tests/Integration/IntegrationTests.swift index 7382369c0c0..d6c3702fd37 100644 --- a/FirebaseVertexAI/Tests/Integration/IntegrationTests.swift +++ b/FirebaseVertexAI/Tests/Integration/IntegrationTests.swift @@ -16,62 +16,59 @@ import FirebaseCore import FirebaseVertexAI import XCTest -// These tests are functional on other platforms but are compiled-out of non-macOS platforms to -// avoid re-running them as part of `swift-build-run` job (iOS-only) in the `spm` workflow on CI: -// https://github.com/firebase/firebase-ios-sdk/blob/0492e83cb22833ec548e61d854bb7b830e83b826/.github/workflows/spm.yml#L57 -// Since these requests are billed, we are running them more sparsely than the unit tests. -#if os(macOS) - - @available(iOS 15.0, macOS 12.0, macCatalyst 15.0, *) - final class IntegrationTests: XCTestCase { - // Set temperature, topP and topK to lowest allowed values to make responses more deterministic. - let generationConfig = GenerationConfig(temperature: 0.0, topP: 0.0, topK: 1) - - var vertex: VertexAI! - var model: GenerativeModel! - - override func setUp() async throws { - let plistPath = try XCTUnwrap(Bundle.module.path( - forResource: "GoogleService-Info", - ofType: "plist" - )) - let options = try XCTUnwrap(FirebaseOptions(contentsOfFile: plistPath)) - FirebaseApp.configure(options: options) - - vertex = VertexAI.vertexAI() - model = vertex.generativeModel( - modelName: "gemini-1.5-flash", - generationConfig: generationConfig - ) - } +@available(iOS 15.0, macOS 12.0, macCatalyst 15.0, *) +final class IntegrationTests: XCTestCase { + // Set temperature, topP and topK to lowest allowed values to make responses more deterministic. + let generationConfig = GenerationConfig(temperature: 0.0, topP: 0.0, topK: 1) + + var vertex: VertexAI! + var model: GenerativeModel! + + override func setUp() async throws { + try XCTSkipIf(ProcessInfo.processInfo.environment["VERTEXAI_RUN_INTEGRATION_TESTS"] == nil, """ + Vertex AI integration tests skipped; to enable them, set the VERTEXAI_RUN_INTEGRATION_TESTS \ + environment variable in Xcode or CI jobs. + """) + + let plistPath = try XCTUnwrap(Bundle.module.path( + forResource: "GoogleService-Info", + ofType: "plist" + )) + let options = try XCTUnwrap(FirebaseOptions(contentsOfFile: plistPath)) + FirebaseApp.configure(options: options) + + vertex = VertexAI.vertexAI() + model = vertex.generativeModel( + modelName: "gemini-1.5-flash", + generationConfig: generationConfig + ) + } - override func tearDown() async throws { - if let app = FirebaseApp.app() { - await app.delete() - } + override func tearDown() async throws { + if let app = FirebaseApp.app() { + await app.delete() } + } - // MARK: - Generate Content + // MARK: - Generate Content - func testGenerateContent() async throws { - let prompt = "Where is Google headquarters located? Answer with the city name only." + func testGenerateContent() async throws { + let prompt = "Where is Google headquarters located? Answer with the city name only." - let response = try await model.generateContent(prompt) + let response = try await model.generateContent(prompt) - let text = try XCTUnwrap(response.text).trimmingCharacters(in: .whitespacesAndNewlines) - XCTAssertEqual(text, "Mountain View") - } + let text = try XCTUnwrap(response.text).trimmingCharacters(in: .whitespacesAndNewlines) + XCTAssertEqual(text, "Mountain View") + } - // MARK: - Count Tokens + // MARK: - Count Tokens - func testCountTokens() async throws { - let prompt = "Why is the sky blue?" + func testCountTokens() async throws { + let prompt = "Why is the sky blue?" - let response = try await model.countTokens(prompt) + let response = try await model.countTokens(prompt) - XCTAssertEqual(response.totalTokens, 6) - XCTAssertEqual(response.totalBillableCharacters, 16) - } + XCTAssertEqual(response.totalTokens, 6) + XCTAssertEqual(response.totalBillableCharacters, 16) } - -#endif // os(macOS) +} diff --git a/Package.swift b/Package.swift index f104e41e747..e2ede8ca1db 100644 --- a/Package.swift +++ b/Package.swift @@ -1381,14 +1381,11 @@ let package = Package( resources: [ .process("CountTokenResponses"), .process("GenerateContentResponses"), - ], - cSettings: [ - .headerSearchPath("../../../"), ] ), .testTarget( name: "FirebaseVertexAIIntegration", - dependencies: ["FirebaseVertexAI", "SharedTestUtilities"], + dependencies: ["FirebaseVertexAI"], path: "FirebaseVertexAI/Tests/Integration", resources: [ .process("Resources"), From 8261562d48ab9b87ae515cf86c22ce5cce3edd66 Mon Sep 17 00:00:00 2001 From: Andrew Heard Date: Fri, 7 Jun 2024 12:34:13 -0400 Subject: [PATCH 2/7] Revert `cSettings` removal --- Package.swift | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Package.swift b/Package.swift index e2ede8ca1db..97883388b12 100644 --- a/Package.swift +++ b/Package.swift @@ -1381,6 +1381,9 @@ let package = Package( resources: [ .process("CountTokenResponses"), .process("GenerateContentResponses"), + ], + cSettings: [ + .headerSearchPath("../../../"), ] ), .testTarget( From 2a6849d654aac7e57574b255ce60aee29f253194 Mon Sep 17 00:00:00 2001 From: Andrew Heard Date: Fri, 7 Jun 2024 15:25:39 -0400 Subject: [PATCH 3/7] Add case in build.sh that checks environment variable --- .../Tests/Integration/IntegrationTests.swift | 5 ----- scripts/build.sh | 9 +++++++++ 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/FirebaseVertexAI/Tests/Integration/IntegrationTests.swift b/FirebaseVertexAI/Tests/Integration/IntegrationTests.swift index d6c3702fd37..fa521ded8b6 100644 --- a/FirebaseVertexAI/Tests/Integration/IntegrationTests.swift +++ b/FirebaseVertexAI/Tests/Integration/IntegrationTests.swift @@ -25,11 +25,6 @@ final class IntegrationTests: XCTestCase { var model: GenerativeModel! override func setUp() async throws { - try XCTSkipIf(ProcessInfo.processInfo.environment["VERTEXAI_RUN_INTEGRATION_TESTS"] == nil, """ - Vertex AI integration tests skipped; to enable them, set the VERTEXAI_RUN_INTEGRATION_TESTS \ - environment variable in Xcode or CI jobs. - """) - let plistPath = try XCTUnwrap(Bundle.module.path( forResource: "GoogleService-Info", ofType: "plist" diff --git a/scripts/build.sh b/scripts/build.sh index e57d1ef6775..712c0858462 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -496,6 +496,15 @@ case "$product-$platform-$method" in build ;; + FirebaseVertexAIIntegration-*-*) + if [[ -n "${VERTEXAI_RUN_INTEGRATION_TESTS:-}" ]]; then + RunXcodebuild \ + -scheme "$product" \ + "${xcb_flags[@]}" \ + test + fi + ;; + VertexSample-*-*) RunXcodebuild \ -project 'FirebaseVertexAI/Sample/VertexAISample.xcodeproj' \ From 73dc5420bca6609414f664635afb9f3357c4167b Mon Sep 17 00:00:00 2001 From: Andrew Heard Date: Fri, 7 Jun 2024 16:08:42 -0400 Subject: [PATCH 4/7] Prefix environment variable with TEST_RUNNER_ --- .github/workflows/vertexai.yml | 2 +- .../Tests/Integration/IntegrationTests.swift | 5 +++++ scripts/build.sh | 9 --------- 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/.github/workflows/vertexai.yml b/.github/workflows/vertexai.yml index f0af71d2d5b..9ad2ac372d1 100644 --- a/.github/workflows/vertexai.yml +++ b/.github/workflows/vertexai.yml @@ -50,7 +50,7 @@ jobs: xcode: Xcode_15.2 runs-on: ${{ matrix.os }} env: - VERTEXAI_RUN_INTEGRATION_TESTS: 1 + TEST_RUNNER_VertexAIRunIntegrationTests: 1 FIREBASECI_USE_LATEST_GOOGLEAPPMEASUREMENT: 1 plist_secret: ${{ secrets.GHASecretsGPGPassphrase1 }} steps: diff --git a/FirebaseVertexAI/Tests/Integration/IntegrationTests.swift b/FirebaseVertexAI/Tests/Integration/IntegrationTests.swift index fa521ded8b6..a67b7b2ace6 100644 --- a/FirebaseVertexAI/Tests/Integration/IntegrationTests.swift +++ b/FirebaseVertexAI/Tests/Integration/IntegrationTests.swift @@ -25,6 +25,11 @@ final class IntegrationTests: XCTestCase { var model: GenerativeModel! override func setUp() async throws { + try XCTSkipIf(ProcessInfo.processInfo.environment["VertexAIRunIntegrationTests"] == nil, """ + Vertex AI integration tests skipped; to enable them, set the VertexAIRunIntegrationTests \ + environment variable in Xcode or CI jobs. + """) + let plistPath = try XCTUnwrap(Bundle.module.path( forResource: "GoogleService-Info", ofType: "plist" diff --git a/scripts/build.sh b/scripts/build.sh index 712c0858462..e57d1ef6775 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -496,15 +496,6 @@ case "$product-$platform-$method" in build ;; - FirebaseVertexAIIntegration-*-*) - if [[ -n "${VERTEXAI_RUN_INTEGRATION_TESTS:-}" ]]; then - RunXcodebuild \ - -scheme "$product" \ - "${xcb_flags[@]}" \ - test - fi - ;; - VertexSample-*-*) RunXcodebuild \ -project 'FirebaseVertexAI/Sample/VertexAISample.xcodeproj' \ From e7981026bc10f636a9c3df8510117994cfbded03 Mon Sep 17 00:00:00 2001 From: Andrew Heard Date: Fri, 7 Jun 2024 16:11:00 -0400 Subject: [PATCH 5/7] Fix formatting --- FirebaseVertexAI/Tests/Integration/IntegrationTests.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/FirebaseVertexAI/Tests/Integration/IntegrationTests.swift b/FirebaseVertexAI/Tests/Integration/IntegrationTests.swift index a67b7b2ace6..ea021f17f6f 100644 --- a/FirebaseVertexAI/Tests/Integration/IntegrationTests.swift +++ b/FirebaseVertexAI/Tests/Integration/IntegrationTests.swift @@ -29,7 +29,7 @@ final class IntegrationTests: XCTestCase { Vertex AI integration tests skipped; to enable them, set the VertexAIRunIntegrationTests \ environment variable in Xcode or CI jobs. """) - + let plistPath = try XCTUnwrap(Bundle.module.path( forResource: "GoogleService-Info", ofType: "plist" From a6c7705a118fe816634f81973c23077b90bf86fb Mon Sep 17 00:00:00 2001 From: Andrew Heard Date: Fri, 7 Jun 2024 16:26:51 -0400 Subject: [PATCH 6/7] Move TEST_RUNNER_VertexAIRunIntegrationTests=1 before build.sh --- .github/workflows/vertexai.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/vertexai.yml b/.github/workflows/vertexai.yml index 9ad2ac372d1..4e17aa617bf 100644 --- a/.github/workflows/vertexai.yml +++ b/.github/workflows/vertexai.yml @@ -50,7 +50,6 @@ jobs: xcode: Xcode_15.2 runs-on: ${{ matrix.os }} env: - TEST_RUNNER_VertexAIRunIntegrationTests: 1 FIREBASECI_USE_LATEST_GOOGLEAPPMEASUREMENT: 1 plist_secret: ${{ secrets.GHASecretsGPGPassphrase1 }} steps: @@ -68,7 +67,8 @@ jobs: max_attempts: 3 retry_on: error retry_wait_seconds: 120 - command: scripts/build.sh FirebaseVertexAIIntegration ${{ matrix.target }} spm + command: TEST_RUNNER_VertexAIRunIntegrationTests=1 scripts/build.sh \ + FirebaseVertexAIIntegration ${{ matrix.target }} spm sample: strategy: From e3d656061582263e58e03ff8f8d10ed7bb5f18ec Mon Sep 17 00:00:00 2001 From: Andrew Heard Date: Fri, 7 Jun 2024 16:31:29 -0400 Subject: [PATCH 7/7] Revert "Move TEST_RUNNER_VertexAIRunIntegrationTests=1 before build.sh" This reverts commit a6c7705a118fe816634f81973c23077b90bf86fb. --- .github/workflows/vertexai.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/vertexai.yml b/.github/workflows/vertexai.yml index 4e17aa617bf..9ad2ac372d1 100644 --- a/.github/workflows/vertexai.yml +++ b/.github/workflows/vertexai.yml @@ -50,6 +50,7 @@ jobs: xcode: Xcode_15.2 runs-on: ${{ matrix.os }} env: + TEST_RUNNER_VertexAIRunIntegrationTests: 1 FIREBASECI_USE_LATEST_GOOGLEAPPMEASUREMENT: 1 plist_secret: ${{ secrets.GHASecretsGPGPassphrase1 }} steps: @@ -67,8 +68,7 @@ jobs: max_attempts: 3 retry_on: error retry_wait_seconds: 120 - command: TEST_RUNNER_VertexAIRunIntegrationTests=1 scripts/build.sh \ - FirebaseVertexAIIntegration ${{ matrix.target }} spm + command: scripts/build.sh FirebaseVertexAIIntegration ${{ matrix.target }} spm sample: strategy: