From bfce084d7019d9a7c435d4343f808ed5a21eada6 Mon Sep 17 00:00:00 2001 From: Andrew Heard Date: Tue, 21 May 2024 17:18:51 -0400 Subject: [PATCH 1/2] [Vertex AI] Add error message for Firebase ML API not enabled --- FirebaseVertexAI/Sources/Errors.swift | 22 +++++++ .../Sources/GenerativeAIService.swift | 25 +++++++- .../Sources/GenerativeModel.swift | 3 + FirebaseVertexAI/Sources/VertexAI.swift | 16 +++-- FirebaseVertexAI/Tests/Unit/ChatTests.swift | 1 + ...ry-failure-firebaseml-api-not-enabled.json | 27 ++++++++ .../Tests/Unit/GenerativeModelTests.swift | 63 +++++++++++++++++++ 7 files changed, 150 insertions(+), 7 deletions(-) create mode 100644 FirebaseVertexAI/Tests/Unit/GenerateContentResponses/unary-failure-firebaseml-api-not-enabled.json diff --git a/FirebaseVertexAI/Sources/Errors.swift b/FirebaseVertexAI/Sources/Errors.swift index 0b1b8848411..17a4dd9e201 100644 --- a/FirebaseVertexAI/Sources/Errors.swift +++ b/FirebaseVertexAI/Sources/Errors.swift @@ -30,6 +30,10 @@ struct RPCError: Error { self.status = status self.details = details } + + func isFirebaseMLServiceDisabledError() -> Bool { + return details.contains { $0.isFirebaseMLServiceDisabledErrorDetails() } + } } extension RPCError: Decodable { @@ -76,10 +80,27 @@ struct ErrorDetails { let type: String let reason: String? let domain: String? + let metadata: [String: String]? func isErrorInfo() -> Bool { return type == ErrorDetails.errorInfoType } + + func isFirebaseMLServiceDisabledErrorDetails() -> Bool { + guard isErrorInfo() else { + return false + } + guard reason == "SERVICE_DISABLED" else { + return false + } + guard domain == "googleapis.com" else { + return false + } + guard let metadata, metadata["service"] == "firebaseml.googleapis.com" else { + return false + } + return true + } } extension ErrorDetails: Decodable, Equatable { @@ -87,6 +108,7 @@ extension ErrorDetails: Decodable, Equatable { case type = "@type" case reason case domain + case metadata } } diff --git a/FirebaseVertexAI/Sources/GenerativeAIService.swift b/FirebaseVertexAI/Sources/GenerativeAIService.swift index 85944adeda9..2d0a7fac503 100644 --- a/FirebaseVertexAI/Sources/GenerativeAIService.swift +++ b/FirebaseVertexAI/Sources/GenerativeAIService.swift @@ -25,6 +25,8 @@ struct GenerativeAIService { /// The Firebase SDK version in the format `fire/`. static let firebaseVersionTag = "fire/\(FirebaseVersion())" + private let projectID: String + /// Gives permission to talk to the backend. private let apiKey: String @@ -34,7 +36,9 @@ struct GenerativeAIService { private let urlSession: URLSession - init(apiKey: String, appCheck: AppCheckInterop?, auth: AuthInterop?, urlSession: URLSession) { + init(projectID: String, apiKey: String, appCheck: AppCheckInterop?, auth: AuthInterop?, + urlSession: URLSession) { + self.projectID = projectID self.apiKey = apiKey self.appCheck = appCheck self.auth = auth @@ -236,13 +240,30 @@ struct GenerativeAIService { private func parseError(responseData: Data) -> Error { do { - return try JSONDecoder().decode(RPCError.self, from: responseData) + let rpcError = try JSONDecoder().decode(RPCError.self, from: responseData) + logRPCError(rpcError) + return rpcError } catch { // TODO: Return an error about an unrecognized error payload with the response body return error } } + // Log specific RPC errors that cannot be mitigated or handled by user code. + // These errors do not produce specific GenerateContentError or CountTokensError cases. + private func logRPCError(_ error: RPCError) { + if error.isFirebaseMLServiceDisabledError() { + Logging.default.error(""" + The Vertex AI for Firebase SDK requires the Firebase ML API `firebaseml.googleapis.com` to \ + be enabled for your project. Get started in the Firebase Console \ + (https://console.firebase.google.com/project/\(projectID)/genai/vertex) or verify that the \ + API is enabled in the Google Cloud Console \ + (https://console.developers.google.com/apis/api/firebaseml.googleapis.com/overview?project=\ + \(projectID)). + """) + } + } + private func parseResponse(_ type: T.Type, from data: Data) throws -> T { do { return try JSONDecoder().decode(type, from: data) diff --git a/FirebaseVertexAI/Sources/GenerativeModel.swift b/FirebaseVertexAI/Sources/GenerativeModel.swift index e0778a4f228..6cce3f02a93 100644 --- a/FirebaseVertexAI/Sources/GenerativeModel.swift +++ b/FirebaseVertexAI/Sources/GenerativeModel.swift @@ -51,6 +51,7 @@ public final class GenerativeModel { /// /// - Parameters: /// - name: The name of the model to use, for example `"gemini-1.0-pro"`. + /// - projectID: The project ID from the Firebase console. /// - apiKey: The API key for your project. /// - generationConfig: The content generation parameters your model should use. /// - safetySettings: A value describing what types of harmful content your model should allow. @@ -61,6 +62,7 @@ public final class GenerativeModel { /// - requestOptions: Configuration parameters for sending requests to the backend. /// - urlSession: The `URLSession` to use for requests; defaults to `URLSession.shared`. init(name: String, + projectID: String, apiKey: String, generationConfig: GenerationConfig? = nil, safetySettings: [SafetySetting]? = nil, @@ -73,6 +75,7 @@ public final class GenerativeModel { urlSession: URLSession = .shared) { modelResourceName = GenerativeModel.modelResourceName(name: name) generativeAIService = GenerativeAIService( + projectID: projectID, apiKey: apiKey, appCheck: appCheck, auth: auth, diff --git a/FirebaseVertexAI/Sources/VertexAI.swift b/FirebaseVertexAI/Sources/VertexAI.swift index 5069476b6ad..2017ea04b5f 100644 --- a/FirebaseVertexAI/Sources/VertexAI.swift +++ b/FirebaseVertexAI/Sources/VertexAI.swift @@ -83,7 +83,15 @@ public class VertexAI: NSObject { systemInstruction: ModelContent? = nil, requestOptions: RequestOptions = RequestOptions()) -> GenerativeModel { - let modelResourceName = modelResourceName(modelName: modelName, location: location) + guard let projectID = app.options.projectID else { + fatalError("The Firebase app named \"\(app.name)\" has no project ID in its configuration.") + } + + let modelResourceName = modelResourceName( + modelName: modelName, + projectID: projectID, + location: location + ) guard let apiKey = app.options.apiKey else { fatalError("The Firebase app named \"\(app.name)\" has no API key in its configuration.") @@ -91,6 +99,7 @@ public class VertexAI: NSObject { return GenerativeModel( name: modelResourceName, + projectID: projectID, apiKey: apiKey, generationConfig: generationConfig, safetySettings: safetySettings, @@ -121,10 +130,7 @@ public class VertexAI: NSObject { auth = ComponentType.instance(for: AuthInterop.self, in: app.container) } - private func modelResourceName(modelName: String, location: String) -> String { - guard let projectID = app.options.projectID else { - fatalError("The Firebase app named \"\(app.name)\" has no project ID in its configuration.") - } + private func modelResourceName(modelName: String, projectID: String, location: String) -> String { guard !modelName.isEmpty && modelName .allSatisfy({ !$0.isWhitespace && !$0.isNewline && $0 != "/" }) else { fatalError(""" diff --git a/FirebaseVertexAI/Tests/Unit/ChatTests.swift b/FirebaseVertexAI/Tests/Unit/ChatTests.swift index ea2633bdf40..2ce063c7ef2 100644 --- a/FirebaseVertexAI/Tests/Unit/ChatTests.swift +++ b/FirebaseVertexAI/Tests/Unit/ChatTests.swift @@ -49,6 +49,7 @@ final class ChatTests: XCTestCase { let model = GenerativeModel( name: "my-model", + projectID: "my-project-id", apiKey: "API_KEY", tools: nil, requestOptions: RequestOptions(), diff --git a/FirebaseVertexAI/Tests/Unit/GenerateContentResponses/unary-failure-firebaseml-api-not-enabled.json b/FirebaseVertexAI/Tests/Unit/GenerateContentResponses/unary-failure-firebaseml-api-not-enabled.json new file mode 100644 index 00000000000..7526cd7ae18 --- /dev/null +++ b/FirebaseVertexAI/Tests/Unit/GenerateContentResponses/unary-failure-firebaseml-api-not-enabled.json @@ -0,0 +1,27 @@ +{ + "error": { + "code": 403, + "message": "Firebase ML API has not been used in project 1234567890 before or it is disabled. Enable it by visiting https://console.developers.google.com/apis/api/firebaseml.googleapis.com/overview?project=1234567890 then retry. If you enabled this API recently, wait a few minutes for the action to propagate to our systems and retry.", + "status": "PERMISSION_DENIED", + "details": [ + { + "@type": "type.googleapis.com/google.rpc.Help", + "links": [ + { + "description": "Google developers console API activation", + "url": "https://console.developers.google.com/apis/api/firebaseml.googleapis.com/overview?project=1234567890" + } + ] + }, + { + "@type": "type.googleapis.com/google.rpc.ErrorInfo", + "reason": "SERVICE_DISABLED", + "domain": "googleapis.com", + "metadata": { + "service": "firebaseml.googleapis.com", + "consumer": "projects/1234567890" + } + } + ] + } +} diff --git a/FirebaseVertexAI/Tests/Unit/GenerativeModelTests.swift b/FirebaseVertexAI/Tests/Unit/GenerativeModelTests.swift index 552c2693fd7..87d308b14a1 100644 --- a/FirebaseVertexAI/Tests/Unit/GenerativeModelTests.swift +++ b/FirebaseVertexAI/Tests/Unit/GenerativeModelTests.swift @@ -38,6 +38,7 @@ final class GenerativeModelTests: XCTestCase { urlSession = try XCTUnwrap(URLSession(configuration: configuration)) model = GenerativeModel( name: "my-model", + projectID: "my-project-id", apiKey: "API_KEY", tools: nil, requestOptions: RequestOptions(), @@ -180,6 +181,7 @@ final class GenerativeModelTests: XCTestCase { let model = GenerativeModel( // Model name is prefixed with "models/". name: "models/test-model", + projectID: "my-project-id", apiKey: "API_KEY", tools: nil, requestOptions: RequestOptions(), @@ -299,6 +301,7 @@ final class GenerativeModelTests: XCTestCase { let appCheckToken = "test-valid-token" model = GenerativeModel( name: "my-model", + projectID: "my-project-id", apiKey: "API_KEY", tools: nil, requestOptions: RequestOptions(), @@ -319,6 +322,7 @@ final class GenerativeModelTests: XCTestCase { func testGenerateContent_appCheck_tokenRefreshError() async throws { model = GenerativeModel( name: "my-model", + projectID: "my-project-id", apiKey: "API_KEY", tools: nil, requestOptions: RequestOptions(), @@ -340,6 +344,7 @@ final class GenerativeModelTests: XCTestCase { let authToken = "test-valid-token" model = GenerativeModel( name: "my-model", + projectID: "my-project-id", apiKey: "API_KEY", tools: nil, requestOptions: RequestOptions(), @@ -360,6 +365,7 @@ final class GenerativeModelTests: XCTestCase { func testGenerateContent_auth_nilAuthToken() async throws { model = GenerativeModel( name: "my-model", + projectID: "my-project-id", apiKey: "API_KEY", tools: nil, requestOptions: RequestOptions(), @@ -380,6 +386,7 @@ final class GenerativeModelTests: XCTestCase { func testGenerateContent_auth_authTokenRefreshError() async throws { model = GenerativeModel( name: "my-model", + projectID: "my-project-id", apiKey: "API_KEY", tools: nil, requestOptions: RequestOptions(), @@ -441,6 +448,29 @@ final class GenerativeModelTests: XCTestCase { } } + func testGenerateContent_failure_firebaseMLAPINotEnabled() async throws { + let expectedStatusCode = 403 + MockURLProtocol + .requestHandler = try httpRequestHandler( + forResource: "unary-failure-firebaseml-api-not-enabled", + withExtension: "json", + statusCode: expectedStatusCode + ) + + do { + _ = try await model.generateContent(testPrompt) + XCTFail("Should throw GenerateContentError.internalError; no error thrown.") + } catch let GenerateContentError.internalError(error as RPCError) { + XCTAssertEqual(error.httpResponseCode, expectedStatusCode) + XCTAssertEqual(error.status, .permissionDenied) + XCTAssertTrue(error.message.starts(with: "Firebase ML API has not been used in project")) + XCTAssertTrue(error.isFirebaseMLServiceDisabledError()) + return + } catch { + XCTFail("Should throw GenerateContentError.internalError(RPCError); error thrown: \(error)") + } + } + func testGenerateContent_failure_emptyContent() async throws { MockURLProtocol .requestHandler = try httpRequestHandler( @@ -701,6 +731,7 @@ final class GenerativeModelTests: XCTestCase { let requestOptions = RequestOptions(timeout: expectedTimeout) model = GenerativeModel( name: "my-model", + projectID: "my-project-id", apiKey: "API_KEY", tools: nil, requestOptions: requestOptions, @@ -738,6 +769,31 @@ final class GenerativeModelTests: XCTestCase { XCTFail("Should have caught an error.") } + func testGenerateContentStream_failure_firebaseMLAPINotEnabled() async throws { + let expectedStatusCode = 403 + MockURLProtocol + .requestHandler = try httpRequestHandler( + forResource: "unary-failure-firebaseml-api-not-enabled", + withExtension: "json", + statusCode: expectedStatusCode + ) + + do { + let stream = model.generateContentStream(testPrompt) + for try await _ in stream { + XCTFail("No content is there, this shouldn't happen.") + } + } catch let GenerateContentError.internalError(error as RPCError) { + XCTAssertEqual(error.httpResponseCode, expectedStatusCode) + XCTAssertEqual(error.status, .permissionDenied) + XCTAssertTrue(error.message.starts(with: "Firebase ML API has not been used in project")) + XCTAssertTrue(error.isFirebaseMLServiceDisabledError()) + return + } + + XCTFail("Should have caught an error.") + } + func testGenerateContentStream_failureEmptyContent() async throws { MockURLProtocol .requestHandler = try httpRequestHandler( @@ -912,6 +968,7 @@ final class GenerativeModelTests: XCTestCase { let appCheckToken = "test-valid-token" model = GenerativeModel( name: "my-model", + projectID: "my-project-id", apiKey: "API_KEY", tools: nil, requestOptions: RequestOptions(), @@ -933,6 +990,7 @@ final class GenerativeModelTests: XCTestCase { func testGenerateContentStream_appCheck_tokenRefreshError() async throws { model = GenerativeModel( name: "my-model", + projectID: "my-project-id", apiKey: "API_KEY", tools: nil, requestOptions: RequestOptions(), @@ -1078,6 +1136,7 @@ final class GenerativeModelTests: XCTestCase { let requestOptions = RequestOptions(timeout: expectedTimeout) model = GenerativeModel( name: "my-model", + projectID: "my-project-id", apiKey: "API_KEY", tools: nil, requestOptions: requestOptions, @@ -1155,6 +1214,7 @@ final class GenerativeModelTests: XCTestCase { let requestOptions = RequestOptions(timeout: expectedTimeout) model = GenerativeModel( name: "my-model", + projectID: "my-project-id", apiKey: "API_KEY", tools: nil, requestOptions: requestOptions, @@ -1176,6 +1236,7 @@ final class GenerativeModelTests: XCTestCase { model = GenerativeModel( name: modelName, + projectID: "my-project-id", apiKey: "API_KEY", tools: nil, requestOptions: RequestOptions(), @@ -1191,6 +1252,7 @@ final class GenerativeModelTests: XCTestCase { model = GenerativeModel( name: modelResourceName, + projectID: "my-project-id", apiKey: "API_KEY", tools: nil, requestOptions: RequestOptions(), @@ -1206,6 +1268,7 @@ final class GenerativeModelTests: XCTestCase { model = GenerativeModel( name: tunedModelResourceName, + projectID: "my-project-id", apiKey: "API_KEY", tools: nil, requestOptions: RequestOptions(), From dfdb631f2b266b66c019d76a4ec1c91c327310a3 Mon Sep 17 00:00:00 2001 From: Andrew Heard Date: Tue, 21 May 2024 17:28:47 -0400 Subject: [PATCH 2/2] Set FIREBASECI_USE_LATEST_GOOGLEAPPMEASUREMENT=1 in sample build job --- .github/workflows/vertexai.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/vertexai.yml b/.github/workflows/vertexai.yml index ebf58529f54..da827b50912 100644 --- a/.github/workflows/vertexai.yml +++ b/.github/workflows/vertexai.yml @@ -51,6 +51,8 @@ jobs: - os: macos-14 xcode: Xcode_15.2 runs-on: ${{ matrix.os }} + env: + FIREBASECI_USE_LATEST_GOOGLEAPPMEASUREMENT: 1 steps: - uses: actions/checkout@v4 - name: Xcode