diff --git a/.github/workflows/vertexai.yml b/.github/workflows/vertexai.yml new file mode 100644 index 00000000000..ebf58529f54 --- /dev/null +++ b/.github/workflows/vertexai.yml @@ -0,0 +1,68 @@ +name: vertexai + +on: + pull_request: + paths: + - 'FirebaseVertexAI**' + - '.github/workflows/vertexai.yml' + - 'Gemfile*' + schedule: + # Run every day at 11pm (PST) - cron uses UTC times + - cron: '0 7 * * *' + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} + cancel-in-progress: true + +jobs: + spm: + strategy: + matrix: + target: [iOS, macOS, catalyst] + os: [macos-13] + include: + - os: macos-13 + xcode: Xcode_15.2 + runs-on: ${{ matrix.os }} + env: + FIREBASECI_USE_LATEST_GOOGLEAPPMEASUREMENT: 1 + steps: + - uses: actions/checkout@v4 + - name: Xcode + run: sudo xcode-select -s /Applications/${{ matrix.xcode }}.app/Contents/Developer + - name: Initialize xcodebuild + run: scripts/setup_spm_tests.sh + - uses: nick-fields/retry@v3 + with: + timeout_minutes: 120 + max_attempts: 3 + retry_on: error + retry_wait_seconds: 120 + command: scripts/build.sh FirebaseVertexAIUnit ${{ matrix.target }} spm + + sample: + strategy: + matrix: + # Test build with debug and release configs (whether or not DEBUG is set and optimization level) + build: [build] + include: + - os: macos-13 + xcode: Xcode_15.0.1 + - os: macos-14 + xcode: Xcode_15.2 + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - name: Xcode + run: sudo xcode-select -s /Applications/${{ matrix.xcode }}.app/Contents/Developer + - name: Initialize xcodebuild + run: xcodebuild -list + - name: Placeholder GoogleService-Info.plist for build testing + run: cp FirebaseCore/Tests/Unit/Resources/GoogleService-Info.plist FirebaseVertexAI/Sample/ + - uses: nick-fields/retry@v3 + with: + timeout_minutes: 120 + max_attempts: 3 + retry_on: error + retry_wait_seconds: 120 + command: scripts/build.sh VertexSample iOS diff --git a/Dangerfile b/Dangerfile index 9579b574add..87b999c2c86 100644 --- a/Dangerfile +++ b/Dangerfile @@ -64,6 +64,7 @@ def labelsForModifiedFiles() labels.push("api: performance") if @has_performance_changes labels.push("api: remoteconfig") if @has_remoteconfig_changes labels.push("api: storage") if @has_storage_changes + labels.push("api: vertexai") if @has_vertexai_changes labels.push("release-tooling") if @has_releasetooling_changes labels.push("public-api-change") if @has_api_changes return labels @@ -100,7 +101,8 @@ has_license_changes = didModify(["LICENSE"]) "Messaging", "Performance", "RemoteConfig", - "Storage" + "Storage", + "VertexAI" ] ## Product directories @@ -149,6 +151,7 @@ has_license_changes = didModify(["LICENSE"]) @has_remoteconfig_changes = hasChangesIn("FirebaseRemoteConfig") @has_remoteconfig_api_changes = hasChangesIn("FirebaseRemoteConfig/Sources/Public/") @has_storage_changes = hasChangesIn("FirebaseStorage") +@has_vertexai_changes = hasChangesIn("FirebaseVertexAI") @has_releasetooling_changes = hasChangesIn("ReleaseTooling/") @has_public_additions = hasAdditionsIn("Public/") diff --git a/FirebaseCore/Sources/FIRApp.m b/FirebaseCore/Sources/FIRApp.m index 196cba65f8f..c942cecfe51 100644 --- a/FirebaseCore/Sources/FIRApp.m +++ b/FirebaseCore/Sources/FIRApp.m @@ -829,6 +829,7 @@ + (void)registerSwiftComponents { @"FIRSessions" : @"fire-ses", @"FIRFunctionsComponent" : @"fire-fun", @"FIRStorageComponent" : @"fire-str", + @"FIRVertexAIComponent" : @"fire-vertex", }; for (NSString *className in swiftComponents.allKeys) { Class klass = NSClassFromString(className); diff --git a/FirebaseCore/Sources/FIROptions.m b/FirebaseCore/Sources/FIROptions.m index d46b657501c..c0625a8bac9 100644 --- a/FirebaseCore/Sources/FIROptions.m +++ b/FirebaseCore/Sources/FIROptions.m @@ -284,7 +284,7 @@ - (NSString *)libraryVersionID { // The unit tests are set up to catch anything that does not properly convert. NSString *version = FIRFirebaseVersion(); NSArray *components = [version componentsSeparatedByString:@"."]; - NSString *major = [components objectAtIndex:0]; + NSString *major = [NSString stringWithFormat:@"%02d", [[components objectAtIndex:0] intValue]]; NSString *minor = [NSString stringWithFormat:@"%02d", [[components objectAtIndex:1] intValue]]; NSString *patch = [NSString stringWithFormat:@"%02d", [[components objectAtIndex:2] intValue]]; kFIRLibraryVersionID = [NSString stringWithFormat:@"%@%@%@000", major, minor, patch]; diff --git a/FirebaseCore/Tests/Unit/FIROptionsTest.m b/FirebaseCore/Tests/Unit/FIROptionsTest.m index 3de059f4191..fd5c901598f 100644 --- a/FirebaseCore/Tests/Unit/FIROptionsTest.m +++ b/FirebaseCore/Tests/Unit/FIROptionsTest.m @@ -624,7 +624,7 @@ - (void)testVersionConsistency { int minor = (versionString[2] - '0') * 10 + versionString[3] - '0'; int patch = (versionString[4] - '0') * 10 + versionString[5] - '0'; NSString *str = [NSString stringWithFormat:@"%d.%d.%d", major, minor, patch]; - XCTAssertEqualObjects(str, FIRFirebaseVersion()); + XCTAssertTrue([FIRFirebaseVersion() hasPrefix:str]); } // Repeat test with more Objective-C. @@ -638,11 +638,11 @@ - (void)testVersionConsistency2 { NSRange major = NSMakeRange(0, 2); NSRange minor = NSMakeRange(2, 2); NSRange patch = NSMakeRange(4, 2); - NSString *str = - [NSString stringWithFormat:@"%@.%d.%d", [kFIRLibraryVersionID substringWithRange:major], - [[kFIRLibraryVersionID substringWithRange:minor] intValue], - [[kFIRLibraryVersionID substringWithRange:patch] intValue]]; - XCTAssertEqualObjects(str, FIRFirebaseVersion()); + NSString *str = [NSString + stringWithFormat:@"%d.%d.%d", [[kFIRLibraryVersionID substringWithRange:major] intValue], + [[kFIRLibraryVersionID substringWithRange:minor] intValue], + [[kFIRLibraryVersionID substringWithRange:patch] intValue]]; + XCTAssertTrue([FIRFirebaseVersion() hasPrefix:str]); } #pragma mark - Helpers diff --git a/FirebaseVertexAI-Docs.podspec b/FirebaseVertexAI-Docs.podspec new file mode 100644 index 00000000000..49765e50e3c --- /dev/null +++ b/FirebaseVertexAI-Docs.podspec @@ -0,0 +1,55 @@ +Pod::Spec.new do |s| + s.name = 'FirebaseVertexAI' + s.version = '10.26.0' + s.summary = 'Firebase VertexAI' + + s.description = <<-DESC + Placeholder podspec for docsgen only. Do not use this pod. + DESC + + s.homepage = 'https://firebase.google.com' + s.license = { :type => 'Apache-2.0', :file => 'LICENSE' } + s.authors = 'Google, Inc.' + + s.source = { + :git => 'https://github.com/firebase/firebase-ios-sdk.git', + # TODO: this should be `'CocoaPods-' + s.version.to_s` (after May 14 2024) + :tag => 'release-10.26' + } + + s.social_media_url = 'https://twitter.com/Firebase' + + ios_deployment_target = '15.0' + osx_deployment_target = '10.14' + + s.ios.deployment_target = ios_deployment_target + s.osx.deployment_target = osx_deployment_target + + s.cocoapods_version = '>= 1.12.0' + s.prefix_header_file = false + + s.source_files = [ + 'FirebaseVertexAI/Sources/**/*.swift', + 'FirebaseCore/Extension/*.h', + 'FirebaseAuth/Interop/*.h', + ] + + s.swift_version = '5.3' + + s.framework = 'Foundation' + s.ios.framework = 'UIKit' + s.osx.framework = 'AppKit' + s.tvos.framework = 'UIKit' + s.watchos.framework = 'WatchKit' + + s.dependency 'FirebaseCore', '~> 10.0' + s.dependency 'FirebaseCoreExtension' + s.dependency 'FirebaseAuthInterop' + s.dependency 'FirebaseAppCheckInterop', '~> 10.17' + + s.pod_target_xcconfig = { + 'GCC_C_LANGUAGE_STANDARD' => 'c99', + 'HEADER_SEARCH_PATHS' => '"${PODS_TARGET_SRCROOT}"', + 'OTHER_CFLAGS' => '-fno-autolink' + } +end diff --git a/FirebaseVertexAI/CHANGELOG.md b/FirebaseVertexAI/CHANGELOG.md new file mode 100644 index 00000000000..1aa3a9005c6 --- /dev/null +++ b/FirebaseVertexAI/CHANGELOG.md @@ -0,0 +1,6 @@ +# 10.26.0 + +- [feature] Initial release of the Vertex AI for Firebase SDK (public preview). + Learn how to + [get started](https://firebase.google.com/docs/vertex-ai/get-started?platform=ios) + with the SDK in your app. diff --git a/FirebaseVertexAI/README.md b/FirebaseVertexAI/README.md new file mode 100644 index 00000000000..398a183eecb --- /dev/null +++ b/FirebaseVertexAI/README.md @@ -0,0 +1,8 @@ +# Vertex AI for Firebase SDK + +**Preview**: Vertex AI for Firebase is in Public Preview, which means that the product is +not subject to any SLA or deprecation policy and could change in backwards-incompatible +ways. + +- For developer documentation, please visit https://firebase.google.com/docs/vertex-ai. +- Try out the [sample app](Sample/README.md) to get started. diff --git a/FirebaseVertexAI/Sample/ChatSample/Assets.xcassets/AccentColor.colorset/Contents.json b/FirebaseVertexAI/Sample/ChatSample/Assets.xcassets/AccentColor.colorset/Contents.json new file mode 100644 index 00000000000..eb878970081 --- /dev/null +++ b/FirebaseVertexAI/Sample/ChatSample/Assets.xcassets/AccentColor.colorset/Contents.json @@ -0,0 +1,11 @@ +{ + "colors" : [ + { + "idiom" : "universal" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/FirebaseVertexAI/Sample/ChatSample/Assets.xcassets/AppIcon.appiconset/Contents.json b/FirebaseVertexAI/Sample/ChatSample/Assets.xcassets/AppIcon.appiconset/Contents.json new file mode 100644 index 00000000000..13613e3ee1a --- /dev/null +++ b/FirebaseVertexAI/Sample/ChatSample/Assets.xcassets/AppIcon.appiconset/Contents.json @@ -0,0 +1,13 @@ +{ + "images" : [ + { + "idiom" : "universal", + "platform" : "ios", + "size" : "1024x1024" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/FirebaseVertexAI/Sample/ChatSample/Assets.xcassets/Contents.json b/FirebaseVertexAI/Sample/ChatSample/Assets.xcassets/Contents.json new file mode 100644 index 00000000000..73c00596a7f --- /dev/null +++ b/FirebaseVertexAI/Sample/ChatSample/Assets.xcassets/Contents.json @@ -0,0 +1,6 @@ +{ + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/FirebaseVertexAI/Sample/ChatSample/Models/ChatMessage.swift b/FirebaseVertexAI/Sample/ChatSample/Models/ChatMessage.swift new file mode 100644 index 00000000000..6f7ab321b12 --- /dev/null +++ b/FirebaseVertexAI/Sample/ChatSample/Models/ChatMessage.swift @@ -0,0 +1,64 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import Foundation + +enum Participant { + case system + case user +} + +struct ChatMessage: Identifiable, Equatable { + let id = UUID().uuidString + var message: String + let participant: Participant + var pending = false + + static func pending(participant: Participant) -> ChatMessage { + Self(message: "", participant: participant, pending: true) + } +} + +extension ChatMessage { + static var samples: [ChatMessage] = [ + .init(message: "Hello. What can I do for you today?", participant: .system), + .init(message: "Show me a simple loop in Swift.", participant: .user), + .init(message: """ + Sure, here is a simple loop in Swift: + + # Example 1 + ``` + for i in 1...5 { + print("Hello, world!") + } + ``` + + This loop will print the string "Hello, world!" five times. The for loop iterates over a range of numbers, + in this case the numbers from 1 to 5. The variable i is assigned each number in the range, and the code inside the loop is executed. + + **Here is another example of a simple loop in Swift:** + ```swift + var sum = 0 + for i in 1...100 { + sum += i + } + print("The sum of the numbers from 1 to 100 is \\(sum).") + ``` + + This loop calculates the sum of the numbers from 1 to 100. The variable sum is initialized to 0, and then the for loop iterates over the range of numbers from 1 to 100. The variable i is assigned each number in the range, and the value of i is added to the sum variable. After the loop has finished executing, the value of sum is printed to the console. + """, participant: .system), + ] + + static var sample = samples[0] +} diff --git a/FirebaseVertexAI/Sample/ChatSample/Preview Content/Preview Assets.xcassets/Contents.json b/FirebaseVertexAI/Sample/ChatSample/Preview Content/Preview Assets.xcassets/Contents.json new file mode 100644 index 00000000000..73c00596a7f --- /dev/null +++ b/FirebaseVertexAI/Sample/ChatSample/Preview Content/Preview Assets.xcassets/Contents.json @@ -0,0 +1,6 @@ +{ + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/FirebaseVertexAI/Sample/ChatSample/Screens/ConversationScreen.swift b/FirebaseVertexAI/Sample/ChatSample/Screens/ConversationScreen.swift new file mode 100644 index 00000000000..a3044a1f430 --- /dev/null +++ b/FirebaseVertexAI/Sample/ChatSample/Screens/ConversationScreen.swift @@ -0,0 +1,127 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import FirebaseVertexAI +import GenerativeAIUIComponents +import SwiftUI + +struct ConversationScreen: View { + @EnvironmentObject + var viewModel: ConversationViewModel + + @State + private var userPrompt = "" + + enum FocusedField: Hashable { + case message + } + + @FocusState + var focusedField: FocusedField? + + var body: some View { + VStack { + ScrollViewReader { scrollViewProxy in + List { + ForEach(viewModel.messages) { message in + MessageView(message: message) + } + if let error = viewModel.error { + ErrorView(error: error) + .tag("errorView") + } + } + .listStyle(.plain) + .onChange(of: viewModel.messages, perform: { newValue in + if viewModel.hasError { + // wait for a short moment to make sure we can actually scroll to the bottom + DispatchQueue.main.asyncAfter(deadline: .now() + 0.05) { + withAnimation { + scrollViewProxy.scrollTo("errorView", anchor: .bottom) + } + focusedField = .message + } + } else { + guard let lastMessage = viewModel.messages.last else { return } + + // wait for a short moment to make sure we can actually scroll to the bottom + DispatchQueue.main.asyncAfter(deadline: .now() + 0.05) { + withAnimation { + scrollViewProxy.scrollTo(lastMessage.id, anchor: .bottom) + } + focusedField = .message + } + } + }) + } + InputField("Message...", text: $userPrompt) { + Image(systemName: viewModel.busy ? "stop.circle.fill" : "arrow.up.circle.fill") + .font(.title) + } + .focused($focusedField, equals: .message) + .onSubmit { sendOrStop() } + } + .toolbar { + ToolbarItem(placement: .primaryAction) { + Button(action: newChat) { + Image(systemName: "square.and.pencil") + } + } + } + .navigationTitle("Chat sample") + .onAppear { + focusedField = .message + } + } + + private func sendMessage() { + Task { + let prompt = userPrompt + userPrompt = "" + await viewModel.sendMessage(prompt, streaming: true) + } + } + + private func sendOrStop() { + if viewModel.busy { + viewModel.stop() + } else { + sendMessage() + } + } + + private func newChat() { + viewModel.startNewChat() + } +} + +struct ConversationScreen_Previews: PreviewProvider { + struct ContainerView: View { + @StateObject var viewModel = ConversationViewModel() + + var body: some View { + ConversationScreen() + .environmentObject(viewModel) + .onAppear { + viewModel.messages = ChatMessage.samples + } + } + } + + static var previews: some View { + NavigationStack { + ConversationScreen() + } + } +} diff --git a/FirebaseVertexAI/Sample/ChatSample/ViewModels/ConversationViewModel.swift b/FirebaseVertexAI/Sample/ChatSample/ViewModels/ConversationViewModel.swift new file mode 100644 index 00000000000..ea63f6af95b --- /dev/null +++ b/FirebaseVertexAI/Sample/ChatSample/ViewModels/ConversationViewModel.swift @@ -0,0 +1,130 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import FirebaseVertexAI +import Foundation +import UIKit + +@MainActor +class ConversationViewModel: ObservableObject { + /// This array holds both the user's and the system's chat messages + @Published var messages = [ChatMessage]() + + /// Indicates we're waiting for the model to finish + @Published var busy = false + + @Published var error: Error? + var hasError: Bool { + return error != nil + } + + private var model: GenerativeModel + private var chat: Chat + private var stopGenerating = false + + private var chatTask: Task? + + init() { + model = VertexAI.vertexAI().generativeModel(modelName: "gemini-1.5-pro-preview-0409") + chat = model.startChat() + } + + func sendMessage(_ text: String, streaming: Bool = true) async { + error = nil + if streaming { + await internalSendMessageStreaming(text) + } else { + await internalSendMessage(text) + } + } + + func startNewChat() { + stop() + error = nil + chat = model.startChat() + messages.removeAll() + } + + func stop() { + chatTask?.cancel() + error = nil + } + + private func internalSendMessageStreaming(_ text: String) async { + chatTask?.cancel() + + chatTask = Task { + busy = true + defer { + busy = false + } + + // first, add the user's message to the chat + let userMessage = ChatMessage(message: text, participant: .user) + messages.append(userMessage) + + // add a pending message while we're waiting for a response from the backend + let systemMessage = ChatMessage.pending(participant: .system) + messages.append(systemMessage) + + do { + let responseStream = chat.sendMessageStream(text) + for try await chunk in responseStream { + messages[messages.count - 1].pending = false + if let text = chunk.text { + messages[messages.count - 1].message += text + } + } + } catch { + self.error = error + print(error.localizedDescription) + messages.removeLast() + } + } + } + + private func internalSendMessage(_ text: String) async { + chatTask?.cancel() + + chatTask = Task { + busy = true + defer { + busy = false + } + + // first, add the user's message to the chat + let userMessage = ChatMessage(message: text, participant: .user) + messages.append(userMessage) + + // add a pending message while we're waiting for a response from the backend + let systemMessage = ChatMessage.pending(participant: .system) + messages.append(systemMessage) + + do { + var response: GenerateContentResponse? + response = try await chat.sendMessage(text) + + if let responseText = response?.text { + // replace pending message with backend response + messages[messages.count - 1].message = responseText + messages[messages.count - 1].pending = false + } + } catch { + self.error = error + print(error.localizedDescription) + messages.removeLast() + } + } + } +} diff --git a/FirebaseVertexAI/Sample/ChatSample/Views/BouncingDots.swift b/FirebaseVertexAI/Sample/ChatSample/Views/BouncingDots.swift new file mode 100644 index 00000000000..6895e6723da --- /dev/null +++ b/FirebaseVertexAI/Sample/ChatSample/Views/BouncingDots.swift @@ -0,0 +1,77 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import SwiftUI + +struct BouncingDots: View { + @State + private var dot1YOffset: CGFloat = 0.0 + + @State + private var dot2YOffset: CGFloat = 0.0 + + @State + private var dot3YOffset: CGFloat = 0.0 + + let animation = Animation.easeInOut(duration: 0.8) + .repeatForever(autoreverses: true) + + var body: some View { + HStack(spacing: 8) { + Circle() + .fill(Color.white) + .frame(width: 10, height: 10) + .offset(y: dot1YOffset) + .onAppear { + withAnimation(self.animation.delay(0.0)) { + self.dot1YOffset = -5 + } + } + Circle() + .fill(Color.white) + .frame(width: 10, height: 10) + .offset(y: dot2YOffset) + .onAppear { + withAnimation(self.animation.delay(0.2)) { + self.dot2YOffset = -5 + } + } + Circle() + .fill(Color.white) + .frame(width: 10, height: 10) + .offset(y: dot3YOffset) + .onAppear { + withAnimation(self.animation.delay(0.4)) { + self.dot3YOffset = -5 + } + } + } + .onAppear { + let baseOffset: CGFloat = -2 + + self.dot1YOffset = baseOffset + self.dot2YOffset = baseOffset + self.dot3YOffset = baseOffset + } + } +} + +struct BouncingDots_Previews: PreviewProvider { + static var previews: some View { + BouncingDots() + .frame(width: 200, height: 50) + .background(.blue) + .roundedCorner(10, corners: [.allCorners]) + } +} diff --git a/FirebaseVertexAI/Sample/ChatSample/Views/ErrorDetailsView.swift b/FirebaseVertexAI/Sample/ChatSample/Views/ErrorDetailsView.swift new file mode 100644 index 00000000000..dc5ce8f9561 --- /dev/null +++ b/FirebaseVertexAI/Sample/ChatSample/Views/ErrorDetailsView.swift @@ -0,0 +1,206 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import FirebaseVertexAI +import MarkdownUI +import SwiftUI + +extension SafetySetting.HarmCategory: CustomStringConvertible { + public var description: String { + switch self { + case .dangerousContent: "Dangerous content" + case .harassment: "Harassment" + case .hateSpeech: "Hate speech" + case .sexuallyExplicit: "Sexually explicit" + case .unknown: "Unknown" + case .unspecified: "Unspecified" + } + } +} + +extension SafetyRating.HarmProbability: CustomStringConvertible { + public var description: String { + switch self { + case .high: "High" + case .low: "Low" + case .medium: "Medium" + case .negligible: "Negligible" + case .unknown: "Unknown" + case .unspecified: "Unspecified" + } + } +} + +private struct SubtitleFormRow: View { + var title: String + var value: String + + var body: some View { + VStack(alignment: .leading) { + Text(title) + .font(.subheadline) + Text(value) + } + } +} + +private struct SubtitleMarkdownFormRow: View { + var title: String + var value: String + + var body: some View { + VStack(alignment: .leading) { + Text(title) + .font(.subheadline) + Markdown(value) + } + } +} + +private struct SafetyRatingsSection: View { + var ratings: [SafetyRating] + + var body: some View { + Section("Safety ratings") { + List(ratings, id: \.self) { rating in + HStack { + Text("\(String(describing: rating.category))") + .font(.subheadline) + Spacer() + Text("\(String(describing: rating.probability))") + } + } + } + } +} + +struct ErrorDetailsView: View { + var error: Error + + var body: some View { + NavigationView { + Form { + switch error { + case let GenerateContentError.internalError(underlying: underlyingError): + Section("Error Type") { + Text("Internal error") + } + + Section("Details") { + SubtitleFormRow(title: "Error description", + value: underlyingError.localizedDescription) + } + + case let GenerateContentError.promptBlocked(response: generateContentResponse): + Section("Error Type") { + Text("Your prompt was blocked") + } + + Section("Details") { + if let reason = generateContentResponse.promptFeedback?.blockReason { + SubtitleFormRow(title: "Reason for blocking", value: reason.rawValue) + } + + if let text = generateContentResponse.text { + SubtitleMarkdownFormRow(title: "Last chunk for the response", value: text) + } + } + + if let ratings = generateContentResponse.candidates.first?.safetyRatings { + SafetyRatingsSection(ratings: ratings) + } + + case let GenerateContentError.responseStoppedEarly( + reason: finishReason, + response: generateContentResponse + ): + + Section("Error Type") { + Text("Response stopped early") + } + + Section("Details") { + SubtitleFormRow(title: "Reason for finishing early", value: finishReason.rawValue) + + if let text = generateContentResponse.text { + SubtitleMarkdownFormRow(title: "Last chunk for the response", value: text) + } + } + + if let ratings = generateContentResponse.candidates.first?.safetyRatings { + SafetyRatingsSection(ratings: ratings) + } + + default: + Section("Error Type") { + Text("Some other error") + } + + Section("Details") { + SubtitleFormRow(title: "Error description", value: error.localizedDescription) + } + } + } + .navigationTitle("Error details") + .navigationBarTitleDisplayMode(.inline) + } + } +} + +#Preview("Response Stopped Early") { + let error = GenerateContentError.responseStoppedEarly( + reason: .maxTokens, + response: GenerateContentResponse(candidates: [ + CandidateResponse(content: ModelContent(role: "model", [ + """ + A _hypothetical_ model response. + Cillum ex aliqua amet aliquip labore amet eiusmod consectetur reprehenderit sit commodo. + """, + ]), + safetyRatings: [ + SafetyRating(category: .dangerousContent, probability: .high), + SafetyRating(category: .harassment, probability: .low), + SafetyRating(category: .hateSpeech, probability: .low), + SafetyRating(category: .sexuallyExplicit, probability: .low), + ], + finishReason: FinishReason.maxTokens, + citationMetadata: nil), + ]) + ) + + return ErrorDetailsView(error: error) +} + +#Preview("Prompt Blocked") { + let error = GenerateContentError.promptBlocked( + response: GenerateContentResponse(candidates: [ + CandidateResponse(content: ModelContent(role: "model", [ + """ + A _hypothetical_ model response. + Cillum ex aliqua amet aliquip labore amet eiusmod consectetur reprehenderit sit commodo. + """, + ]), + safetyRatings: [ + SafetyRating(category: .dangerousContent, probability: .high), + SafetyRating(category: .harassment, probability: .low), + SafetyRating(category: .hateSpeech, probability: .low), + SafetyRating(category: .sexuallyExplicit, probability: .low), + ], + finishReason: FinishReason.other, + citationMetadata: nil), + ]) + ) + + return ErrorDetailsView(error: error) +} diff --git a/FirebaseVertexAI/Sample/ChatSample/Views/ErrorView.swift b/FirebaseVertexAI/Sample/ChatSample/Views/ErrorView.swift new file mode 100644 index 00000000000..d4db2d67dc5 --- /dev/null +++ b/FirebaseVertexAI/Sample/ChatSample/Views/ErrorView.swift @@ -0,0 +1,64 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import FirebaseVertexAI +import SwiftUI + +struct ErrorView: View { + var error: Error + @State private var isDetailsSheetPresented = false + var body: some View { + HStack { + Text("An error occurred.") + Button(action: { isDetailsSheetPresented.toggle() }) { + Image(systemName: "info.circle") + } + } + .frame(maxWidth: .infinity, alignment: .center) + .listRowSeparator(.hidden) + .sheet(isPresented: $isDetailsSheetPresented) { + ErrorDetailsView(error: error) + } + } +} + +#Preview { + NavigationView { + let errorPromptBlocked = GenerateContentError.promptBlocked( + response: GenerateContentResponse(candidates: [ + CandidateResponse(content: ModelContent(role: "model", [ + """ + A _hypothetical_ model response. + Cillum ex aliqua amet aliquip labore amet eiusmod consectetur reprehenderit sit commodo. + """, + ]), + safetyRatings: [ + SafetyRating(category: .dangerousContent, probability: .high), + SafetyRating(category: .harassment, probability: .low), + SafetyRating(category: .hateSpeech, probability: .low), + SafetyRating(category: .sexuallyExplicit, probability: .low), + ], + finishReason: FinishReason.other, + citationMetadata: nil), + ]) + ) + List { + MessageView(message: ChatMessage.samples[0]) + MessageView(message: ChatMessage.samples[1]) + ErrorView(error: errorPromptBlocked) + } + .listStyle(.plain) + .navigationTitle("Chat sample") + } +} diff --git a/FirebaseVertexAI/Sample/ChatSample/Views/MessageView.swift b/FirebaseVertexAI/Sample/ChatSample/Views/MessageView.swift new file mode 100644 index 00000000000..79894503ffd --- /dev/null +++ b/FirebaseVertexAI/Sample/ChatSample/Views/MessageView.swift @@ -0,0 +1,108 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import MarkdownUI +import SwiftUI + +struct RoundedCorner: Shape { + var radius: CGFloat = .infinity + var corners: UIRectCorner = .allCorners + + func path(in rect: CGRect) -> Path { + let path = UIBezierPath( + roundedRect: rect, + byRoundingCorners: corners, + cornerRadii: CGSize(width: radius, height: radius) + ) + return Path(path.cgPath) + } +} + +extension View { + func roundedCorner(_ radius: CGFloat, corners: UIRectCorner) -> some View { + clipShape(RoundedCorner(radius: radius, corners: corners)) + } +} + +struct MessageContentView: View { + var message: ChatMessage + + var body: some View { + if message.pending { + BouncingDots() + } else { + Markdown(message.message) + .markdownTextStyle { + FontFamilyVariant(.normal) + FontSize(.em(0.85)) + ForegroundColor(message.participant == .system ? Color(UIColor.label) : .white) + } + .markdownBlockStyle(\.codeBlock) { configuration in + configuration.label + .relativeLineSpacing(.em(0.25)) + .markdownTextStyle { + FontFamilyVariant(.monospaced) + FontSize(.em(0.85)) + ForegroundColor(Color(.label)) + } + .padding() + .background(Color(.secondarySystemBackground)) + .clipShape(RoundedRectangle(cornerRadius: 8)) + .markdownMargin(top: .zero, bottom: .em(0.8)) + } + } + } +} + +struct MessageView: View { + var message: ChatMessage + + var body: some View { + HStack { + if message.participant == .user { + Spacer() + } + MessageContentView(message: message) + .padding(10) + .background(message.participant == .system + ? Color(UIColor.systemFill) + : Color(UIColor.systemBlue)) + .roundedCorner(10, + corners: [ + .topLeft, + .topRight, + message.participant == .system ? .bottomRight : .bottomLeft, + ]) + if message.participant == .system { + Spacer() + } + } + .listRowSeparator(.hidden) + } +} + +struct MessageView_Previews: PreviewProvider { + static var previews: some View { + NavigationView { + List { + MessageView(message: ChatMessage.samples[0]) + MessageView(message: ChatMessage.samples[1]) + MessageView(message: ChatMessage.samples[2]) + MessageView(message: ChatMessage(message: "Hello!", participant: .system, pending: true)) + } + .listStyle(.plain) + .navigationTitle("Chat sample") + } + } +} diff --git a/FirebaseVertexAI/Sample/FunctionCallingSample/Screens/FunctionCallingScreen.swift b/FirebaseVertexAI/Sample/FunctionCallingSample/Screens/FunctionCallingScreen.swift new file mode 100644 index 00000000000..30d222e29bc --- /dev/null +++ b/FirebaseVertexAI/Sample/FunctionCallingSample/Screens/FunctionCallingScreen.swift @@ -0,0 +1,128 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import FirebaseVertexAI +import GenerativeAIUIComponents +import SwiftUI + +struct FunctionCallingScreen: View { + @EnvironmentObject + var viewModel: FunctionCallingViewModel + + @State + private var userPrompt = "What is 100 Euros in U.S. Dollars?" + + enum FocusedField: Hashable { + case message + } + + @FocusState + var focusedField: FocusedField? + + var body: some View { + VStack { + ScrollViewReader { scrollViewProxy in + List { + Text("Interact with a currency conversion API using function calling in Gemini.") + ForEach(viewModel.messages) { message in + MessageView(message: message) + } + if let error = viewModel.error { + ErrorView(error: error) + .tag("errorView") + } + } + .listStyle(.plain) + .onChange(of: viewModel.messages, perform: { newValue in + if viewModel.hasError { + // Wait for a short moment to make sure we can actually scroll to the bottom. + DispatchQueue.main.asyncAfter(deadline: .now() + 0.05) { + withAnimation { + scrollViewProxy.scrollTo("errorView", anchor: .bottom) + } + focusedField = .message + } + } else { + guard let lastMessage = viewModel.messages.last else { return } + + // Wait for a short moment to make sure we can actually scroll to the bottom. + DispatchQueue.main.asyncAfter(deadline: .now() + 0.05) { + withAnimation { + scrollViewProxy.scrollTo(lastMessage.id, anchor: .bottom) + } + focusedField = .message + } + } + }) + } + InputField("Message...", text: $userPrompt) { + Image(systemName: viewModel.busy ? "stop.circle.fill" : "arrow.up.circle.fill") + .font(.title) + } + .focused($focusedField, equals: .message) + .onSubmit { sendOrStop() } + } + .toolbar { + ToolbarItem(placement: .primaryAction) { + Button(action: newChat) { + Image(systemName: "square.and.pencil") + } + } + } + .navigationTitle("Function Calling") + .onAppear { + focusedField = .message + } + } + + private func sendMessage() { + Task { + let prompt = userPrompt + userPrompt = "" + await viewModel.sendMessage(prompt, streaming: true) + } + } + + private func sendOrStop() { + if viewModel.busy { + viewModel.stop() + } else { + sendMessage() + } + } + + private func newChat() { + viewModel.startNewChat() + } +} + +struct FunctionCallingScreen_Previews: PreviewProvider { + struct ContainerView: View { + @EnvironmentObject + var viewModel: FunctionCallingViewModel + + var body: some View { + FunctionCallingScreen() + .onAppear { + viewModel.messages = ChatMessage.samples + } + } + } + + static var previews: some View { + NavigationStack { + FunctionCallingScreen().environmentObject(FunctionCallingViewModel()) + } + } +} diff --git a/FirebaseVertexAI/Sample/FunctionCallingSample/ViewModels/FunctionCallingViewModel.swift b/FirebaseVertexAI/Sample/FunctionCallingSample/ViewModels/FunctionCallingViewModel.swift new file mode 100644 index 00000000000..43a267e0c09 --- /dev/null +++ b/FirebaseVertexAI/Sample/FunctionCallingSample/ViewModels/FunctionCallingViewModel.swift @@ -0,0 +1,264 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import FirebaseVertexAI +import Foundation +import UIKit + +@MainActor +class FunctionCallingViewModel: ObservableObject { + /// This array holds both the user's and the system's chat messages + @Published var messages = [ChatMessage]() + + /// Indicates we're waiting for the model to finish + @Published var busy = false + + @Published var error: Error? + var hasError: Bool { + return error != nil + } + + /// Function calls pending processing + private var functionCalls = [FunctionCall]() + + private var model: GenerativeModel + private var chat: Chat + + private var chatTask: Task? + + init() { + model = VertexAI.vertexAI().generativeModel( + modelName: "gemini-1.5-pro-preview-0409", + tools: [Tool(functionDeclarations: [ + FunctionDeclaration( + name: "get_exchange_rate", + description: "Get the exchange rate for currencies between countries", + parameters: [ + "currency_from": Schema( + type: .string, + format: "enum", + description: "The currency to convert from in ISO 4217 format", + enumValues: ["USD", "EUR", "JPY", "GBP", "AUD", "CAD"] + ), + "currency_to": Schema( + type: .string, + format: "enum", + description: "The currency to convert to in ISO 4217 format", + enumValues: ["USD", "EUR", "JPY", "GBP", "AUD", "CAD"] + ), + ], + requiredParameters: ["currency_from", "currency_to"] + ), + ])] + ) + chat = model.startChat() + } + + func sendMessage(_ text: String, streaming: Bool = true) async { + error = nil + chatTask?.cancel() + + chatTask = Task { + busy = true + defer { + busy = false + } + + // first, add the user's message to the chat + let userMessage = ChatMessage(message: text, participant: .user) + messages.append(userMessage) + + // add a pending message while we're waiting for a response from the backend + let systemMessage = ChatMessage.pending(participant: .system) + messages.append(systemMessage) + + print(messages) + do { + repeat { + if streaming { + try await internalSendMessageStreaming(text) + } else { + try await internalSendMessage(text) + } + } while !functionCalls.isEmpty + } catch { + self.error = error + print(error.localizedDescription) + messages.removeLast() + } + } + } + + func startNewChat() { + stop() + error = nil + chat = model.startChat() + messages.removeAll() + } + + func stop() { + chatTask?.cancel() + error = nil + } + + private func internalSendMessageStreaming(_ text: String) async throws { + let functionResponses = try await processFunctionCalls() + let responseStream: AsyncThrowingStream + if functionResponses.isEmpty { + responseStream = chat.sendMessageStream(text) + } else { + for functionResponse in functionResponses { + messages.insert(functionResponse.chatMessage(), at: messages.count - 1) + } + responseStream = chat.sendMessageStream(functionResponses.modelContent()) + } + for try await chunk in responseStream { + processResponseContent(content: chunk) + } + } + + private func internalSendMessage(_ text: String) async throws { + let functionResponses = try await processFunctionCalls() + let response: GenerateContentResponse + if functionResponses.isEmpty { + response = try await chat.sendMessage(text) + } else { + for functionResponse in functionResponses { + messages.insert(functionResponse.chatMessage(), at: messages.count - 1) + } + response = try await chat.sendMessage(functionResponses.modelContent()) + } + processResponseContent(content: response) + } + + func processResponseContent(content: GenerateContentResponse) { + guard let candidate = content.candidates.first else { + fatalError("No candidate.") + } + + for part in candidate.content.parts { + switch part { + case let .text(text): + // replace pending message with backend response + messages[messages.count - 1].message += text + messages[messages.count - 1].pending = false + case let .functionCall(functionCall): + messages.insert(functionCall.chatMessage(), at: messages.count - 1) + functionCalls.append(functionCall) + case .data, .fileData, .functionResponse: + fatalError("Unsupported response content.") + } + } + } + + func processFunctionCalls() async throws -> [FunctionResponse] { + var functionResponses = [FunctionResponse]() + for functionCall in functionCalls { + switch functionCall.name { + case "get_exchange_rate": + let exchangeRates = getExchangeRate(args: functionCall.args) + functionResponses.append(FunctionResponse( + name: "get_exchange_rate", + response: exchangeRates + )) + default: + fatalError("Unknown function named \"\(functionCall.name)\".") + } + } + functionCalls = [] + + return functionResponses + } + + // MARK: - Callable Functions + + func getExchangeRate(args: JSONObject) -> JSONObject { + // 1. Validate and extract the parameters provided by the model (from a `FunctionCall`) + guard case let .string(from) = args["currency_from"] else { + fatalError("Missing `currency_from` parameter.") + } + guard case let .string(to) = args["currency_to"] else { + fatalError("Missing `currency_to` parameter.") + } + + // 2. Get the exchange rate + let allRates: [String: [String: Double]] = [ + "AUD": ["CAD": 0.89265, "EUR": 0.6072, "GBP": 0.51714, "JPY": 97.75, "USD": 0.66379], + "CAD": ["AUD": 1.1203, "EUR": 0.68023, "GBP": 0.57933, "JPY": 109.51, "USD": 0.74362], + "EUR": ["AUD": 1.6469, "CAD": 1.4701, "GBP": 0.85168, "JPY": 160.99, "USD": 1.0932], + "GBP": ["AUD": 1.9337, "CAD": 1.7261, "EUR": 1.1741, "JPY": 189.03, "USD": 1.2836], + "JPY": ["AUD": 0.01023, "CAD": 0.00913, "EUR": 0.00621, "GBP": 0.00529, "USD": 0.00679], + "USD": ["AUD": 1.5065, "CAD": 1.3448, "EUR": 0.91475, "GBP": 0.77907, "JPY": 147.26], + ] + guard let fromRates = allRates[from] else { + return ["error": .string("No data for currency \(from).")] + } + guard let toRate = fromRates[to] else { + return ["error": .string("No data for currency \(to).")] + } + + // 3. Return the exchange rates as a JSON object (returned to the model in a `FunctionResponse`) + return ["rates": .number(toRate)] + } +} + +private extension FunctionCall { + func chatMessage() -> ChatMessage { + let encoder = JSONEncoder() + encoder.outputFormatting = .prettyPrinted + + let jsonData: Data + do { + jsonData = try encoder.encode(self) + } catch { + fatalError("JSON Encoding Failed: \(error.localizedDescription)") + } + guard let json = String(data: jsonData, encoding: .utf8) else { + fatalError("Failed to convert JSON data to a String.") + } + let messageText = "Function call requested by model:\n```\n\(json)\n```" + + return ChatMessage(message: messageText, participant: .system) + } +} + +private extension FunctionResponse { + func chatMessage() -> ChatMessage { + let encoder = JSONEncoder() + encoder.outputFormatting = .prettyPrinted + + let jsonData: Data + do { + jsonData = try encoder.encode(self) + } catch { + fatalError("JSON Encoding Failed: \(error.localizedDescription)") + } + guard let json = String(data: jsonData, encoding: .utf8) else { + fatalError("Failed to convert JSON data to a String.") + } + let messageText = "Function response returned by app:\n```\n\(json)\n```" + + return ChatMessage(message: messageText, participant: .user) + } +} + +private extension [FunctionResponse] { + func modelContent() -> [ModelContent] { + return self.map { ModelContent( + role: "function", + parts: [ModelContent.Part.functionResponse($0)] + ) + } + } +} diff --git a/FirebaseVertexAI/Sample/GenerativeAIMultimodalSample/Assets.xcassets/AccentColor.colorset/Contents.json b/FirebaseVertexAI/Sample/GenerativeAIMultimodalSample/Assets.xcassets/AccentColor.colorset/Contents.json new file mode 100644 index 00000000000..eb878970081 --- /dev/null +++ b/FirebaseVertexAI/Sample/GenerativeAIMultimodalSample/Assets.xcassets/AccentColor.colorset/Contents.json @@ -0,0 +1,11 @@ +{ + "colors" : [ + { + "idiom" : "universal" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/FirebaseVertexAI/Sample/GenerativeAIMultimodalSample/Assets.xcassets/AppIcon.appiconset/Contents.json b/FirebaseVertexAI/Sample/GenerativeAIMultimodalSample/Assets.xcassets/AppIcon.appiconset/Contents.json new file mode 100644 index 00000000000..13613e3ee1a --- /dev/null +++ b/FirebaseVertexAI/Sample/GenerativeAIMultimodalSample/Assets.xcassets/AppIcon.appiconset/Contents.json @@ -0,0 +1,13 @@ +{ + "images" : [ + { + "idiom" : "universal", + "platform" : "ios", + "size" : "1024x1024" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/FirebaseVertexAI/Sample/GenerativeAIMultimodalSample/Assets.xcassets/Contents.json b/FirebaseVertexAI/Sample/GenerativeAIMultimodalSample/Assets.xcassets/Contents.json new file mode 100644 index 00000000000..73c00596a7f --- /dev/null +++ b/FirebaseVertexAI/Sample/GenerativeAIMultimodalSample/Assets.xcassets/Contents.json @@ -0,0 +1,6 @@ +{ + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/FirebaseVertexAI/Sample/GenerativeAIMultimodalSample/Preview Content/Preview Assets.xcassets/Contents.json b/FirebaseVertexAI/Sample/GenerativeAIMultimodalSample/Preview Content/Preview Assets.xcassets/Contents.json new file mode 100644 index 00000000000..73c00596a7f --- /dev/null +++ b/FirebaseVertexAI/Sample/GenerativeAIMultimodalSample/Preview Content/Preview Assets.xcassets/Contents.json @@ -0,0 +1,6 @@ +{ + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/FirebaseVertexAI/Sample/GenerativeAIMultimodalSample/Screens/PhotoReasoningScreen.swift b/FirebaseVertexAI/Sample/GenerativeAIMultimodalSample/Screens/PhotoReasoningScreen.swift new file mode 100644 index 00000000000..98f327585db --- /dev/null +++ b/FirebaseVertexAI/Sample/GenerativeAIMultimodalSample/Screens/PhotoReasoningScreen.swift @@ -0,0 +1,65 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import GenerativeAIUIComponents +import MarkdownUI +import PhotosUI +import SwiftUI + +struct PhotoReasoningScreen: View { + @StateObject var viewModel = PhotoReasoningViewModel() + + var body: some View { + VStack { + MultimodalInputField(text: $viewModel.userInput, selection: $viewModel.selectedItems) + .onSubmit { + onSendTapped() + } + + ScrollViewReader { scrollViewProxy in + List { + if let outputText = viewModel.outputText { + HStack(alignment: .top) { + if viewModel.inProgress { + ProgressView() + } else { + Image(systemName: "cloud.circle.fill") + .font(.title2) + } + + Markdown("\(outputText)") + } + .listRowSeparator(.hidden) + } + } + .listStyle(.plain) + } + } + .navigationTitle("Multimodal sample") + } + + // MARK: - Actions + + private func onSendTapped() { + Task { + await viewModel.reason() + } + } +} + +#Preview { + NavigationStack { + PhotoReasoningScreen() + } +} diff --git a/FirebaseVertexAI/Sample/GenerativeAIMultimodalSample/ViewModels/PhotoReasoningViewModel.swift b/FirebaseVertexAI/Sample/GenerativeAIMultimodalSample/ViewModels/PhotoReasoningViewModel.swift new file mode 100644 index 00000000000..3568d6cb335 --- /dev/null +++ b/FirebaseVertexAI/Sample/GenerativeAIMultimodalSample/ViewModels/PhotoReasoningViewModel.swift @@ -0,0 +1,119 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import FirebaseVertexAI +import Foundation +import OSLog +import PhotosUI +import SwiftUI + +@MainActor +class PhotoReasoningViewModel: ObservableObject { + // Maximum value for the larger of the two image dimensions (height and width) in pixels. This is + // being used to reduce the image size in bytes. + private static let largestImageDimension = 768.0 + + private var logger = Logger(subsystem: Bundle.main.bundleIdentifier!, category: "generative-ai") + + @Published + var userInput: String = "" + + @Published + var selectedItems = [PhotosPickerItem]() + + @Published + var outputText: String? = nil + + @Published + var errorMessage: String? + + @Published + var inProgress = false + + private var model: GenerativeModel? + + init() { + model = VertexAI.vertexAI().generativeModel(modelName: "gemini-1.5-pro-preview-0409") + } + + func reason() async { + defer { + inProgress = false + } + guard let model else { + return + } + + do { + inProgress = true + errorMessage = nil + outputText = "" + + let prompt = "Look at the image(s), and then answer the following question: \(userInput)" + + var images = [any ThrowingPartsRepresentable]() + for item in selectedItems { + if let data = try? await item.loadTransferable(type: Data.self) { + guard let image = UIImage(data: data) else { + logger.error("Failed to parse data as an image, skipping.") + continue + } + if image.size.fits(largestDimension: PhotoReasoningViewModel.largestImageDimension) { + images.append(image) + } else { + guard let resizedImage = image + .preparingThumbnail(of: image.size + .aspectFit(largestDimension: PhotoReasoningViewModel.largestImageDimension)) else { + logger.error("Failed to resize image: \(image)") + continue + } + + images.append(resizedImage) + } + } + } + + let outputContentStream = model.generateContentStream(prompt, images) + + // stream response + for try await outputContent in outputContentStream { + guard let line = outputContent.text else { + return + } + + outputText = (outputText ?? "") + line + } + } catch { + logger.error("\(error.localizedDescription)") + errorMessage = error.localizedDescription + } + } +} + +private extension CGSize { + func fits(largestDimension length: CGFloat) -> Bool { + return width <= length && height <= length + } + + func aspectFit(largestDimension length: CGFloat) -> CGSize { + let aspectRatio = width / height + if width > height { + let width = min(self.width, length) + return CGSize(width: width, height: round(width / aspectRatio)) + } else { + let height = min(self.height, length) + return CGSize(width: round(height * aspectRatio), height: height) + } + } +} diff --git a/FirebaseVertexAI/Sample/GenerativeAITextSample/Assets.xcassets/AccentColor.colorset/Contents.json b/FirebaseVertexAI/Sample/GenerativeAITextSample/Assets.xcassets/AccentColor.colorset/Contents.json new file mode 100644 index 00000000000..eb878970081 --- /dev/null +++ b/FirebaseVertexAI/Sample/GenerativeAITextSample/Assets.xcassets/AccentColor.colorset/Contents.json @@ -0,0 +1,11 @@ +{ + "colors" : [ + { + "idiom" : "universal" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/FirebaseVertexAI/Sample/GenerativeAITextSample/Assets.xcassets/AppIcon.appiconset/Contents.json b/FirebaseVertexAI/Sample/GenerativeAITextSample/Assets.xcassets/AppIcon.appiconset/Contents.json new file mode 100644 index 00000000000..13613e3ee1a --- /dev/null +++ b/FirebaseVertexAI/Sample/GenerativeAITextSample/Assets.xcassets/AppIcon.appiconset/Contents.json @@ -0,0 +1,13 @@ +{ + "images" : [ + { + "idiom" : "universal", + "platform" : "ios", + "size" : "1024x1024" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/FirebaseVertexAI/Sample/GenerativeAITextSample/Assets.xcassets/Contents.json b/FirebaseVertexAI/Sample/GenerativeAITextSample/Assets.xcassets/Contents.json new file mode 100644 index 00000000000..73c00596a7f --- /dev/null +++ b/FirebaseVertexAI/Sample/GenerativeAITextSample/Assets.xcassets/Contents.json @@ -0,0 +1,6 @@ +{ + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/FirebaseVertexAI/Sample/GenerativeAITextSample/Preview Content/Preview Assets.xcassets/Contents.json b/FirebaseVertexAI/Sample/GenerativeAITextSample/Preview Content/Preview Assets.xcassets/Contents.json new file mode 100644 index 00000000000..73c00596a7f --- /dev/null +++ b/FirebaseVertexAI/Sample/GenerativeAITextSample/Preview Content/Preview Assets.xcassets/Contents.json @@ -0,0 +1,6 @@ +{ + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/FirebaseVertexAI/Sample/GenerativeAITextSample/Screens/SummarizeScreen.swift b/FirebaseVertexAI/Sample/GenerativeAITextSample/Screens/SummarizeScreen.swift new file mode 100644 index 00000000000..8fbb89f4482 --- /dev/null +++ b/FirebaseVertexAI/Sample/GenerativeAITextSample/Screens/SummarizeScreen.swift @@ -0,0 +1,74 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import MarkdownUI +import SwiftUI + +struct SummarizeScreen: View { + @StateObject var viewModel = SummarizeViewModel() + @State var userInput = "" + + enum FocusedField: Hashable { + case message + } + + @FocusState + var focusedField: FocusedField? + + var body: some View { + VStack { + Text("Enter some text, then tap on _Go_ to summarize it.") + HStack(alignment: .top) { + TextField("Enter text summarize", text: $userInput, axis: .vertical) + .textFieldStyle(.roundedBorder) + .onSubmit { + onSummarizeTapped() + } + Button("Go") { + onSummarizeTapped() + } + .padding(.top, 4) + } + .padding([.horizontal, .bottom]) + + List { + HStack(alignment: .top) { + if viewModel.inProgress { + ProgressView() + } else { + Image(systemName: "cloud.circle.fill") + .font(.title2) + } + + Markdown("\(viewModel.outputText)") + } + .listRowSeparator(.hidden) + } + .listStyle(.plain) + } + .navigationTitle("Text sample") + } + + private func onSummarizeTapped() { + Task { + await viewModel.summarize(inputText: userInput) + } + } +} + +#Preview { + NavigationStack { + SummarizeScreen() + } +} diff --git a/FirebaseVertexAI/Sample/GenerativeAITextSample/ViewModels/SummarizeViewModel.swift b/FirebaseVertexAI/Sample/GenerativeAITextSample/ViewModels/SummarizeViewModel.swift new file mode 100644 index 00000000000..95e18d72114 --- /dev/null +++ b/FirebaseVertexAI/Sample/GenerativeAITextSample/ViewModels/SummarizeViewModel.swift @@ -0,0 +1,68 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import FirebaseVertexAI +import Foundation +import OSLog + +@MainActor +class SummarizeViewModel: ObservableObject { + private var logger = Logger(subsystem: Bundle.main.bundleIdentifier!, category: "generative-ai") + + @Published + var outputText = "" + + @Published + var errorMessage: String? + + @Published + var inProgress = false + + private var model: GenerativeModel? + + init() { + model = VertexAI.vertexAI().generativeModel(modelName: "gemini-1.5-pro-preview-0409") + } + + func summarize(inputText: String) async { + defer { + inProgress = false + } + guard let model else { + return + } + + do { + inProgress = true + errorMessage = nil + outputText = "" + + let prompt = "Summarize the following text for me: \(inputText)" + + let outputContentStream = model.generateContentStream(prompt) + + // stream response + for try await outputContent in outputContentStream { + guard let line = outputContent.text else { + return + } + + outputText = outputText + line + } + } catch { + logger.error("\(error.localizedDescription)") + errorMessage = error.localizedDescription + } + } +} diff --git a/FirebaseVertexAI/Sample/GenerativeAIUIComponents/Package.swift b/FirebaseVertexAI/Sample/GenerativeAIUIComponents/Package.swift new file mode 100644 index 00000000000..808f5f42a97 --- /dev/null +++ b/FirebaseVertexAI/Sample/GenerativeAIUIComponents/Package.swift @@ -0,0 +1,35 @@ +// swift-tools-version: 5.9 +// The swift-tools-version declares the minimum version of Swift required to build this package. + +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +import PackageDescription + +let package = Package( + name: "GenerativeAIUIComponents", + platforms: [ + .iOS(.v16), + ], + products: [ + .library( + name: "GenerativeAIUIComponents", + targets: ["GenerativeAIUIComponents"] + ), + ], + targets: [ + .target( + name: "GenerativeAIUIComponents" + ), + ] +) diff --git a/FirebaseVertexAI/Sample/GenerativeAIUIComponents/Sources/GenerativeAIUIComponents/InputField.swift b/FirebaseVertexAI/Sample/GenerativeAIUIComponents/Sources/GenerativeAIUIComponents/InputField.swift new file mode 100644 index 00000000000..3f12ea65d8a --- /dev/null +++ b/FirebaseVertexAI/Sample/GenerativeAIUIComponents/Sources/GenerativeAIUIComponents/InputField.swift @@ -0,0 +1,83 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import SwiftUI + +public struct InputField