diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index 4e5ba16eb98..ce0a4ac8790 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -79,7 +79,7 @@ jobs: env: plist_secret: ${{ secrets.GHASecretsGPGPassphrase1 }} signin_secret: ${{ secrets.GHASecretsGPGPassphrase1 }} - runs-on: macos-12 # TODO: the legacy ObjC quickstarts don't run with Xcode 15. + runs-on: macos-14 steps: - uses: actions/checkout@v4 - uses: ruby/setup-ruby@v1 @@ -90,8 +90,9 @@ jobs: quickstart-ios/performance/GoogleService-Info.plist "$plist_secret" - name: Test swift quickstart run: ([ -z $plist_secret ] || scripts/third_party/travis/retry.sh scripts/test_quickstart.sh Performance true swift) - - name: Test objc quickstart - run: ([ -z $plist_secret ] || scripts/third_party/travis/retry.sh scripts/test_quickstart.sh Performance true) + # TODO: The legacy ObjC quickstarts don't run with Xcode 15, re-able if we get these working. + # - name: Test objc quickstart + # run: ([ -z $plist_secret ] || scripts/third_party/travis/retry.sh scripts/test_quickstart.sh Performance true) quickstart-ftl-cron-only: if: github.repository == 'Firebase/firebase-ios-sdk' && github.event_name == 'schedule' diff --git a/.github/workflows/vertexai.yml b/.github/workflows/vertexai.yml new file mode 100644 index 00000000000..ebf58529f54 --- /dev/null +++ b/.github/workflows/vertexai.yml @@ -0,0 +1,68 @@ +name: vertexai + +on: + pull_request: + paths: + - 'FirebaseVertexAI**' + - '.github/workflows/vertexai.yml' + - 'Gemfile*' + schedule: + # Run every day at 11pm (PST) - cron uses UTC times + - cron: '0 7 * * *' + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} + cancel-in-progress: true + +jobs: + spm: + strategy: + matrix: + target: [iOS, macOS, catalyst] + os: [macos-13] + include: + - os: macos-13 + xcode: Xcode_15.2 + runs-on: ${{ matrix.os }} + env: + FIREBASECI_USE_LATEST_GOOGLEAPPMEASUREMENT: 1 + steps: + - uses: actions/checkout@v4 + - name: Xcode + run: sudo xcode-select -s /Applications/${{ matrix.xcode }}.app/Contents/Developer + - name: Initialize xcodebuild + run: scripts/setup_spm_tests.sh + - uses: nick-fields/retry@v3 + with: + timeout_minutes: 120 + max_attempts: 3 + retry_on: error + retry_wait_seconds: 120 + command: scripts/build.sh FirebaseVertexAIUnit ${{ matrix.target }} spm + + sample: + strategy: + matrix: + # Test build with debug and release configs (whether or not DEBUG is set and optimization level) + build: [build] + include: + - os: macos-13 + xcode: Xcode_15.0.1 + - os: macos-14 + xcode: Xcode_15.2 + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - name: Xcode + run: sudo xcode-select -s /Applications/${{ matrix.xcode }}.app/Contents/Developer + - name: Initialize xcodebuild + run: xcodebuild -list + - name: Placeholder GoogleService-Info.plist for build testing + run: cp FirebaseCore/Tests/Unit/Resources/GoogleService-Info.plist FirebaseVertexAI/Sample/ + - uses: nick-fields/retry@v3 + with: + timeout_minutes: 120 + max_attempts: 3 + retry_on: error + retry_wait_seconds: 120 + command: scripts/build.sh VertexSample iOS diff --git a/FirebaseCore/Sources/FIRApp.m b/FirebaseCore/Sources/FIRApp.m index 8f4aefe4bc7..12afc134321 100644 --- a/FirebaseCore/Sources/FIRApp.m +++ b/FirebaseCore/Sources/FIRApp.m @@ -829,6 +829,7 @@ + (void)registerSwiftComponents { @"FIRSessions" : @"fire-ses", @"FIRFunctionsComponent" : @"fire-fun", @"FIRStorageComponent" : @"fire-str", + @"FIRVertexAIComponent" : @"fire-vertex", }; for (NSString *className in swiftComponents.allKeys) { Class klass = NSClassFromString(className); diff --git a/FirebaseCore/Sources/FIROptions.m b/FirebaseCore/Sources/FIROptions.m index d46b657501c..c0625a8bac9 100644 --- a/FirebaseCore/Sources/FIROptions.m +++ b/FirebaseCore/Sources/FIROptions.m @@ -284,7 +284,7 @@ - (NSString *)libraryVersionID { // The unit tests are set up to catch anything that does not properly convert. NSString *version = FIRFirebaseVersion(); NSArray *components = [version componentsSeparatedByString:@"."]; - NSString *major = [components objectAtIndex:0]; + NSString *major = [NSString stringWithFormat:@"%02d", [[components objectAtIndex:0] intValue]]; NSString *minor = [NSString stringWithFormat:@"%02d", [[components objectAtIndex:1] intValue]]; NSString *patch = [NSString stringWithFormat:@"%02d", [[components objectAtIndex:2] intValue]]; kFIRLibraryVersionID = [NSString stringWithFormat:@"%@%@%@000", major, minor, patch]; diff --git a/FirebaseCore/Tests/Unit/FIROptionsTest.m b/FirebaseCore/Tests/Unit/FIROptionsTest.m index 3de059f4191..fd5c901598f 100644 --- a/FirebaseCore/Tests/Unit/FIROptionsTest.m +++ b/FirebaseCore/Tests/Unit/FIROptionsTest.m @@ -624,7 +624,7 @@ - (void)testVersionConsistency { int minor = (versionString[2] - '0') * 10 + versionString[3] - '0'; int patch = (versionString[4] - '0') * 10 + versionString[5] - '0'; NSString *str = [NSString stringWithFormat:@"%d.%d.%d", major, minor, patch]; - XCTAssertEqualObjects(str, FIRFirebaseVersion()); + XCTAssertTrue([FIRFirebaseVersion() hasPrefix:str]); } // Repeat test with more Objective-C. @@ -638,11 +638,11 @@ - (void)testVersionConsistency2 { NSRange major = NSMakeRange(0, 2); NSRange minor = NSMakeRange(2, 2); NSRange patch = NSMakeRange(4, 2); - NSString *str = - [NSString stringWithFormat:@"%@.%d.%d", [kFIRLibraryVersionID substringWithRange:major], - [[kFIRLibraryVersionID substringWithRange:minor] intValue], - [[kFIRLibraryVersionID substringWithRange:patch] intValue]]; - XCTAssertEqualObjects(str, FIRFirebaseVersion()); + NSString *str = [NSString + stringWithFormat:@"%d.%d.%d", [[kFIRLibraryVersionID substringWithRange:major] intValue], + [[kFIRLibraryVersionID substringWithRange:minor] intValue], + [[kFIRLibraryVersionID substringWithRange:patch] intValue]]; + XCTAssertTrue([FIRFirebaseVersion() hasPrefix:str]); } #pragma mark - Helpers diff --git a/FirebaseVertexAI/Sample/ChatSample/Assets.xcassets/AccentColor.colorset/Contents.json b/FirebaseVertexAI/Sample/ChatSample/Assets.xcassets/AccentColor.colorset/Contents.json new file mode 100644 index 00000000000..eb878970081 --- /dev/null +++ b/FirebaseVertexAI/Sample/ChatSample/Assets.xcassets/AccentColor.colorset/Contents.json @@ -0,0 +1,11 @@ +{ + "colors" : [ + { + "idiom" : "universal" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/FirebaseVertexAI/Sample/ChatSample/Assets.xcassets/AppIcon.appiconset/Contents.json b/FirebaseVertexAI/Sample/ChatSample/Assets.xcassets/AppIcon.appiconset/Contents.json new file mode 100644 index 00000000000..13613e3ee1a --- /dev/null +++ b/FirebaseVertexAI/Sample/ChatSample/Assets.xcassets/AppIcon.appiconset/Contents.json @@ -0,0 +1,13 @@ +{ + "images" : [ + { + "idiom" : "universal", + "platform" : "ios", + "size" : "1024x1024" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/FirebaseVertexAI/Sample/ChatSample/Assets.xcassets/Contents.json b/FirebaseVertexAI/Sample/ChatSample/Assets.xcassets/Contents.json new file mode 100644 index 00000000000..73c00596a7f --- /dev/null +++ b/FirebaseVertexAI/Sample/ChatSample/Assets.xcassets/Contents.json @@ -0,0 +1,6 @@ +{ + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/FirebaseVertexAI/Sample/ChatSample/Models/ChatMessage.swift b/FirebaseVertexAI/Sample/ChatSample/Models/ChatMessage.swift new file mode 100644 index 00000000000..6f7ab321b12 --- /dev/null +++ b/FirebaseVertexAI/Sample/ChatSample/Models/ChatMessage.swift @@ -0,0 +1,64 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import Foundation + +enum Participant { + case system + case user +} + +struct ChatMessage: Identifiable, Equatable { + let id = UUID().uuidString + var message: String + let participant: Participant + var pending = false + + static func pending(participant: Participant) -> ChatMessage { + Self(message: "", participant: participant, pending: true) + } +} + +extension ChatMessage { + static var samples: [ChatMessage] = [ + .init(message: "Hello. What can I do for you today?", participant: .system), + .init(message: "Show me a simple loop in Swift.", participant: .user), + .init(message: """ + Sure, here is a simple loop in Swift: + + # Example 1 + ``` + for i in 1...5 { + print("Hello, world!") + } + ``` + + This loop will print the string "Hello, world!" five times. The for loop iterates over a range of numbers, + in this case the numbers from 1 to 5. The variable i is assigned each number in the range, and the code inside the loop is executed. + + **Here is another example of a simple loop in Swift:** + ```swift + var sum = 0 + for i in 1...100 { + sum += i + } + print("The sum of the numbers from 1 to 100 is \\(sum).") + ``` + + This loop calculates the sum of the numbers from 1 to 100. The variable sum is initialized to 0, and then the for loop iterates over the range of numbers from 1 to 100. The variable i is assigned each number in the range, and the value of i is added to the sum variable. After the loop has finished executing, the value of sum is printed to the console. + """, participant: .system), + ] + + static var sample = samples[0] +} diff --git a/FirebaseVertexAI/Sample/ChatSample/Preview Content/Preview Assets.xcassets/Contents.json b/FirebaseVertexAI/Sample/ChatSample/Preview Content/Preview Assets.xcassets/Contents.json new file mode 100644 index 00000000000..73c00596a7f --- /dev/null +++ b/FirebaseVertexAI/Sample/ChatSample/Preview Content/Preview Assets.xcassets/Contents.json @@ -0,0 +1,6 @@ +{ + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/FirebaseVertexAI/Sample/ChatSample/Screens/ConversationScreen.swift b/FirebaseVertexAI/Sample/ChatSample/Screens/ConversationScreen.swift new file mode 100644 index 00000000000..a3044a1f430 --- /dev/null +++ b/FirebaseVertexAI/Sample/ChatSample/Screens/ConversationScreen.swift @@ -0,0 +1,127 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import FirebaseVertexAI +import GenerativeAIUIComponents +import SwiftUI + +struct ConversationScreen: View { + @EnvironmentObject + var viewModel: ConversationViewModel + + @State + private var userPrompt = "" + + enum FocusedField: Hashable { + case message + } + + @FocusState + var focusedField: FocusedField? + + var body: some View { + VStack { + ScrollViewReader { scrollViewProxy in + List { + ForEach(viewModel.messages) { message in + MessageView(message: message) + } + if let error = viewModel.error { + ErrorView(error: error) + .tag("errorView") + } + } + .listStyle(.plain) + .onChange(of: viewModel.messages, perform: { newValue in + if viewModel.hasError { + // wait for a short moment to make sure we can actually scroll to the bottom + DispatchQueue.main.asyncAfter(deadline: .now() + 0.05) { + withAnimation { + scrollViewProxy.scrollTo("errorView", anchor: .bottom) + } + focusedField = .message + } + } else { + guard let lastMessage = viewModel.messages.last else { return } + + // wait for a short moment to make sure we can actually scroll to the bottom + DispatchQueue.main.asyncAfter(deadline: .now() + 0.05) { + withAnimation { + scrollViewProxy.scrollTo(lastMessage.id, anchor: .bottom) + } + focusedField = .message + } + } + }) + } + InputField("Message...", text: $userPrompt) { + Image(systemName: viewModel.busy ? "stop.circle.fill" : "arrow.up.circle.fill") + .font(.title) + } + .focused($focusedField, equals: .message) + .onSubmit { sendOrStop() } + } + .toolbar { + ToolbarItem(placement: .primaryAction) { + Button(action: newChat) { + Image(systemName: "square.and.pencil") + } + } + } + .navigationTitle("Chat sample") + .onAppear { + focusedField = .message + } + } + + private func sendMessage() { + Task { + let prompt = userPrompt + userPrompt = "" + await viewModel.sendMessage(prompt, streaming: true) + } + } + + private func sendOrStop() { + if viewModel.busy { + viewModel.stop() + } else { + sendMessage() + } + } + + private func newChat() { + viewModel.startNewChat() + } +} + +struct ConversationScreen_Previews: PreviewProvider { + struct ContainerView: View { + @StateObject var viewModel = ConversationViewModel() + + var body: some View { + ConversationScreen() + .environmentObject(viewModel) + .onAppear { + viewModel.messages = ChatMessage.samples + } + } + } + + static var previews: some View { + NavigationStack { + ConversationScreen() + } + } +} diff --git a/FirebaseVertexAI/Sample/ChatSample/ViewModels/ConversationViewModel.swift b/FirebaseVertexAI/Sample/ChatSample/ViewModels/ConversationViewModel.swift new file mode 100644 index 00000000000..a7c3d21a960 --- /dev/null +++ b/FirebaseVertexAI/Sample/ChatSample/ViewModels/ConversationViewModel.swift @@ -0,0 +1,130 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import FirebaseVertexAI +import Foundation +import UIKit + +@MainActor +class ConversationViewModel: ObservableObject { + /// This array holds both the user's and the system's chat messages + @Published var messages = [ChatMessage]() + + /// Indicates we're waiting for the model to finish + @Published var busy = false + + @Published var error: Error? + var hasError: Bool { + return error != nil + } + + private var model: GenerativeModel + private var chat: Chat + private var stopGenerating = false + + private var chatTask: Task? + + init() { + model = VertexAI.vertexAI().generativeModel(modelName: "gemini-1.0-pro") + chat = model.startChat() + } + + func sendMessage(_ text: String, streaming: Bool = true) async { + error = nil + if streaming { + await internalSendMessageStreaming(text) + } else { + await internalSendMessage(text) + } + } + + func startNewChat() { + stop() + error = nil + chat = model.startChat() + messages.removeAll() + } + + func stop() { + chatTask?.cancel() + error = nil + } + + private func internalSendMessageStreaming(_ text: String) async { + chatTask?.cancel() + + chatTask = Task { + busy = true + defer { + busy = false + } + + // first, add the user's message to the chat + let userMessage = ChatMessage(message: text, participant: .user) + messages.append(userMessage) + + // add a pending message while we're waiting for a response from the backend + let systemMessage = ChatMessage.pending(participant: .system) + messages.append(systemMessage) + + do { + let responseStream = chat.sendMessageStream(text) + for try await chunk in responseStream { + messages[messages.count - 1].pending = false + if let text = chunk.text { + messages[messages.count - 1].message += text + } + } + } catch { + self.error = error + print(error.localizedDescription) + messages.removeLast() + } + } + } + + private func internalSendMessage(_ text: String) async { + chatTask?.cancel() + + chatTask = Task { + busy = true + defer { + busy = false + } + + // first, add the user's message to the chat + let userMessage = ChatMessage(message: text, participant: .user) + messages.append(userMessage) + + // add a pending message while we're waiting for a response from the backend + let systemMessage = ChatMessage.pending(participant: .system) + messages.append(systemMessage) + + do { + var response: GenerateContentResponse? + response = try await chat.sendMessage(text) + + if let responseText = response?.text { + // replace pending message with backend response + messages[messages.count - 1].message = responseText + messages[messages.count - 1].pending = false + } + } catch { + self.error = error + print(error.localizedDescription) + messages.removeLast() + } + } + } +} diff --git a/FirebaseVertexAI/Sample/ChatSample/Views/BouncingDots.swift b/FirebaseVertexAI/Sample/ChatSample/Views/BouncingDots.swift new file mode 100644 index 00000000000..6895e6723da --- /dev/null +++ b/FirebaseVertexAI/Sample/ChatSample/Views/BouncingDots.swift @@ -0,0 +1,77 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import SwiftUI + +struct BouncingDots: View { + @State + private var dot1YOffset: CGFloat = 0.0 + + @State + private var dot2YOffset: CGFloat = 0.0 + + @State + private var dot3YOffset: CGFloat = 0.0 + + let animation = Animation.easeInOut(duration: 0.8) + .repeatForever(autoreverses: true) + + var body: some View { + HStack(spacing: 8) { + Circle() + .fill(Color.white) + .frame(width: 10, height: 10) + .offset(y: dot1YOffset) + .onAppear { + withAnimation(self.animation.delay(0.0)) { + self.dot1YOffset = -5 + } + } + Circle() + .fill(Color.white) + .frame(width: 10, height: 10) + .offset(y: dot2YOffset) + .onAppear { + withAnimation(self.animation.delay(0.2)) { + self.dot2YOffset = -5 + } + } + Circle() + .fill(Color.white) + .frame(width: 10, height: 10) + .offset(y: dot3YOffset) + .onAppear { + withAnimation(self.animation.delay(0.4)) { + self.dot3YOffset = -5 + } + } + } + .onAppear { + let baseOffset: CGFloat = -2 + + self.dot1YOffset = baseOffset + self.dot2YOffset = baseOffset + self.dot3YOffset = baseOffset + } + } +} + +struct BouncingDots_Previews: PreviewProvider { + static var previews: some View { + BouncingDots() + .frame(width: 200, height: 50) + .background(.blue) + .roundedCorner(10, corners: [.allCorners]) + } +} diff --git a/FirebaseVertexAI/Sample/ChatSample/Views/ErrorDetailsView.swift b/FirebaseVertexAI/Sample/ChatSample/Views/ErrorDetailsView.swift new file mode 100644 index 00000000000..dc5ce8f9561 --- /dev/null +++ b/FirebaseVertexAI/Sample/ChatSample/Views/ErrorDetailsView.swift @@ -0,0 +1,206 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import FirebaseVertexAI +import MarkdownUI +import SwiftUI + +extension SafetySetting.HarmCategory: CustomStringConvertible { + public var description: String { + switch self { + case .dangerousContent: "Dangerous content" + case .harassment: "Harassment" + case .hateSpeech: "Hate speech" + case .sexuallyExplicit: "Sexually explicit" + case .unknown: "Unknown" + case .unspecified: "Unspecified" + } + } +} + +extension SafetyRating.HarmProbability: CustomStringConvertible { + public var description: String { + switch self { + case .high: "High" + case .low: "Low" + case .medium: "Medium" + case .negligible: "Negligible" + case .unknown: "Unknown" + case .unspecified: "Unspecified" + } + } +} + +private struct SubtitleFormRow: View { + var title: String + var value: String + + var body: some View { + VStack(alignment: .leading) { + Text(title) + .font(.subheadline) + Text(value) + } + } +} + +private struct SubtitleMarkdownFormRow: View { + var title: String + var value: String + + var body: some View { + VStack(alignment: .leading) { + Text(title) + .font(.subheadline) + Markdown(value) + } + } +} + +private struct SafetyRatingsSection: View { + var ratings: [SafetyRating] + + var body: some View { + Section("Safety ratings") { + List(ratings, id: \.self) { rating in + HStack { + Text("\(String(describing: rating.category))") + .font(.subheadline) + Spacer() + Text("\(String(describing: rating.probability))") + } + } + } + } +} + +struct ErrorDetailsView: View { + var error: Error + + var body: some View { + NavigationView { + Form { + switch error { + case let GenerateContentError.internalError(underlying: underlyingError): + Section("Error Type") { + Text("Internal error") + } + + Section("Details") { + SubtitleFormRow(title: "Error description", + value: underlyingError.localizedDescription) + } + + case let GenerateContentError.promptBlocked(response: generateContentResponse): + Section("Error Type") { + Text("Your prompt was blocked") + } + + Section("Details") { + if let reason = generateContentResponse.promptFeedback?.blockReason { + SubtitleFormRow(title: "Reason for blocking", value: reason.rawValue) + } + + if let text = generateContentResponse.text { + SubtitleMarkdownFormRow(title: "Last chunk for the response", value: text) + } + } + + if let ratings = generateContentResponse.candidates.first?.safetyRatings { + SafetyRatingsSection(ratings: ratings) + } + + case let GenerateContentError.responseStoppedEarly( + reason: finishReason, + response: generateContentResponse + ): + + Section("Error Type") { + Text("Response stopped early") + } + + Section("Details") { + SubtitleFormRow(title: "Reason for finishing early", value: finishReason.rawValue) + + if let text = generateContentResponse.text { + SubtitleMarkdownFormRow(title: "Last chunk for the response", value: text) + } + } + + if let ratings = generateContentResponse.candidates.first?.safetyRatings { + SafetyRatingsSection(ratings: ratings) + } + + default: + Section("Error Type") { + Text("Some other error") + } + + Section("Details") { + SubtitleFormRow(title: "Error description", value: error.localizedDescription) + } + } + } + .navigationTitle("Error details") + .navigationBarTitleDisplayMode(.inline) + } + } +} + +#Preview("Response Stopped Early") { + let error = GenerateContentError.responseStoppedEarly( + reason: .maxTokens, + response: GenerateContentResponse(candidates: [ + CandidateResponse(content: ModelContent(role: "model", [ + """ + A _hypothetical_ model response. + Cillum ex aliqua amet aliquip labore amet eiusmod consectetur reprehenderit sit commodo. + """, + ]), + safetyRatings: [ + SafetyRating(category: .dangerousContent, probability: .high), + SafetyRating(category: .harassment, probability: .low), + SafetyRating(category: .hateSpeech, probability: .low), + SafetyRating(category: .sexuallyExplicit, probability: .low), + ], + finishReason: FinishReason.maxTokens, + citationMetadata: nil), + ]) + ) + + return ErrorDetailsView(error: error) +} + +#Preview("Prompt Blocked") { + let error = GenerateContentError.promptBlocked( + response: GenerateContentResponse(candidates: [ + CandidateResponse(content: ModelContent(role: "model", [ + """ + A _hypothetical_ model response. + Cillum ex aliqua amet aliquip labore amet eiusmod consectetur reprehenderit sit commodo. + """, + ]), + safetyRatings: [ + SafetyRating(category: .dangerousContent, probability: .high), + SafetyRating(category: .harassment, probability: .low), + SafetyRating(category: .hateSpeech, probability: .low), + SafetyRating(category: .sexuallyExplicit, probability: .low), + ], + finishReason: FinishReason.other, + citationMetadata: nil), + ]) + ) + + return ErrorDetailsView(error: error) +} diff --git a/FirebaseVertexAI/Sample/ChatSample/Views/ErrorView.swift b/FirebaseVertexAI/Sample/ChatSample/Views/ErrorView.swift new file mode 100644 index 00000000000..d4db2d67dc5 --- /dev/null +++ b/FirebaseVertexAI/Sample/ChatSample/Views/ErrorView.swift @@ -0,0 +1,64 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import FirebaseVertexAI +import SwiftUI + +struct ErrorView: View { + var error: Error + @State private var isDetailsSheetPresented = false + var body: some View { + HStack { + Text("An error occurred.") + Button(action: { isDetailsSheetPresented.toggle() }) { + Image(systemName: "info.circle") + } + } + .frame(maxWidth: .infinity, alignment: .center) + .listRowSeparator(.hidden) + .sheet(isPresented: $isDetailsSheetPresented) { + ErrorDetailsView(error: error) + } + } +} + +#Preview { + NavigationView { + let errorPromptBlocked = GenerateContentError.promptBlocked( + response: GenerateContentResponse(candidates: [ + CandidateResponse(content: ModelContent(role: "model", [ + """ + A _hypothetical_ model response. + Cillum ex aliqua amet aliquip labore amet eiusmod consectetur reprehenderit sit commodo. + """, + ]), + safetyRatings: [ + SafetyRating(category: .dangerousContent, probability: .high), + SafetyRating(category: .harassment, probability: .low), + SafetyRating(category: .hateSpeech, probability: .low), + SafetyRating(category: .sexuallyExplicit, probability: .low), + ], + finishReason: FinishReason.other, + citationMetadata: nil), + ]) + ) + List { + MessageView(message: ChatMessage.samples[0]) + MessageView(message: ChatMessage.samples[1]) + ErrorView(error: errorPromptBlocked) + } + .listStyle(.plain) + .navigationTitle("Chat sample") + } +} diff --git a/FirebaseVertexAI/Sample/ChatSample/Views/MessageView.swift b/FirebaseVertexAI/Sample/ChatSample/Views/MessageView.swift new file mode 100644 index 00000000000..79894503ffd --- /dev/null +++ b/FirebaseVertexAI/Sample/ChatSample/Views/MessageView.swift @@ -0,0 +1,108 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import MarkdownUI +import SwiftUI + +struct RoundedCorner: Shape { + var radius: CGFloat = .infinity + var corners: UIRectCorner = .allCorners + + func path(in rect: CGRect) -> Path { + let path = UIBezierPath( + roundedRect: rect, + byRoundingCorners: corners, + cornerRadii: CGSize(width: radius, height: radius) + ) + return Path(path.cgPath) + } +} + +extension View { + func roundedCorner(_ radius: CGFloat, corners: UIRectCorner) -> some View { + clipShape(RoundedCorner(radius: radius, corners: corners)) + } +} + +struct MessageContentView: View { + var message: ChatMessage + + var body: some View { + if message.pending { + BouncingDots() + } else { + Markdown(message.message) + .markdownTextStyle { + FontFamilyVariant(.normal) + FontSize(.em(0.85)) + ForegroundColor(message.participant == .system ? Color(UIColor.label) : .white) + } + .markdownBlockStyle(\.codeBlock) { configuration in + configuration.label + .relativeLineSpacing(.em(0.25)) + .markdownTextStyle { + FontFamilyVariant(.monospaced) + FontSize(.em(0.85)) + ForegroundColor(Color(.label)) + } + .padding() + .background(Color(.secondarySystemBackground)) + .clipShape(RoundedRectangle(cornerRadius: 8)) + .markdownMargin(top: .zero, bottom: .em(0.8)) + } + } + } +} + +struct MessageView: View { + var message: ChatMessage + + var body: some View { + HStack { + if message.participant == .user { + Spacer() + } + MessageContentView(message: message) + .padding(10) + .background(message.participant == .system + ? Color(UIColor.systemFill) + : Color(UIColor.systemBlue)) + .roundedCorner(10, + corners: [ + .topLeft, + .topRight, + message.participant == .system ? .bottomRight : .bottomLeft, + ]) + if message.participant == .system { + Spacer() + } + } + .listRowSeparator(.hidden) + } +} + +struct MessageView_Previews: PreviewProvider { + static var previews: some View { + NavigationView { + List { + MessageView(message: ChatMessage.samples[0]) + MessageView(message: ChatMessage.samples[1]) + MessageView(message: ChatMessage.samples[2]) + MessageView(message: ChatMessage(message: "Hello!", participant: .system, pending: true)) + } + .listStyle(.plain) + .navigationTitle("Chat sample") + } + } +} diff --git a/FirebaseVertexAI/Sample/FunctionCallingSample/Screens/FunctionCallingScreen.swift b/FirebaseVertexAI/Sample/FunctionCallingSample/Screens/FunctionCallingScreen.swift new file mode 100644 index 00000000000..30d222e29bc --- /dev/null +++ b/FirebaseVertexAI/Sample/FunctionCallingSample/Screens/FunctionCallingScreen.swift @@ -0,0 +1,128 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import FirebaseVertexAI +import GenerativeAIUIComponents +import SwiftUI + +struct FunctionCallingScreen: View { + @EnvironmentObject + var viewModel: FunctionCallingViewModel + + @State + private var userPrompt = "What is 100 Euros in U.S. Dollars?" + + enum FocusedField: Hashable { + case message + } + + @FocusState + var focusedField: FocusedField? + + var body: some View { + VStack { + ScrollViewReader { scrollViewProxy in + List { + Text("Interact with a currency conversion API using function calling in Gemini.") + ForEach(viewModel.messages) { message in + MessageView(message: message) + } + if let error = viewModel.error { + ErrorView(error: error) + .tag("errorView") + } + } + .listStyle(.plain) + .onChange(of: viewModel.messages, perform: { newValue in + if viewModel.hasError { + // Wait for a short moment to make sure we can actually scroll to the bottom. + DispatchQueue.main.asyncAfter(deadline: .now() + 0.05) { + withAnimation { + scrollViewProxy.scrollTo("errorView", anchor: .bottom) + } + focusedField = .message + } + } else { + guard let lastMessage = viewModel.messages.last else { return } + + // Wait for a short moment to make sure we can actually scroll to the bottom. + DispatchQueue.main.asyncAfter(deadline: .now() + 0.05) { + withAnimation { + scrollViewProxy.scrollTo(lastMessage.id, anchor: .bottom) + } + focusedField = .message + } + } + }) + } + InputField("Message...", text: $userPrompt) { + Image(systemName: viewModel.busy ? "stop.circle.fill" : "arrow.up.circle.fill") + .font(.title) + } + .focused($focusedField, equals: .message) + .onSubmit { sendOrStop() } + } + .toolbar { + ToolbarItem(placement: .primaryAction) { + Button(action: newChat) { + Image(systemName: "square.and.pencil") + } + } + } + .navigationTitle("Function Calling") + .onAppear { + focusedField = .message + } + } + + private func sendMessage() { + Task { + let prompt = userPrompt + userPrompt = "" + await viewModel.sendMessage(prompt, streaming: true) + } + } + + private func sendOrStop() { + if viewModel.busy { + viewModel.stop() + } else { + sendMessage() + } + } + + private func newChat() { + viewModel.startNewChat() + } +} + +struct FunctionCallingScreen_Previews: PreviewProvider { + struct ContainerView: View { + @EnvironmentObject + var viewModel: FunctionCallingViewModel + + var body: some View { + FunctionCallingScreen() + .onAppear { + viewModel.messages = ChatMessage.samples + } + } + } + + static var previews: some View { + NavigationStack { + FunctionCallingScreen().environmentObject(FunctionCallingViewModel()) + } + } +} diff --git a/FirebaseVertexAI/Sample/FunctionCallingSample/ViewModels/FunctionCallingViewModel.swift b/FirebaseVertexAI/Sample/FunctionCallingSample/ViewModels/FunctionCallingViewModel.swift new file mode 100644 index 00000000000..f18c0e85fc1 --- /dev/null +++ b/FirebaseVertexAI/Sample/FunctionCallingSample/ViewModels/FunctionCallingViewModel.swift @@ -0,0 +1,264 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import FirebaseVertexAI +import Foundation +import UIKit + +@MainActor +class FunctionCallingViewModel: ObservableObject { + /// This array holds both the user's and the system's chat messages + @Published var messages = [ChatMessage]() + + /// Indicates we're waiting for the model to finish + @Published var busy = false + + @Published var error: Error? + var hasError: Bool { + return error != nil + } + + /// Function calls pending processing + private var functionCalls = [FunctionCall]() + + private var model: GenerativeModel + private var chat: Chat + + private var chatTask: Task? + + init() { + model = VertexAI.vertexAI().generativeModel( + modelName: "gemini-1.0-pro", + tools: [Tool(functionDeclarations: [ + FunctionDeclaration( + name: "get_exchange_rate", + description: "Get the exchange rate for currencies between countries", + parameters: [ + "currency_from": Schema( + type: .string, + format: "enum", + description: "The currency to convert from in ISO 4217 format", + enumValues: ["USD", "EUR", "JPY", "GBP", "AUD", "CAD"] + ), + "currency_to": Schema( + type: .string, + format: "enum", + description: "The currency to convert to in ISO 4217 format", + enumValues: ["USD", "EUR", "JPY", "GBP", "AUD", "CAD"] + ), + ], + requiredParameters: ["currency_from", "currency_to"] + ), + ])] + ) + chat = model.startChat() + } + + func sendMessage(_ text: String, streaming: Bool = true) async { + error = nil + chatTask?.cancel() + + chatTask = Task { + busy = true + defer { + busy = false + } + + // first, add the user's message to the chat + let userMessage = ChatMessage(message: text, participant: .user) + messages.append(userMessage) + + // add a pending message while we're waiting for a response from the backend + let systemMessage = ChatMessage.pending(participant: .system) + messages.append(systemMessage) + + print(messages) + do { + repeat { + if streaming { + try await internalSendMessageStreaming(text) + } else { + try await internalSendMessage(text) + } + } while !functionCalls.isEmpty + } catch { + self.error = error + print(error.localizedDescription) + messages.removeLast() + } + } + } + + func startNewChat() { + stop() + error = nil + chat = model.startChat() + messages.removeAll() + } + + func stop() { + chatTask?.cancel() + error = nil + } + + private func internalSendMessageStreaming(_ text: String) async throws { + let functionResponses = try await processFunctionCalls() + let responseStream: AsyncThrowingStream + if functionResponses.isEmpty { + responseStream = chat.sendMessageStream(text) + } else { + for functionResponse in functionResponses { + messages.insert(functionResponse.chatMessage(), at: messages.count - 1) + } + responseStream = chat.sendMessageStream(functionResponses.modelContent()) + } + for try await chunk in responseStream { + processResponseContent(content: chunk) + } + } + + private func internalSendMessage(_ text: String) async throws { + let functionResponses = try await processFunctionCalls() + let response: GenerateContentResponse + if functionResponses.isEmpty { + response = try await chat.sendMessage(text) + } else { + for functionResponse in functionResponses { + messages.insert(functionResponse.chatMessage(), at: messages.count - 1) + } + response = try await chat.sendMessage(functionResponses.modelContent()) + } + processResponseContent(content: response) + } + + func processResponseContent(content: GenerateContentResponse) { + guard let candidate = content.candidates.first else { + fatalError("No candidate.") + } + + for part in candidate.content.parts { + switch part { + case let .text(text): + // replace pending message with backend response + messages[messages.count - 1].message += text + messages[messages.count - 1].pending = false + case let .functionCall(functionCall): + messages.insert(functionCall.chatMessage(), at: messages.count - 1) + functionCalls.append(functionCall) + case .data, .functionResponse: + fatalError("Unsupported response content.") + } + } + } + + func processFunctionCalls() async throws -> [FunctionResponse] { + var functionResponses = [FunctionResponse]() + for functionCall in functionCalls { + switch functionCall.name { + case "get_exchange_rate": + let exchangeRates = getExchangeRate(args: functionCall.args) + functionResponses.append(FunctionResponse( + name: "get_exchange_rate", + response: exchangeRates + )) + default: + fatalError("Unknown function named \"\(functionCall.name)\".") + } + } + functionCalls = [] + + return functionResponses + } + + // MARK: - Callable Functions + + func getExchangeRate(args: JSONObject) -> JSONObject { + // 1. Validate and extract the parameters provided by the model (from a `FunctionCall`) + guard case let .string(from) = args["currency_from"] else { + fatalError("Missing `currency_from` parameter.") + } + guard case let .string(to) = args["currency_to"] else { + fatalError("Missing `currency_to` parameter.") + } + + // 2. Get the exchange rate + let allRates: [String: [String: Double]] = [ + "AUD": ["CAD": 0.89265, "EUR": 0.6072, "GBP": 0.51714, "JPY": 97.75, "USD": 0.66379], + "CAD": ["AUD": 1.1203, "EUR": 0.68023, "GBP": 0.57933, "JPY": 109.51, "USD": 0.74362], + "EUR": ["AUD": 1.6469, "CAD": 1.4701, "GBP": 0.85168, "JPY": 160.99, "USD": 1.0932], + "GBP": ["AUD": 1.9337, "CAD": 1.7261, "EUR": 1.1741, "JPY": 189.03, "USD": 1.2836], + "JPY": ["AUD": 0.01023, "CAD": 0.00913, "EUR": 0.00621, "GBP": 0.00529, "USD": 0.00679], + "USD": ["AUD": 1.5065, "CAD": 1.3448, "EUR": 0.91475, "GBP": 0.77907, "JPY": 147.26], + ] + guard let fromRates = allRates[from] else { + return ["error": .string("No data for currency \(from).")] + } + guard let toRate = fromRates[to] else { + return ["error": .string("No data for currency \(to).")] + } + + // 3. Return the exchange rates as a JSON object (returned to the model in a `FunctionResponse`) + return ["rates": .number(toRate)] + } +} + +private extension FunctionCall { + func chatMessage() -> ChatMessage { + let encoder = JSONEncoder() + encoder.outputFormatting = .prettyPrinted + + let jsonData: Data + do { + jsonData = try encoder.encode(self) + } catch { + fatalError("JSON Encoding Failed: \(error.localizedDescription)") + } + guard let json = String(data: jsonData, encoding: .utf8) else { + fatalError("Failed to convert JSON data to a String.") + } + let messageText = "Function call requested by model:\n```\n\(json)\n```" + + return ChatMessage(message: messageText, participant: .system) + } +} + +private extension FunctionResponse { + func chatMessage() -> ChatMessage { + let encoder = JSONEncoder() + encoder.outputFormatting = .prettyPrinted + + let jsonData: Data + do { + jsonData = try encoder.encode(self) + } catch { + fatalError("JSON Encoding Failed: \(error.localizedDescription)") + } + guard let json = String(data: jsonData, encoding: .utf8) else { + fatalError("Failed to convert JSON data to a String.") + } + let messageText = "Function response returned by app:\n```\n\(json)\n```" + + return ChatMessage(message: messageText, participant: .user) + } +} + +private extension [FunctionResponse] { + func modelContent() -> [ModelContent] { + return self.map { ModelContent( + role: "function", + parts: [ModelContent.Part.functionResponse($0)] + ) + } + } +} diff --git a/FirebaseVertexAI/Sample/GenerativeAIMultimodalSample/Assets.xcassets/AccentColor.colorset/Contents.json b/FirebaseVertexAI/Sample/GenerativeAIMultimodalSample/Assets.xcassets/AccentColor.colorset/Contents.json new file mode 100644 index 00000000000..eb878970081 --- /dev/null +++ b/FirebaseVertexAI/Sample/GenerativeAIMultimodalSample/Assets.xcassets/AccentColor.colorset/Contents.json @@ -0,0 +1,11 @@ +{ + "colors" : [ + { + "idiom" : "universal" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/FirebaseVertexAI/Sample/GenerativeAIMultimodalSample/Assets.xcassets/AppIcon.appiconset/Contents.json b/FirebaseVertexAI/Sample/GenerativeAIMultimodalSample/Assets.xcassets/AppIcon.appiconset/Contents.json new file mode 100644 index 00000000000..13613e3ee1a --- /dev/null +++ b/FirebaseVertexAI/Sample/GenerativeAIMultimodalSample/Assets.xcassets/AppIcon.appiconset/Contents.json @@ -0,0 +1,13 @@ +{ + "images" : [ + { + "idiom" : "universal", + "platform" : "ios", + "size" : "1024x1024" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/FirebaseVertexAI/Sample/GenerativeAIMultimodalSample/Assets.xcassets/Contents.json b/FirebaseVertexAI/Sample/GenerativeAIMultimodalSample/Assets.xcassets/Contents.json new file mode 100644 index 00000000000..73c00596a7f --- /dev/null +++ b/FirebaseVertexAI/Sample/GenerativeAIMultimodalSample/Assets.xcassets/Contents.json @@ -0,0 +1,6 @@ +{ + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/FirebaseVertexAI/Sample/GenerativeAIMultimodalSample/Preview Content/Preview Assets.xcassets/Contents.json b/FirebaseVertexAI/Sample/GenerativeAIMultimodalSample/Preview Content/Preview Assets.xcassets/Contents.json new file mode 100644 index 00000000000..73c00596a7f --- /dev/null +++ b/FirebaseVertexAI/Sample/GenerativeAIMultimodalSample/Preview Content/Preview Assets.xcassets/Contents.json @@ -0,0 +1,6 @@ +{ + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/FirebaseVertexAI/Sample/GenerativeAIMultimodalSample/Screens/PhotoReasoningScreen.swift b/FirebaseVertexAI/Sample/GenerativeAIMultimodalSample/Screens/PhotoReasoningScreen.swift new file mode 100644 index 00000000000..98f327585db --- /dev/null +++ b/FirebaseVertexAI/Sample/GenerativeAIMultimodalSample/Screens/PhotoReasoningScreen.swift @@ -0,0 +1,65 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import GenerativeAIUIComponents +import MarkdownUI +import PhotosUI +import SwiftUI + +struct PhotoReasoningScreen: View { + @StateObject var viewModel = PhotoReasoningViewModel() + + var body: some View { + VStack { + MultimodalInputField(text: $viewModel.userInput, selection: $viewModel.selectedItems) + .onSubmit { + onSendTapped() + } + + ScrollViewReader { scrollViewProxy in + List { + if let outputText = viewModel.outputText { + HStack(alignment: .top) { + if viewModel.inProgress { + ProgressView() + } else { + Image(systemName: "cloud.circle.fill") + .font(.title2) + } + + Markdown("\(outputText)") + } + .listRowSeparator(.hidden) + } + } + .listStyle(.plain) + } + } + .navigationTitle("Multimodal sample") + } + + // MARK: - Actions + + private func onSendTapped() { + Task { + await viewModel.reason() + } + } +} + +#Preview { + NavigationStack { + PhotoReasoningScreen() + } +} diff --git a/FirebaseVertexAI/Sample/GenerativeAIMultimodalSample/ViewModels/PhotoReasoningViewModel.swift b/FirebaseVertexAI/Sample/GenerativeAIMultimodalSample/ViewModels/PhotoReasoningViewModel.swift new file mode 100644 index 00000000000..b27d518fe86 --- /dev/null +++ b/FirebaseVertexAI/Sample/GenerativeAIMultimodalSample/ViewModels/PhotoReasoningViewModel.swift @@ -0,0 +1,119 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import FirebaseVertexAI +import Foundation +import OSLog +import PhotosUI +import SwiftUI + +@MainActor +class PhotoReasoningViewModel: ObservableObject { + // Maximum value for the larger of the two image dimensions (height and width) in pixels. This is + // being used to reduce the image size in bytes. + private static let largestImageDimension = 768.0 + + private var logger = Logger(subsystem: Bundle.main.bundleIdentifier!, category: "generative-ai") + + @Published + var userInput: String = "" + + @Published + var selectedItems = [PhotosPickerItem]() + + @Published + var outputText: String? = nil + + @Published + var errorMessage: String? + + @Published + var inProgress = false + + private var model: GenerativeModel? + + init() { + model = VertexAI.vertexAI().generativeModel(modelName: "gemini-1.0-pro-vision") + } + + func reason() async { + defer { + inProgress = false + } + guard let model else { + return + } + + do { + inProgress = true + errorMessage = nil + outputText = "" + + let prompt = "Look at the image(s), and then answer the following question: \(userInput)" + + var images = [any ThrowingPartsRepresentable]() + for item in selectedItems { + if let data = try? await item.loadTransferable(type: Data.self) { + guard let image = UIImage(data: data) else { + logger.error("Failed to parse data as an image, skipping.") + continue + } + if image.size.fits(largestDimension: PhotoReasoningViewModel.largestImageDimension) { + images.append(image) + } else { + guard let resizedImage = image + .preparingThumbnail(of: image.size + .aspectFit(largestDimension: PhotoReasoningViewModel.largestImageDimension)) else { + logger.error("Failed to resize image: \(image)") + continue + } + + images.append(resizedImage) + } + } + } + + let outputContentStream = model.generateContentStream(prompt, images) + + // stream response + for try await outputContent in outputContentStream { + guard let line = outputContent.text else { + return + } + + outputText = (outputText ?? "") + line + } + } catch { + logger.error("\(error.localizedDescription)") + errorMessage = error.localizedDescription + } + } +} + +private extension CGSize { + func fits(largestDimension length: CGFloat) -> Bool { + return width <= length && height <= length + } + + func aspectFit(largestDimension length: CGFloat) -> CGSize { + let aspectRatio = width / height + if width > height { + let width = min(self.width, length) + return CGSize(width: width, height: round(width / aspectRatio)) + } else { + let height = min(self.height, length) + return CGSize(width: round(height * aspectRatio), height: height) + } + } +} diff --git a/FirebaseVertexAI/Sample/GenerativeAITextSample/Assets.xcassets/AccentColor.colorset/Contents.json b/FirebaseVertexAI/Sample/GenerativeAITextSample/Assets.xcassets/AccentColor.colorset/Contents.json new file mode 100644 index 00000000000..eb878970081 --- /dev/null +++ b/FirebaseVertexAI/Sample/GenerativeAITextSample/Assets.xcassets/AccentColor.colorset/Contents.json @@ -0,0 +1,11 @@ +{ + "colors" : [ + { + "idiom" : "universal" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/FirebaseVertexAI/Sample/GenerativeAITextSample/Assets.xcassets/AppIcon.appiconset/Contents.json b/FirebaseVertexAI/Sample/GenerativeAITextSample/Assets.xcassets/AppIcon.appiconset/Contents.json new file mode 100644 index 00000000000..13613e3ee1a --- /dev/null +++ b/FirebaseVertexAI/Sample/GenerativeAITextSample/Assets.xcassets/AppIcon.appiconset/Contents.json @@ -0,0 +1,13 @@ +{ + "images" : [ + { + "idiom" : "universal", + "platform" : "ios", + "size" : "1024x1024" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/FirebaseVertexAI/Sample/GenerativeAITextSample/Assets.xcassets/Contents.json b/FirebaseVertexAI/Sample/GenerativeAITextSample/Assets.xcassets/Contents.json new file mode 100644 index 00000000000..73c00596a7f --- /dev/null +++ b/FirebaseVertexAI/Sample/GenerativeAITextSample/Assets.xcassets/Contents.json @@ -0,0 +1,6 @@ +{ + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/FirebaseVertexAI/Sample/GenerativeAITextSample/Preview Content/Preview Assets.xcassets/Contents.json b/FirebaseVertexAI/Sample/GenerativeAITextSample/Preview Content/Preview Assets.xcassets/Contents.json new file mode 100644 index 00000000000..73c00596a7f --- /dev/null +++ b/FirebaseVertexAI/Sample/GenerativeAITextSample/Preview Content/Preview Assets.xcassets/Contents.json @@ -0,0 +1,6 @@ +{ + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/FirebaseVertexAI/Sample/GenerativeAITextSample/Screens/SummarizeScreen.swift b/FirebaseVertexAI/Sample/GenerativeAITextSample/Screens/SummarizeScreen.swift new file mode 100644 index 00000000000..8fbb89f4482 --- /dev/null +++ b/FirebaseVertexAI/Sample/GenerativeAITextSample/Screens/SummarizeScreen.swift @@ -0,0 +1,74 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import MarkdownUI +import SwiftUI + +struct SummarizeScreen: View { + @StateObject var viewModel = SummarizeViewModel() + @State var userInput = "" + + enum FocusedField: Hashable { + case message + } + + @FocusState + var focusedField: FocusedField? + + var body: some View { + VStack { + Text("Enter some text, then tap on _Go_ to summarize it.") + HStack(alignment: .top) { + TextField("Enter text summarize", text: $userInput, axis: .vertical) + .textFieldStyle(.roundedBorder) + .onSubmit { + onSummarizeTapped() + } + Button("Go") { + onSummarizeTapped() + } + .padding(.top, 4) + } + .padding([.horizontal, .bottom]) + + List { + HStack(alignment: .top) { + if viewModel.inProgress { + ProgressView() + } else { + Image(systemName: "cloud.circle.fill") + .font(.title2) + } + + Markdown("\(viewModel.outputText)") + } + .listRowSeparator(.hidden) + } + .listStyle(.plain) + } + .navigationTitle("Text sample") + } + + private func onSummarizeTapped() { + Task { + await viewModel.summarize(inputText: userInput) + } + } +} + +#Preview { + NavigationStack { + SummarizeScreen() + } +} diff --git a/FirebaseVertexAI/Sample/GenerativeAITextSample/ViewModels/SummarizeViewModel.swift b/FirebaseVertexAI/Sample/GenerativeAITextSample/ViewModels/SummarizeViewModel.swift new file mode 100644 index 00000000000..99b374efefa --- /dev/null +++ b/FirebaseVertexAI/Sample/GenerativeAITextSample/ViewModels/SummarizeViewModel.swift @@ -0,0 +1,68 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import FirebaseVertexAI +import Foundation +import OSLog + +@MainActor +class SummarizeViewModel: ObservableObject { + private var logger = Logger(subsystem: Bundle.main.bundleIdentifier!, category: "generative-ai") + + @Published + var outputText = "" + + @Published + var errorMessage: String? + + @Published + var inProgress = false + + private var model: GenerativeModel? + + init() { + model = VertexAI.vertexAI().generativeModel(modelName: "gemini-1.0-pro") + } + + func summarize(inputText: String) async { + defer { + inProgress = false + } + guard let model else { + return + } + + do { + inProgress = true + errorMessage = nil + outputText = "" + + let prompt = "Summarize the following text for me: \(inputText)" + + let outputContentStream = model.generateContentStream(prompt) + + // stream response + for try await outputContent in outputContentStream { + guard let line = outputContent.text else { + return + } + + outputText = outputText + line + } + } catch { + logger.error("\(error.localizedDescription)") + errorMessage = error.localizedDescription + } + } +} diff --git a/FirebaseVertexAI/Sample/GenerativeAIUIComponents/Package.swift b/FirebaseVertexAI/Sample/GenerativeAIUIComponents/Package.swift new file mode 100644 index 00000000000..808f5f42a97 --- /dev/null +++ b/FirebaseVertexAI/Sample/GenerativeAIUIComponents/Package.swift @@ -0,0 +1,35 @@ +// swift-tools-version: 5.9 +// The swift-tools-version declares the minimum version of Swift required to build this package. + +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +import PackageDescription + +let package = Package( + name: "GenerativeAIUIComponents", + platforms: [ + .iOS(.v16), + ], + products: [ + .library( + name: "GenerativeAIUIComponents", + targets: ["GenerativeAIUIComponents"] + ), + ], + targets: [ + .target( + name: "GenerativeAIUIComponents" + ), + ] +) diff --git a/FirebaseVertexAI/Sample/GenerativeAIUIComponents/Sources/GenerativeAIUIComponents/InputField.swift b/FirebaseVertexAI/Sample/GenerativeAIUIComponents/Sources/GenerativeAIUIComponents/InputField.swift new file mode 100644 index 00000000000..3f12ea65d8a --- /dev/null +++ b/FirebaseVertexAI/Sample/GenerativeAIUIComponents/Sources/GenerativeAIUIComponents/InputField.swift @@ -0,0 +1,83 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import SwiftUI + +public struct InputField