From 2f50a298c0c7b9904344a9231e0276c84c40a060 Mon Sep 17 00:00:00 2001 From: Morgan Chen Date: Tue, 9 Jan 2024 14:53:01 -0800 Subject: [PATCH 01/15] almost nonbreaking change --- .../ViewModels/PhotoReasoningViewModel.swift | 2 +- Sources/GoogleAI/Chat.swift | 4 +- Sources/GoogleAI/GenerativeModel.swift | 6 +- Sources/GoogleAI/ModelContent.swift | 2 +- Sources/GoogleAI/PartsRepresentable.swift | 87 ++++++++++++------- 5 files changed, 65 insertions(+), 36 deletions(-) diff --git a/Examples/GenerativeAISample/GenerativeAIMultimodalSample/ViewModels/PhotoReasoningViewModel.swift b/Examples/GenerativeAISample/GenerativeAIMultimodalSample/ViewModels/PhotoReasoningViewModel.swift index e4613d2..0822175 100644 --- a/Examples/GenerativeAISample/GenerativeAIMultimodalSample/ViewModels/PhotoReasoningViewModel.swift +++ b/Examples/GenerativeAISample/GenerativeAIMultimodalSample/ViewModels/PhotoReasoningViewModel.swift @@ -58,7 +58,7 @@ class PhotoReasoningViewModel: ObservableObject { let prompt = "Look at the image(s), and then answer the following question: \(userInput)" - var images = [PartsRepresentable]() + var images = [any PartsRepresentable]() for item in selectedItems { if let data = try? await item.loadTransferable(type: Data.self) { images.append(ModelContent.Part.png(data)) diff --git a/Sources/GoogleAI/Chat.swift b/Sources/GoogleAI/Chat.swift index 968cd0b..55dcdf6 100644 --- a/Sources/GoogleAI/Chat.swift +++ b/Sources/GoogleAI/Chat.swift @@ -30,7 +30,7 @@ public class Chat { public var history: [ModelContent] /// See ``sendMessage(_:)-3ify5``. - public func sendMessage(_ parts: PartsRepresentable...) async throws -> GenerateContentResponse { + public func sendMessage(_ parts: any PartsRepresentable...) async throws -> GenerateContentResponse { return try await sendMessage([ModelContent(parts: parts)]) } @@ -66,7 +66,7 @@ public class Chat { /// See ``sendMessageStream(_:)-4abs3``. @available(macOS 12.0, *) - public func sendMessageStream(_ parts: PartsRepresentable...) + public func sendMessageStream(_ parts: any PartsRepresentable...) -> AsyncThrowingStream { return sendMessageStream([ModelContent(parts: parts)]) } diff --git a/Sources/GoogleAI/GenerativeModel.swift b/Sources/GoogleAI/GenerativeModel.swift index 49b5796..ea97053 100644 --- a/Sources/GoogleAI/GenerativeModel.swift +++ b/Sources/GoogleAI/GenerativeModel.swift @@ -88,7 +88,7 @@ public final class GenerativeModel { /// for conforming types). /// - Returns: The content generated by the model. /// - Throws: A ``GenerateContentError`` if the request failed. - public func generateContent(_ parts: PartsRepresentable...) + public func generateContent(_ parts: any PartsRepresentable...) async throws -> GenerateContentResponse { return try await generateContent([ModelContent(parts: parts)]) } @@ -139,7 +139,7 @@ public final class GenerativeModel { /// - Returns: A stream wrapping content generated by the model or a ``GenerateContentError`` /// error if an error occurred. @available(macOS 12.0, *) - public func generateContentStream(_ parts: PartsRepresentable...) + public func generateContentStream(_ parts: any PartsRepresentable...) -> AsyncThrowingStream { return generateContentStream([ModelContent(parts: parts)]) } @@ -208,7 +208,7 @@ public final class GenerativeModel { /// - Returns: The results of running the model's tokenizer on the input; contains /// ``CountTokensResponse/totalTokens``. /// - Throws: A ``CountTokensError`` if the tokenization request failed. - public func countTokens(_ parts: PartsRepresentable...) async throws -> CountTokensResponse { + public func countTokens(_ parts: any PartsRepresentable...) async throws -> CountTokensResponse { return try await countTokens([ModelContent(parts: parts)]) } diff --git a/Sources/GoogleAI/ModelContent.swift b/Sources/GoogleAI/ModelContent.swift index 898d10f..981c615 100644 --- a/Sources/GoogleAI/ModelContent.swift +++ b/Sources/GoogleAI/ModelContent.swift @@ -117,7 +117,7 @@ public struct ModelContent: Codable, Equatable { /// Creates a new value from any data interpretable as a ``Part``. See ``PartsRepresentable`` /// for types that can be interpreted as `Part`s. - public init(role: String? = "user", _ parts: PartsRepresentable...) { + public init(role: String? = "user", _ parts: any PartsRepresentable...) { self.init(role: role, parts: parts) } } diff --git a/Sources/GoogleAI/PartsRepresentable.swift b/Sources/GoogleAI/PartsRepresentable.swift index 50b0979..d2d92b9 100644 --- a/Sources/GoogleAI/PartsRepresentable.swift +++ b/Sources/GoogleAI/PartsRepresentable.swift @@ -22,88 +22,118 @@ import UniformTypeIdentifiers private let imageCompressionQuality: CGFloat = 0.8 -/// A protocol describing any data that could be interpreted as model input data. +/// A protocol describing any data that could be serialized to model-interpretable input data, +/// where the serialization process might fail with an error. public protocol PartsRepresentable { - var partsValue: [ModelContent.Part] { get } + associatedtype ErrorType where ErrorType: Error + func toModelContentParts() -> Result<[ModelContent.Part], ErrorType> +} + +public extension PartsRepresentable { + var partsValue: [ModelContent.Part] { + let content = toModelContentParts() + switch content { + case .success(let success): + return success + case .failure(let failure): + Logging.default + .error("Error converting \(type(of: self)) value to model content parts: \(failure)") + return [] + } + } } /// Enables a `String` to be passed in as ``PartsRepresentable``. extension String: PartsRepresentable { - public var partsValue: [ModelContent.Part] { - return [.text(self)] + public typealias ErrorType = Never + + public func toModelContentParts() -> Result<[ModelContent.Part], Never> { + return .success([.text(self)]) } } /// Enables a ``ModelContent.Part`` to be passed in as ``PartsRepresentable``. extension ModelContent.Part: PartsRepresentable { - public var partsValue: [ModelContent.Part] { - return [self] + public typealias ErrorType = Never + public func toModelContentParts() -> Result<[ModelContent.Part], Never> { + return .success([self]) } } /// Enable an `Array` of ``PartsRepresentable`` values to be passed in as a single /// ``PartsRepresentable``. extension [any PartsRepresentable]: PartsRepresentable { - public var partsValue: [ModelContent.Part] { - return flatMap { $0.partsValue } + public typealias ErrorType = Never + public func toModelContentParts() -> Result<[ModelContent.Part], Never> { + return .success(flatMap { $0.partsValue }) } } +/// An enum describing failures that can occur when converting image types to model content data. +/// For some image types like `CIImage`, creating valid model content requires creating a JPEG +/// representation of the image that may not yet exist, which may be computationally expensive. +public enum ImageConversionError: Error { + + /// The underlying image was invalid. The error will be accompanied by the actual image object. + case invalidUnderlyingImage(Any) + + /// A valid image destination could not be constructed. + case couldNotAllocateDestination + + /// JPEG image data conversion failed, accompanied by the original image. + case couldNotConvertToJPEG(Any) +} + #if canImport(UIKit) /// Enables images to be representable as ``PartsRepresentable``. extension UIImage: PartsRepresentable { - public var partsValue: [ModelContent.Part] { + + public func toModelContentParts() -> Result<[ModelContent.Part], ImageConversionError> { guard let data = jpegData(compressionQuality: imageCompressionQuality) else { - Logging.default.error("[GoogleGenerativeAI] Couldn't create JPEG from UIImage.") - return [] + return .failure(.couldNotConvertToJPEG(self)) } - - return [ModelContent.Part.data(mimetype: "image/jpeg", data)] + return .success([ModelContent.Part.data(mimetype: "image/jpeg", data)]) } } #elseif canImport(AppKit) /// Enables images to be representable as ``PartsRepresentable``. extension NSImage: PartsRepresentable { - public var partsValue: [ModelContent.Part] { + public func toModelContentParts() -> Result<[ModelContent.Part], ImageConversionError> { guard let cgImage = cgImage(forProposedRect: nil, context: nil, hints: nil) else { - Logging.default.error("[GoogleGenerativeAI] Couldn't create CGImage from NSImage.") - return [] + return .failure(.invalidUnderlyingImage(self)) } let bmp = NSBitmapImageRep(cgImage: cgImage) guard let data = bmp.representation(using: .jpeg, properties: [.compressionFactor: 0.8]) else { - Logging.default.error("[GoogleGenerativeAI] Couldn't create BMP from CGImage.") - return [] + return .failure(.couldNotConvertToJPEG(bmp)) } - return [ModelContent.Part.data(mimetype: "image/jpeg", data)] + return .success([ModelContent.Part.data(mimetype: "image/jpeg", data)]) } } #endif extension CGImage: PartsRepresentable { - public var partsValue: [ModelContent.Part] { + public func toModelContentParts() -> Result<[ModelContent.Part], ImageConversionError> { let output = NSMutableData() guard let imageDestination = CGImageDestinationCreateWithData( output, UTType.jpeg.identifier as CFString, 1, nil ) else { - Logging.default.error("[GoogleGenerativeAI] Couldn't create JPEG from CGImage.") - return [] + return .failure(.couldNotAllocateDestination) } CGImageDestinationAddImage(imageDestination, self, nil) CGImageDestinationSetProperties(imageDestination, [ kCGImageDestinationLossyCompressionQuality: imageCompressionQuality, ] as CFDictionary) if CGImageDestinationFinalize(imageDestination) { - return [.data(mimetype: "image/jpeg", output as Data)] + return .success([.data(mimetype: "image/jpeg", output as Data)]) } - Logging.default.error("[GoogleGenerativeAI] Couldn't create JPEG from CGImage.") - return [] + return .failure(.couldNotConvertToJPEG(self)) } } extension CIImage: PartsRepresentable { - public var partsValue: [ModelContent.Part] { + public func toModelContentParts() -> Result<[ModelContent.Part], ImageConversionError> { let context = CIContext() let jpegData = (colorSpace ?? CGColorSpace(name: CGColorSpace.sRGB)) .flatMap { @@ -113,9 +143,8 @@ extension CIImage: PartsRepresentable { context.jpegRepresentation(of: self, colorSpace: $0, options: [:]) } if let jpegData = jpegData { - return [.data(mimetype: "image/jpeg", jpegData)] + return .success([.data(mimetype: "image/jpeg", jpegData)]) } - Logging.default.error("[GoogleGenerativeAI] Couldn't create JPEG from CIImage.") - return [] + return .failure(.couldNotConvertToJPEG(self)) } } From 1585a969dee2d86d0395e63eb31d3c25a3567e66 Mon Sep 17 00:00:00 2001 From: Morgan Chen Date: Wed, 10 Jan 2024 14:24:36 -0800 Subject: [PATCH 02/15] style --- Sources/GoogleAI/Chat.swift | 3 ++- Sources/GoogleAI/PartsRepresentable.swift | 8 +++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/Sources/GoogleAI/Chat.swift b/Sources/GoogleAI/Chat.swift index 55dcdf6..51c0fc4 100644 --- a/Sources/GoogleAI/Chat.swift +++ b/Sources/GoogleAI/Chat.swift @@ -30,7 +30,8 @@ public class Chat { public var history: [ModelContent] /// See ``sendMessage(_:)-3ify5``. - public func sendMessage(_ parts: any PartsRepresentable...) async throws -> GenerateContentResponse { + public func sendMessage(_ parts: any PartsRepresentable...) async throws + -> GenerateContentResponse { return try await sendMessage([ModelContent(parts: parts)]) } diff --git a/Sources/GoogleAI/PartsRepresentable.swift b/Sources/GoogleAI/PartsRepresentable.swift index d2d92b9..20b7641 100644 --- a/Sources/GoogleAI/PartsRepresentable.swift +++ b/Sources/GoogleAI/PartsRepresentable.swift @@ -33,11 +33,11 @@ public extension PartsRepresentable { var partsValue: [ModelContent.Part] { let content = toModelContentParts() switch content { - case .success(let success): + case let .success(success): return success - case .failure(let failure): + case let .failure(failure): Logging.default - .error("Error converting \(type(of: self)) value to model content parts: \(failure)") + .error("Error converting \(type(of: self)) value to model content parts: \(failure)") return [] } } @@ -73,7 +73,6 @@ extension [any PartsRepresentable]: PartsRepresentable { /// For some image types like `CIImage`, creating valid model content requires creating a JPEG /// representation of the image that may not yet exist, which may be computationally expensive. public enum ImageConversionError: Error { - /// The underlying image was invalid. The error will be accompanied by the actual image object. case invalidUnderlyingImage(Any) @@ -87,7 +86,6 @@ public enum ImageConversionError: Error { #if canImport(UIKit) /// Enables images to be representable as ``PartsRepresentable``. extension UIImage: PartsRepresentable { - public func toModelContentParts() -> Result<[ModelContent.Part], ImageConversionError> { guard let data = jpegData(compressionQuality: imageCompressionQuality) else { return .failure(.couldNotConvertToJPEG(self)) From b47d4dd14a7b4d929b9cf577e6ed005b2d0e1a85 Mon Sep 17 00:00:00 2001 From: Morgan Chen Date: Fri, 19 Jan 2024 14:12:57 -0800 Subject: [PATCH 03/15] make partsvalue accessible only when error is never --- Sources/GoogleAI/ModelContent.swift | 7 +- .../GoogleAI/PartsRepresentable+Image.swift | 101 +++++++++++++++ Sources/GoogleAI/PartsRepresentable.swift | 118 ++++-------------- 3 files changed, 129 insertions(+), 97 deletions(-) create mode 100644 Sources/GoogleAI/PartsRepresentable+Image.swift diff --git a/Sources/GoogleAI/ModelContent.swift b/Sources/GoogleAI/ModelContent.swift index 981c615..df99efa 100644 --- a/Sources/GoogleAI/ModelContent.swift +++ b/Sources/GoogleAI/ModelContent.swift @@ -106,7 +106,12 @@ public struct ModelContent: Codable, Equatable { /// ``Part``. See ``PartsRepresentable`` for types that can be interpreted as `Part`s. public init(role: String? = "user", parts: some PartsRepresentable) { self.role = role - self.parts = parts.partsValue + do { + try self.parts = parts.tryPartsValue() + } catch { + Logging.default.error("Error creating parts: \(error)") + self.parts = [] + } } /// Creates a new value from a list of ``Part``s. diff --git a/Sources/GoogleAI/PartsRepresentable+Image.swift b/Sources/GoogleAI/PartsRepresentable+Image.swift new file mode 100644 index 0000000..de12291 --- /dev/null +++ b/Sources/GoogleAI/PartsRepresentable+Image.swift @@ -0,0 +1,101 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import UniformTypeIdentifiers +#if canImport(UIKit) + import UIKit // For UIImage extensions. +#elseif canImport(AppKit) + import AppKit // For NSImage extensions. +#endif + +private let imageCompressionQuality: CGFloat = 0.8 + +/// An enum describing failures that can occur when converting image types to model content data. +/// For some image types like `CIImage`, creating valid model content requires creating a JPEG +/// representation of the image that may not yet exist, which may be computationally expensive. +public enum ImageConversionError: Error { + /// The underlying image was invalid. The error will be accompanied by the actual image object. + case invalidUnderlyingImage(CGImage) + + /// A valid image destination could not be allocated. + case couldNotAllocateDestination + + /// JPEG image data conversion failed, accompanied by the original image, which may be an + /// instance of `NSImageRep`, `UIImage`, `CGImage`, or `CIImage`. + case couldNotConvertToJPEG(Any) +} + +#if canImport(UIKit) + /// Enables images to be representable as ``PartsRepresentable``. + extension UIImage: PartsRepresentable { + public func toModelContentParts() -> Result<[ModelContent.Part], ImageConversionError> { + guard let data = jpegData(compressionQuality: imageCompressionQuality) else { + return .failure(.couldNotConvertToJPEG(self)) + } + return .success([ModelContent.Part.data(mimetype: "image/jpeg", data)]) + } + } + +#elseif canImport(AppKit) + /// Enables images to be representable as ``PartsRepresentable``. + extension NSImage: PartsRepresentable { + public func toModelContentParts() -> Result<[ModelContent.Part], ImageConversionError> { + guard let cgImage = cgImage(forProposedRect: nil, context: nil, hints: nil) else { + return .failure(.invalidUnderlyingImage(self)) + } + let bmp = NSBitmapImageRep(cgImage: cgImage) + guard let data = bmp.representation(using: .jpeg, properties: [.compressionFactor: 0.8]) + else { + return .failure(.couldNotConvertToJPEG(bmp)) + } + return .success([ModelContent.Part.data(mimetype: "image/jpeg", data)]) + } + } +#endif + +extension CGImage: PartsRepresentable { + public func toModelContentParts() -> Result<[ModelContent.Part], ImageConversionError> { + let output = NSMutableData() + guard let imageDestination = CGImageDestinationCreateWithData( + output, UTType.jpeg.identifier as CFString, 1, nil + ) else { + return .failure(.couldNotAllocateDestination) + } + CGImageDestinationAddImage(imageDestination, self, nil) + CGImageDestinationSetProperties(imageDestination, [ + kCGImageDestinationLossyCompressionQuality: imageCompressionQuality, + ] as CFDictionary) + if CGImageDestinationFinalize(imageDestination) { + return .success([.data(mimetype: "image/jpeg", output as Data)]) + } + return .failure(.couldNotConvertToJPEG(self)) + } +} + +extension CIImage: PartsRepresentable { + public func toModelContentParts() -> Result<[ModelContent.Part], ImageConversionError> { + let context = CIContext() + let jpegData = (colorSpace ?? CGColorSpace(name: CGColorSpace.sRGB)) + .flatMap { + // The docs specify kCGImageDestinationLossyCompressionQuality as a supported option, but + // Swift's type system does not allow this. + // [kCGImageDestinationLossyCompressionQuality: imageCompressionQuality] + context.jpegRepresentation(of: self, colorSpace: $0, options: [:]) + } + if let jpegData = jpegData { + return .success([.data(mimetype: "image/jpeg", jpegData)]) + } + return .failure(.couldNotConvertToJPEG(self)) + } +} diff --git a/Sources/GoogleAI/PartsRepresentable.swift b/Sources/GoogleAI/PartsRepresentable.swift index 20b7641..dde96cf 100644 --- a/Sources/GoogleAI/PartsRepresentable.swift +++ b/Sources/GoogleAI/PartsRepresentable.swift @@ -13,14 +13,6 @@ // limitations under the License. import Foundation -import UniformTypeIdentifiers -#if canImport(UIKit) - import UIKit // For UIImage extensions. -#elseif canImport(AppKit) - import AppKit // For NSImage extensions. -#endif - -private let imageCompressionQuality: CGFloat = 0.8 /// A protocol describing any data that could be serialized to model-interpretable input data, /// where the serialization process might fail with an error. @@ -30,28 +22,24 @@ public protocol PartsRepresentable { } public extension PartsRepresentable { + + func tryPartsValue() throws -> [ModelContent.Part] { + return try toModelContentParts().get() + } + +} + +public extension PartsRepresentable where ErrorType == Never { + var partsValue: [ModelContent.Part] { let content = toModelContentParts() switch content { case let .success(success): return success - case let .failure(failure): - Logging.default - .error("Error converting \(type(of: self)) value to model content parts: \(failure)") - return [] } } } -/// Enables a `String` to be passed in as ``PartsRepresentable``. -extension String: PartsRepresentable { - public typealias ErrorType = Never - - public func toModelContentParts() -> Result<[ModelContent.Part], Never> { - return .success([.text(self)]) - } -} - /// Enables a ``ModelContent.Part`` to be passed in as ``PartsRepresentable``. extension ModelContent.Part: PartsRepresentable { public typealias ErrorType = Never @@ -63,86 +51,24 @@ extension ModelContent.Part: PartsRepresentable { /// Enable an `Array` of ``PartsRepresentable`` values to be passed in as a single /// ``PartsRepresentable``. extension [any PartsRepresentable]: PartsRepresentable { - public typealias ErrorType = Never - public func toModelContentParts() -> Result<[ModelContent.Part], Never> { - return .success(flatMap { $0.partsValue }) - } -} - -/// An enum describing failures that can occur when converting image types to model content data. -/// For some image types like `CIImage`, creating valid model content requires creating a JPEG -/// representation of the image that may not yet exist, which may be computationally expensive. -public enum ImageConversionError: Error { - /// The underlying image was invalid. The error will be accompanied by the actual image object. - case invalidUnderlyingImage(Any) + public typealias ErrorType = Error - /// A valid image destination could not be constructed. - case couldNotAllocateDestination - - /// JPEG image data conversion failed, accompanied by the original image. - case couldNotConvertToJPEG(Any) -} - -#if canImport(UIKit) - /// Enables images to be representable as ``PartsRepresentable``. - extension UIImage: PartsRepresentable { - public func toModelContentParts() -> Result<[ModelContent.Part], ImageConversionError> { - guard let data = jpegData(compressionQuality: imageCompressionQuality) else { - return .failure(.couldNotConvertToJPEG(self)) - } - return .success([ModelContent.Part.data(mimetype: "image/jpeg", data)]) - } - } - -#elseif canImport(AppKit) - /// Enables images to be representable as ``PartsRepresentable``. - extension NSImage: PartsRepresentable { - public func toModelContentParts() -> Result<[ModelContent.Part], ImageConversionError> { - guard let cgImage = cgImage(forProposedRect: nil, context: nil, hints: nil) else { - return .failure(.invalidUnderlyingImage(self)) - } - let bmp = NSBitmapImageRep(cgImage: cgImage) - guard let data = bmp.representation(using: .jpeg, properties: [.compressionFactor: 0.8]) - else { - return .failure(.couldNotConvertToJPEG(bmp)) + public func toModelContentParts() -> Result<[ModelContent.Part], Error> { + let result = { () throws -> [ModelContent.Part] in + try compactMap { element in + return try element.tryPartsValue() } - return .success([ModelContent.Part.data(mimetype: "image/jpeg", data)]) - } - } -#endif - -extension CGImage: PartsRepresentable { - public func toModelContentParts() -> Result<[ModelContent.Part], ImageConversionError> { - let output = NSMutableData() - guard let imageDestination = CGImageDestinationCreateWithData( - output, UTType.jpeg.identifier as CFString, 1, nil - ) else { - return .failure(.couldNotAllocateDestination) - } - CGImageDestinationAddImage(imageDestination, self, nil) - CGImageDestinationSetProperties(imageDestination, [ - kCGImageDestinationLossyCompressionQuality: imageCompressionQuality, - ] as CFDictionary) - if CGImageDestinationFinalize(imageDestination) { - return .success([.data(mimetype: "image/jpeg", output as Data)]) + .flatMap({ $0 }) } - return .failure(.couldNotConvertToJPEG(self)) + return Result(catching: result) } } -extension CIImage: PartsRepresentable { - public func toModelContentParts() -> Result<[ModelContent.Part], ImageConversionError> { - let context = CIContext() - let jpegData = (colorSpace ?? CGColorSpace(name: CGColorSpace.sRGB)) - .flatMap { - // The docs specify kCGImageDestinationLossyCompressionQuality as a supported option, but - // Swift's type system does not allow this. - // [kCGImageDestinationLossyCompressionQuality: imageCompressionQuality] - context.jpegRepresentation(of: self, colorSpace: $0, options: [:]) - } - if let jpegData = jpegData { - return .success([.data(mimetype: "image/jpeg", jpegData)]) - } - return .failure(.couldNotConvertToJPEG(self)) +/// Enables a `String` to be passed in as ``PartsRepresentable``. +extension String: PartsRepresentable { + public typealias ErrorType = Never + + public func toModelContentParts() -> Result<[ModelContent.Part], Never> { + return .success([.text(self)]) } } From ed2b972cf35f4a1b236e1999227dd713a1814e2c Mon Sep 17 00:00:00 2001 From: Morgan Chen Date: Fri, 19 Jan 2024 14:19:06 -0800 Subject: [PATCH 04/15] fix tests --- Sources/GoogleAI/PartsRepresentable.swift | 7 ++----- Tests/GoogleAITests/PartsRepresentableTests.swift | 8 ++++---- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/Sources/GoogleAI/PartsRepresentable.swift b/Sources/GoogleAI/PartsRepresentable.swift index dde96cf..deaf5a9 100644 --- a/Sources/GoogleAI/PartsRepresentable.swift +++ b/Sources/GoogleAI/PartsRepresentable.swift @@ -22,15 +22,12 @@ public protocol PartsRepresentable { } public extension PartsRepresentable { - func tryPartsValue() throws -> [ModelContent.Part] { return try toModelContentParts().get() } - } public extension PartsRepresentable where ErrorType == Never { - var partsValue: [ModelContent.Part] { let content = toModelContentParts() switch content { @@ -56,9 +53,9 @@ extension [any PartsRepresentable]: PartsRepresentable { public func toModelContentParts() -> Result<[ModelContent.Part], Error> { let result = { () throws -> [ModelContent.Part] in try compactMap { element in - return try element.tryPartsValue() + try element.tryPartsValue() } - .flatMap({ $0 }) + .flatMap { $0 } } return Result(catching: result) } diff --git a/Tests/GoogleAITests/PartsRepresentableTests.swift b/Tests/GoogleAITests/PartsRepresentableTests.swift index d513f08..5feb3f5 100644 --- a/Tests/GoogleAITests/PartsRepresentableTests.swift +++ b/Tests/GoogleAITests/PartsRepresentableTests.swift @@ -38,14 +38,14 @@ final class PartsRepresentableTests: XCTestCase { )! return ctx.makeImage()! } - let modelContent = image.partsValue + let modelContent = try image.tryPartsValue() XCTAssert(modelContent.count > 0, "Expected non-empty model content for CGImage: \(image)") } func testModelContentFromCIImageIsNotEmpty() throws { let image = CIImage(color: CIColor.red) .cropped(to: CGRect(origin: CGPointZero, size: CGSize(width: 16, height: 16))) - let modelContent = image.partsValue + let modelContent = try image.tryPartsValue() XCTAssert(modelContent.count > 0, "Expected non-empty model content for CGImage: \(image)") } @@ -54,7 +54,7 @@ final class PartsRepresentableTests: XCTestCase { let coreImage = CIImage(color: CIColor.red) .cropped(to: CGRect(origin: CGPointZero, size: CGSize(width: 16, height: 16))) let image = UIImage(ciImage: coreImage) - let modelContent = image.partsValue + let modelContent = try image.tryPartsValue() XCTAssert(modelContent.count > 0, "Expected non-empty model content for UIImage: \(image)") } #else @@ -64,7 +64,7 @@ final class PartsRepresentableTests: XCTestCase { let rep = NSCIImageRep(ciImage: coreImage) let image = NSImage(size: rep.size) image.addRepresentation(rep) - let modelContent = image.partsValue + let modelContent = try image.tryPartsValue() XCTAssert(modelContent.count > 0, "Expected non-empty model content for NSImage: \(image)") } #endif From 5088f30f04134434b89b4ed7fbb374a275c2b1c2 Mon Sep 17 00:00:00 2001 From: Morgan Chen Date: Fri, 19 Jan 2024 14:44:35 -0800 Subject: [PATCH 05/15] fix macos --- Sources/GoogleAI/PartsRepresentable+Image.swift | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Sources/GoogleAI/PartsRepresentable+Image.swift b/Sources/GoogleAI/PartsRepresentable+Image.swift index de12291..09a3fac 100644 --- a/Sources/GoogleAI/PartsRepresentable+Image.swift +++ b/Sources/GoogleAI/PartsRepresentable+Image.swift @@ -25,8 +25,10 @@ private let imageCompressionQuality: CGFloat = 0.8 /// For some image types like `CIImage`, creating valid model content requires creating a JPEG /// representation of the image that may not yet exist, which may be computationally expensive. public enum ImageConversionError: Error { - /// The underlying image was invalid. The error will be accompanied by the actual image object. - case invalidUnderlyingImage(CGImage) + #if canImport(AppKit) + /// The image (the receiver of the call `toModelContentParts()`) was invalid. + case invalidUnderlyingImage + #endif /// A valid image destination could not be allocated. case couldNotAllocateDestination @@ -52,7 +54,7 @@ public enum ImageConversionError: Error { extension NSImage: PartsRepresentable { public func toModelContentParts() -> Result<[ModelContent.Part], ImageConversionError> { guard let cgImage = cgImage(forProposedRect: nil, context: nil, hints: nil) else { - return .failure(.invalidUnderlyingImage(self)) + return .failure(.invalidUnderlyingImage) } let bmp = NSBitmapImageRep(cgImage: cgImage) guard let data = bmp.representation(using: .jpeg, properties: [.compressionFactor: 0.8]) From 9913ae5a5f4386e66550860a101913935f0f1c9c Mon Sep 17 00:00:00 2001 From: Morgan Chen Date: Tue, 23 Jan 2024 16:04:13 -0800 Subject: [PATCH 06/15] put errors into api methods --- .../ChatSample/Views/ErrorDetailsView.swift | 4 +- .../ChatSample/Views/ErrorView.swift | 2 +- Sources/GoogleAI/Chat.swift | 22 +++++++++-- Sources/GoogleAI/GenerativeModel.swift | 13 ++++++- Sources/GoogleAI/ModelContent.swift | 13 ++----- Tests/GoogleAITests/ChatTests.swift | 2 +- Tests/GoogleAITests/GoogleAITests.swift | 38 +++++++++---------- 7 files changed, 56 insertions(+), 38 deletions(-) diff --git a/Examples/GenerativeAISample/ChatSample/Views/ErrorDetailsView.swift b/Examples/GenerativeAISample/ChatSample/Views/ErrorDetailsView.swift index 5483f03..7f7e25c 100644 --- a/Examples/GenerativeAISample/ChatSample/Views/ErrorDetailsView.swift +++ b/Examples/GenerativeAISample/ChatSample/Views/ErrorDetailsView.swift @@ -162,7 +162,7 @@ struct ErrorDetailsView: View { NavigationView { let _ = GenerateContentError.promptBlocked( response: GenerateContentResponse(candidates: [ - CandidateResponse(content: ModelContent(role: "model", [ + CandidateResponse(content: try! ModelContent(role: "model", [ """ A _hypothetical_ model response. Cillum ex aliqua amet aliquip labore amet eiusmod consectetur reprehenderit sit commodo. @@ -183,7 +183,7 @@ struct ErrorDetailsView: View { let errorFinishedEarly = GenerateContentError.responseStoppedEarly( reason: .maxTokens, response: GenerateContentResponse(candidates: [ - CandidateResponse(content: ModelContent(role: "model", [ + CandidateResponse(content: try! ModelContent(role: "model", [ """ A _hypothetical_ model response. Cillum ex aliqua amet aliquip labore amet eiusmod consectetur reprehenderit sit commodo. diff --git a/Examples/GenerativeAISample/ChatSample/Views/ErrorView.swift b/Examples/GenerativeAISample/ChatSample/Views/ErrorView.swift index aafdcd2..77abea6 100644 --- a/Examples/GenerativeAISample/ChatSample/Views/ErrorView.swift +++ b/Examples/GenerativeAISample/ChatSample/Views/ErrorView.swift @@ -37,7 +37,7 @@ struct ErrorView: View { NavigationView { let errorPromptBlocked = GenerateContentError.promptBlocked( response: GenerateContentResponse(candidates: [ - CandidateResponse(content: ModelContent(role: "model", [ + CandidateResponse(content: try! ModelContent(role: "model", [ """ A _hypothetical_ model response. Cillum ex aliqua amet aliquip labore amet eiusmod consectetur reprehenderit sit commodo. diff --git a/Sources/GoogleAI/Chat.swift b/Sources/GoogleAI/Chat.swift index 51c0fc4..78ecb1f 100644 --- a/Sources/GoogleAI/Chat.swift +++ b/Sources/GoogleAI/Chat.swift @@ -40,9 +40,14 @@ public class Chat { /// - Parameter content: The new content to send as a single chat message. /// - Returns: The model's response if no error occurred. /// - Throws: A ``GenerateContentError`` if an error occurred. - public func sendMessage(_ content: [ModelContent]) async throws -> GenerateContentResponse { + public func sendMessage(_ content: @autoclosure () throws -> [ModelContent]) async throws -> GenerateContentResponse { // Ensure that the new content has the role set. - let newContent: [ModelContent] = content.map(populateContentRole(_:)) + let newContent: [ModelContent] + do { + newContent = try content().map(populateContentRole(_:)) + } catch(let underlying) { + throw GenerateContentError.internalError(underlying: underlying) + } // Send the history alongside the new message as context. let request = history + newContent @@ -69,7 +74,7 @@ public class Chat { @available(macOS 12.0, *) public func sendMessageStream(_ parts: any PartsRepresentable...) -> AsyncThrowingStream { - return sendMessageStream([ModelContent(parts: parts)]) + return sendMessageStream(try [ModelContent(parts: parts)]) } /// Sends a message using the existing history of this chat as context. If successful, the message @@ -77,8 +82,17 @@ public class Chat { /// - Parameter content: The new content to send as a single chat message. /// - Returns: A stream containing the model's response or an error if an error occurred. @available(macOS 12.0, *) - public func sendMessageStream(_ content: [ModelContent]) + public func sendMessageStream(_ contentClosure: @autoclosure () throws -> [ModelContent]) -> AsyncThrowingStream { + let content: [ModelContent] + do { + content = try contentClosure() + } catch(let underlying) { + return AsyncThrowingStream { continuation in + continuation.finish(throwing: GenerateContentError.internalError(underlying: underlying)) + } + } + return AsyncThrowingStream { continuation in Task { var aggregatedContent: [ModelContent] = [] diff --git a/Sources/GoogleAI/GenerativeModel.swift b/Sources/GoogleAI/GenerativeModel.swift index ea97053..e73811f 100644 --- a/Sources/GoogleAI/GenerativeModel.swift +++ b/Sources/GoogleAI/GenerativeModel.swift @@ -141,7 +141,7 @@ public final class GenerativeModel { @available(macOS 12.0, *) public func generateContentStream(_ parts: any PartsRepresentable...) -> AsyncThrowingStream { - return generateContentStream([ModelContent(parts: parts)]) + return generateContentStream(try [ModelContent(parts: parts)]) } /// Generates new content from input content given to the model as a prompt. @@ -150,8 +150,17 @@ public final class GenerativeModel { /// - Returns: A stream wrapping content generated by the model or a ``GenerateContentError`` /// error if an error occurred. @available(macOS 12.0, *) - public func generateContentStream(_ content: [ModelContent]) + public func generateContentStream(_ contentClosure: @autoclosure () throws -> [ModelContent]) -> AsyncThrowingStream { + let content: [ModelContent] + do { + content = try contentClosure() + } catch(let underlying) { + return AsyncThrowingStream { continuation in + continuation.finish(throwing: GenerateContentError.internalError(underlying: underlying)) + } + } + let generateContentRequest = GenerateContentRequest(model: modelResourceName, contents: content, generationConfig: generationConfig, diff --git a/Sources/GoogleAI/ModelContent.swift b/Sources/GoogleAI/ModelContent.swift index df99efa..05148bf 100644 --- a/Sources/GoogleAI/ModelContent.swift +++ b/Sources/GoogleAI/ModelContent.swift @@ -104,14 +104,9 @@ public struct ModelContent: Codable, Equatable { /// Creates a new value from any data or `Array` of data interpretable as a /// ``Part``. See ``PartsRepresentable`` for types that can be interpreted as `Part`s. - public init(role: String? = "user", parts: some PartsRepresentable) { + public init(role: String? = "user", parts: some PartsRepresentable) throws { self.role = role - do { - try self.parts = parts.tryPartsValue() - } catch { - Logging.default.error("Error creating parts: \(error)") - self.parts = [] - } + try self.parts = parts.tryPartsValue() } /// Creates a new value from a list of ``Part``s. @@ -122,7 +117,7 @@ public struct ModelContent: Codable, Equatable { /// Creates a new value from any data interpretable as a ``Part``. See ``PartsRepresentable`` /// for types that can be interpreted as `Part`s. - public init(role: String? = "user", _ parts: any PartsRepresentable...) { - self.init(role: role, parts: parts) + public init(role: String? = "user", _ parts: any PartsRepresentable...) throws { + try self.init(role: role, parts: parts) } } diff --git a/Tests/GoogleAITests/ChatTests.swift b/Tests/GoogleAITests/ChatTests.swift index 4020d4b..0757327 100644 --- a/Tests/GoogleAITests/ChatTests.swift +++ b/Tests/GoogleAITests/ChatTests.swift @@ -60,7 +60,7 @@ final class ChatTests: XCTestCase { XCTAssertEqual(chat.history[0].parts[0].text, input) let finalText = "1 2 3 4 5 6 7 8 9 10" - let assembledExpectation = ModelContent(role: "model", parts: finalText) + let assembledExpectation = try ModelContent(role: "model", parts: finalText) XCTAssertEqual(chat.history[0].parts[0].text, input) XCTAssertEqual(chat.history[1], assembledExpectation) } diff --git a/Tests/GoogleAITests/GoogleAITests.swift b/Tests/GoogleAITests/GoogleAITests.swift index 6389ddb..be84d46 100644 --- a/Tests/GoogleAITests/GoogleAITests.swift +++ b/Tests/GoogleAITests/GoogleAITests.swift @@ -69,8 +69,8 @@ final class GoogleGenerativeAITests: XCTestCase { .generateContent([str, UIImage(), ModelContent.Part.text(str)]) _ = try await genAI.generateContent(str, UIImage(), "def", UIImage()) _ = try await genAI.generateContent([str, UIImage(), "def", UIImage()]) - _ = try await genAI.generateContent([ModelContent("def", UIImage()), - ModelContent("def", UIImage())]) + _ = try await genAI.generateContent([try ModelContent("def", UIImage()), + try ModelContent("def", UIImage())]) #elseif canImport(AppKit) _ = try await genAI.generateContent(NSImage()) _ = try await genAI.generateContent([NSImage()]) @@ -81,40 +81,40 @@ final class GoogleGenerativeAITests: XCTestCase { // PartsRepresentable combinations. let _ = ModelContent(parts: [.text(str)]) let _ = ModelContent(role: "model", parts: [.text(str)]) - let _ = ModelContent(parts: "Constant String") - let _ = ModelContent(parts: str) - let _ = ModelContent(parts: [str]) + let _ = try ModelContent(parts: "Constant String") + let _ = try ModelContent(parts: str) + let _ = try ModelContent(parts: [str]) // Note: without `as [any PartsRepresentable]` this will fail to compile with "Cannot convert // value of type `[Any]` to expected type `[any PartsRepresentable]`. Not sure if there's a // way we can get it to work. - let _ = ModelContent(parts: [str, ModelContent.Part.data( + let _ = try ModelContent(parts: [str, ModelContent.Part.data( mimetype: "foo", Data() )] as [any PartsRepresentable]) #if canImport(UIKit) - _ = ModelContent(role: "user", parts: UIImage()) - _ = ModelContent(role: "user", parts: [UIImage()]) + _ = try ModelContent(role: "user", parts: UIImage()) + _ = try ModelContent(role: "user", parts: [UIImage()]) // Note: without `as [any PartsRepresentable]` this will fail to compile with "Cannot convert // value of type `[Any]` to expected type `[any PartsRepresentable]`. Not sure if there's a // way we can get it to work. - _ = ModelContent(parts: [str, UIImage()] as [any PartsRepresentable]) + _ = try ModelContent(parts: [str, UIImage()] as [any PartsRepresentable]) // Alternatively, you can explicitly declare the type in a variable and pass it in. let representable2: [any PartsRepresentable] = [str, UIImage()] - _ = ModelContent(parts: representable2) - _ = ModelContent(parts: [str, UIImage(), + _ = try ModelContent(parts: representable2) + _ = try ModelContent(parts: [str, UIImage(), ModelContent.Part.text(str)] as [any PartsRepresentable]) #elseif canImport(AppKit) - _ = ModelContent(role: "user", parts: NSImage()) - _ = ModelContent(role: "user", parts: [NSImage()]) + _ = try ModelContent(role: "user", parts: NSImage()) + _ = try ModelContent(role: "user", parts: [NSImage()]) // Note: without `as [any PartsRepresentable]` this will fail to compile with "Cannot convert // value of type `[Any]` to expected type `[any PartsRepresentable]`. Not sure if there's a // way we can get it to work. - _ = ModelContent(parts: [str, NSImage()] as [any PartsRepresentable]) + _ = try ModelContent(parts: [str, NSImage()] as [any PartsRepresentable]) // Alternatively, you can explicitly declare the type in a variable and pass it in. let representable2: [any PartsRepresentable] = [str, NSImage()] - _ = ModelContent(parts: representable2) + _ = try ModelContent(parts: representable2) _ = - ModelContent(parts: [str, NSImage(), + try ModelContent(parts: [str, NSImage(), ModelContent.Part.text(str)] as [any PartsRepresentable]) #endif @@ -124,14 +124,14 @@ final class GoogleGenerativeAITests: XCTestCase { let _: CountTokensResponse = try await genAI.countTokens("What color is the Sky?", UIImage()) let _: CountTokensResponse = try await genAI.countTokens([ - ModelContent("What color is the Sky?", UIImage()), - ModelContent(UIImage(), "What color is the Sky?", UIImage()), + try ModelContent("What color is the Sky?", UIImage()), + try ModelContent(UIImage(), "What color is the Sky?", UIImage()), ]) #endif // Chat _ = genAI.startChat() - _ = genAI.startChat(history: [ModelContent(parts: "abc")]) + _ = genAI.startChat(history: [try ModelContent(parts: "abc")]) } // Result builder alternative From 4b0a290e002fbe78cbbdd0d7108fbc1be33bc6f4 Mon Sep 17 00:00:00 2001 From: Morgan Chen Date: Tue, 23 Jan 2024 16:04:46 -0800 Subject: [PATCH 07/15] style --- Sources/GoogleAI/Chat.swift | 21 +++++++++++---------- Sources/GoogleAI/GenerativeModel.swift | 16 ++++++++-------- Tests/GoogleAITests/GoogleAITests.swift | 14 +++++++------- 3 files changed, 26 insertions(+), 25 deletions(-) diff --git a/Sources/GoogleAI/Chat.swift b/Sources/GoogleAI/Chat.swift index 78ecb1f..daf5ea0 100644 --- a/Sources/GoogleAI/Chat.swift +++ b/Sources/GoogleAI/Chat.swift @@ -40,12 +40,13 @@ public class Chat { /// - Parameter content: The new content to send as a single chat message. /// - Returns: The model's response if no error occurred. /// - Throws: A ``GenerateContentError`` if an error occurred. - public func sendMessage(_ content: @autoclosure () throws -> [ModelContent]) async throws -> GenerateContentResponse { + public func sendMessage(_ content: @autoclosure () throws -> [ModelContent]) async throws + -> GenerateContentResponse { // Ensure that the new content has the role set. let newContent: [ModelContent] do { newContent = try content().map(populateContentRole(_:)) - } catch(let underlying) { + } catch let underlying { throw GenerateContentError.internalError(underlying: underlying) } @@ -74,7 +75,7 @@ public class Chat { @available(macOS 12.0, *) public func sendMessageStream(_ parts: any PartsRepresentable...) -> AsyncThrowingStream { - return sendMessageStream(try [ModelContent(parts: parts)]) + return try sendMessageStream([ModelContent(parts: parts)]) } /// Sends a message using the existing history of this chat as context. If successful, the message @@ -84,14 +85,14 @@ public class Chat { @available(macOS 12.0, *) public func sendMessageStream(_ contentClosure: @autoclosure () throws -> [ModelContent]) -> AsyncThrowingStream { - let content: [ModelContent] - do { - content = try contentClosure() - } catch(let underlying) { - return AsyncThrowingStream { continuation in - continuation.finish(throwing: GenerateContentError.internalError(underlying: underlying)) - } + let content: [ModelContent] + do { + content = try contentClosure() + } catch let underlying { + return AsyncThrowingStream { continuation in + continuation.finish(throwing: GenerateContentError.internalError(underlying: underlying)) } + } return AsyncThrowingStream { continuation in Task { diff --git a/Sources/GoogleAI/GenerativeModel.swift b/Sources/GoogleAI/GenerativeModel.swift index e73811f..6676a7f 100644 --- a/Sources/GoogleAI/GenerativeModel.swift +++ b/Sources/GoogleAI/GenerativeModel.swift @@ -141,7 +141,7 @@ public final class GenerativeModel { @available(macOS 12.0, *) public func generateContentStream(_ parts: any PartsRepresentable...) -> AsyncThrowingStream { - return generateContentStream(try [ModelContent(parts: parts)]) + return try generateContentStream([ModelContent(parts: parts)]) } /// Generates new content from input content given to the model as a prompt. @@ -152,14 +152,14 @@ public final class GenerativeModel { @available(macOS 12.0, *) public func generateContentStream(_ contentClosure: @autoclosure () throws -> [ModelContent]) -> AsyncThrowingStream { - let content: [ModelContent] - do { - content = try contentClosure() - } catch(let underlying) { - return AsyncThrowingStream { continuation in - continuation.finish(throwing: GenerateContentError.internalError(underlying: underlying)) - } + let content: [ModelContent] + do { + content = try contentClosure() + } catch let underlying { + return AsyncThrowingStream { continuation in + continuation.finish(throwing: GenerateContentError.internalError(underlying: underlying)) } + } let generateContentRequest = GenerateContentRequest(model: modelResourceName, contents: content, diff --git a/Tests/GoogleAITests/GoogleAITests.swift b/Tests/GoogleAITests/GoogleAITests.swift index be84d46..ea8250e 100644 --- a/Tests/GoogleAITests/GoogleAITests.swift +++ b/Tests/GoogleAITests/GoogleAITests.swift @@ -69,8 +69,8 @@ final class GoogleGenerativeAITests: XCTestCase { .generateContent([str, UIImage(), ModelContent.Part.text(str)]) _ = try await genAI.generateContent(str, UIImage(), "def", UIImage()) _ = try await genAI.generateContent([str, UIImage(), "def", UIImage()]) - _ = try await genAI.generateContent([try ModelContent("def", UIImage()), - try ModelContent("def", UIImage())]) + _ = try await genAI.generateContent([ModelContent("def", UIImage()), + ModelContent("def", UIImage())]) #elseif canImport(AppKit) _ = try await genAI.generateContent(NSImage()) _ = try await genAI.generateContent([NSImage()]) @@ -102,7 +102,7 @@ final class GoogleGenerativeAITests: XCTestCase { let representable2: [any PartsRepresentable] = [str, UIImage()] _ = try ModelContent(parts: representable2) _ = try ModelContent(parts: [str, UIImage(), - ModelContent.Part.text(str)] as [any PartsRepresentable]) + ModelContent.Part.text(str)] as [any PartsRepresentable]) #elseif canImport(AppKit) _ = try ModelContent(role: "user", parts: NSImage()) _ = try ModelContent(role: "user", parts: [NSImage()]) @@ -115,7 +115,7 @@ final class GoogleGenerativeAITests: XCTestCase { _ = try ModelContent(parts: representable2) _ = try ModelContent(parts: [str, NSImage(), - ModelContent.Part.text(str)] as [any PartsRepresentable]) + ModelContent.Part.text(str)] as [any PartsRepresentable]) #endif // countTokens API @@ -124,14 +124,14 @@ final class GoogleGenerativeAITests: XCTestCase { let _: CountTokensResponse = try await genAI.countTokens("What color is the Sky?", UIImage()) let _: CountTokensResponse = try await genAI.countTokens([ - try ModelContent("What color is the Sky?", UIImage()), - try ModelContent(UIImage(), "What color is the Sky?", UIImage()), + ModelContent("What color is the Sky?", UIImage()), + ModelContent(UIImage(), "What color is the Sky?", UIImage()), ]) #endif // Chat _ = genAI.startChat() - _ = genAI.startChat(history: [try ModelContent(parts: "abc")]) + _ = try genAI.startChat(history: [ModelContent(parts: "abc")]) } // Result builder alternative From a68afb4e481ce0a20bc1a9f6f757c05a9c784c19 Mon Sep 17 00:00:00 2001 From: Morgan Chen Date: Thu, 25 Jan 2024 15:32:02 -0800 Subject: [PATCH 08/15] remove generic error --- .../GoogleAI/PartsRepresentable+Image.swift | 28 ++++++------ Sources/GoogleAI/PartsRepresentable.swift | 44 +++++-------------- 2 files changed, 24 insertions(+), 48 deletions(-) diff --git a/Sources/GoogleAI/PartsRepresentable+Image.swift b/Sources/GoogleAI/PartsRepresentable+Image.swift index 09a3fac..3fe33a2 100644 --- a/Sources/GoogleAI/PartsRepresentable+Image.swift +++ b/Sources/GoogleAI/PartsRepresentable+Image.swift @@ -41,52 +41,52 @@ public enum ImageConversionError: Error { #if canImport(UIKit) /// Enables images to be representable as ``PartsRepresentable``. extension UIImage: PartsRepresentable { - public func toModelContentParts() -> Result<[ModelContent.Part], ImageConversionError> { + public func tryPartsValue() throws -> [ModelContent.Part] { guard let data = jpegData(compressionQuality: imageCompressionQuality) else { - return .failure(.couldNotConvertToJPEG(self)) + throw ImageConversionError.couldNotConvertToJPEG(self) } - return .success([ModelContent.Part.data(mimetype: "image/jpeg", data)]) + return [ModelContent.Part.data(mimetype: "image/jpeg", data)] } } #elseif canImport(AppKit) /// Enables images to be representable as ``PartsRepresentable``. extension NSImage: PartsRepresentable { - public func toModelContentParts() -> Result<[ModelContent.Part], ImageConversionError> { + public func tryPartsValue() throws -> [ModelContent.Part] { guard let cgImage = cgImage(forProposedRect: nil, context: nil, hints: nil) else { - return .failure(.invalidUnderlyingImage) + throw ImageConversionError.invalidUnderlyingImage } let bmp = NSBitmapImageRep(cgImage: cgImage) guard let data = bmp.representation(using: .jpeg, properties: [.compressionFactor: 0.8]) else { - return .failure(.couldNotConvertToJPEG(bmp)) + throw ImageConversionError.couldNotConvertToJPEG(bmp) } - return .success([ModelContent.Part.data(mimetype: "image/jpeg", data)]) + return [ModelContent.Part.data(mimetype: "image/jpeg", data)] } } #endif extension CGImage: PartsRepresentable { - public func toModelContentParts() -> Result<[ModelContent.Part], ImageConversionError> { + public func tryPartsValue() throws -> [ModelContent.Part] { let output = NSMutableData() guard let imageDestination = CGImageDestinationCreateWithData( output, UTType.jpeg.identifier as CFString, 1, nil ) else { - return .failure(.couldNotAllocateDestination) + throw ImageConversionError.couldNotAllocateDestination } CGImageDestinationAddImage(imageDestination, self, nil) CGImageDestinationSetProperties(imageDestination, [ kCGImageDestinationLossyCompressionQuality: imageCompressionQuality, ] as CFDictionary) if CGImageDestinationFinalize(imageDestination) { - return .success([.data(mimetype: "image/jpeg", output as Data)]) + return [.data(mimetype: "image/jpeg", output as Data)] } - return .failure(.couldNotConvertToJPEG(self)) + throw ImageConversionError.couldNotConvertToJPEG(self) } } extension CIImage: PartsRepresentable { - public func toModelContentParts() -> Result<[ModelContent.Part], ImageConversionError> { + public func tryPartsValue() throws -> [ModelContent.Part] { let context = CIContext() let jpegData = (colorSpace ?? CGColorSpace(name: CGColorSpace.sRGB)) .flatMap { @@ -96,8 +96,8 @@ extension CIImage: PartsRepresentable { context.jpegRepresentation(of: self, colorSpace: $0, options: [:]) } if let jpegData = jpegData { - return .success([.data(mimetype: "image/jpeg", jpegData)]) + return [.data(mimetype: "image/jpeg", jpegData)] } - return .failure(.couldNotConvertToJPEG(self)) + throw ImageConversionError.couldNotConvertToJPEG(self) } } diff --git a/Sources/GoogleAI/PartsRepresentable.swift b/Sources/GoogleAI/PartsRepresentable.swift index deaf5a9..cbdda35 100644 --- a/Sources/GoogleAI/PartsRepresentable.swift +++ b/Sources/GoogleAI/PartsRepresentable.swift @@ -17,55 +17,31 @@ import Foundation /// A protocol describing any data that could be serialized to model-interpretable input data, /// where the serialization process might fail with an error. public protocol PartsRepresentable { - associatedtype ErrorType where ErrorType: Error - func toModelContentParts() -> Result<[ModelContent.Part], ErrorType> -} - -public extension PartsRepresentable { - func tryPartsValue() throws -> [ModelContent.Part] { - return try toModelContentParts().get() - } -} - -public extension PartsRepresentable where ErrorType == Never { - var partsValue: [ModelContent.Part] { - let content = toModelContentParts() - switch content { - case let .success(success): - return success - } - } + func tryPartsValue() throws -> [ModelContent.Part] } /// Enables a ``ModelContent.Part`` to be passed in as ``PartsRepresentable``. extension ModelContent.Part: PartsRepresentable { public typealias ErrorType = Never - public func toModelContentParts() -> Result<[ModelContent.Part], Never> { - return .success([self]) + public func tryPartsValue() throws -> [ModelContent.Part] { + return [self] } } /// Enable an `Array` of ``PartsRepresentable`` values to be passed in as a single /// ``PartsRepresentable``. -extension [any PartsRepresentable]: PartsRepresentable { - public typealias ErrorType = Error - - public func toModelContentParts() -> Result<[ModelContent.Part], Error> { - let result = { () throws -> [ModelContent.Part] in - try compactMap { element in - try element.tryPartsValue() - } - .flatMap { $0 } +extension [PartsRepresentable]: PartsRepresentable { + public func tryPartsValue() throws -> [ModelContent.Part] { + return try compactMap { element in + try element.tryPartsValue() } - return Result(catching: result) + .flatMap { $0 } } } /// Enables a `String` to be passed in as ``PartsRepresentable``. extension String: PartsRepresentable { - public typealias ErrorType = Never - - public func toModelContentParts() -> Result<[ModelContent.Part], Never> { - return .success([.text(self)]) + public func tryPartsValue() throws -> [ModelContent.Part] { + return [.text(self)] } } From 0a8fa14a91a0c25e9ef56ca449857e0b5600afb8 Mon Sep 17 00:00:00 2001 From: Morgan Chen Date: Thu, 25 Jan 2024 16:28:35 -0800 Subject: [PATCH 09/15] add non-erroring protocol so force unwraps arent required --- .../ChatSample/Views/ErrorDetailsView.swift | 4 +- .../ChatSample/Views/ErrorView.swift | 2 +- .../ViewModels/PhotoReasoningViewModel.swift | 2 +- Sources/GoogleAI/Chat.swift | 4 +- Sources/GoogleAI/GenerativeModel.swift | 16 ++++--- Sources/GoogleAI/ModelContent.swift | 27 +++++++++--- .../GoogleAI/PartsRepresentable+Image.swift | 12 +++--- Sources/GoogleAI/PartsRepresentable.swift | 29 +++++++++---- Tests/GoogleAITests/ChatTests.swift | 2 +- Tests/GoogleAITests/GoogleAITests.swift | 43 +++++++++++-------- .../PartsRepresentableTests.swift | 2 +- 11 files changed, 92 insertions(+), 51 deletions(-) diff --git a/Examples/GenerativeAISample/ChatSample/Views/ErrorDetailsView.swift b/Examples/GenerativeAISample/ChatSample/Views/ErrorDetailsView.swift index 7f7e25c..5483f03 100644 --- a/Examples/GenerativeAISample/ChatSample/Views/ErrorDetailsView.swift +++ b/Examples/GenerativeAISample/ChatSample/Views/ErrorDetailsView.swift @@ -162,7 +162,7 @@ struct ErrorDetailsView: View { NavigationView { let _ = GenerateContentError.promptBlocked( response: GenerateContentResponse(candidates: [ - CandidateResponse(content: try! ModelContent(role: "model", [ + CandidateResponse(content: ModelContent(role: "model", [ """ A _hypothetical_ model response. Cillum ex aliqua amet aliquip labore amet eiusmod consectetur reprehenderit sit commodo. @@ -183,7 +183,7 @@ struct ErrorDetailsView: View { let errorFinishedEarly = GenerateContentError.responseStoppedEarly( reason: .maxTokens, response: GenerateContentResponse(candidates: [ - CandidateResponse(content: try! ModelContent(role: "model", [ + CandidateResponse(content: ModelContent(role: "model", [ """ A _hypothetical_ model response. Cillum ex aliqua amet aliquip labore amet eiusmod consectetur reprehenderit sit commodo. diff --git a/Examples/GenerativeAISample/ChatSample/Views/ErrorView.swift b/Examples/GenerativeAISample/ChatSample/Views/ErrorView.swift index 77abea6..aafdcd2 100644 --- a/Examples/GenerativeAISample/ChatSample/Views/ErrorView.swift +++ b/Examples/GenerativeAISample/ChatSample/Views/ErrorView.swift @@ -37,7 +37,7 @@ struct ErrorView: View { NavigationView { let errorPromptBlocked = GenerateContentError.promptBlocked( response: GenerateContentResponse(candidates: [ - CandidateResponse(content: try! ModelContent(role: "model", [ + CandidateResponse(content: ModelContent(role: "model", [ """ A _hypothetical_ model response. Cillum ex aliqua amet aliquip labore amet eiusmod consectetur reprehenderit sit commodo. diff --git a/Examples/GenerativeAISample/GenerativeAIMultimodalSample/ViewModels/PhotoReasoningViewModel.swift b/Examples/GenerativeAISample/GenerativeAIMultimodalSample/ViewModels/PhotoReasoningViewModel.swift index 0822175..d434557 100644 --- a/Examples/GenerativeAISample/GenerativeAIMultimodalSample/ViewModels/PhotoReasoningViewModel.swift +++ b/Examples/GenerativeAISample/GenerativeAIMultimodalSample/ViewModels/PhotoReasoningViewModel.swift @@ -58,7 +58,7 @@ class PhotoReasoningViewModel: ObservableObject { let prompt = "Look at the image(s), and then answer the following question: \(userInput)" - var images = [any PartsRepresentable]() + var images = [any ThrowingPartsRepresentable]() for item in selectedItems { if let data = try? await item.loadTransferable(type: Data.self) { images.append(ModelContent.Part.png(data)) diff --git a/Sources/GoogleAI/Chat.swift b/Sources/GoogleAI/Chat.swift index daf5ea0..1fc3a67 100644 --- a/Sources/GoogleAI/Chat.swift +++ b/Sources/GoogleAI/Chat.swift @@ -30,7 +30,7 @@ public class Chat { public var history: [ModelContent] /// See ``sendMessage(_:)-3ify5``. - public func sendMessage(_ parts: any PartsRepresentable...) async throws + public func sendMessage(_ parts: any ThrowingPartsRepresentable...) async throws -> GenerateContentResponse { return try await sendMessage([ModelContent(parts: parts)]) } @@ -73,7 +73,7 @@ public class Chat { /// See ``sendMessageStream(_:)-4abs3``. @available(macOS 12.0, *) - public func sendMessageStream(_ parts: any PartsRepresentable...) + public func sendMessageStream(_ parts: any ThrowingPartsRepresentable...) -> AsyncThrowingStream { return try sendMessageStream([ModelContent(parts: parts)]) } diff --git a/Sources/GoogleAI/GenerativeModel.swift b/Sources/GoogleAI/GenerativeModel.swift index 6676a7f..d2c25bf 100644 --- a/Sources/GoogleAI/GenerativeModel.swift +++ b/Sources/GoogleAI/GenerativeModel.swift @@ -84,11 +84,12 @@ public final class GenerativeModel { /// [few-shot](https://developers.google.com/machine-learning/glossary/generative#few-shot-prompting) /// prompts, see ``generateContent(_:)-58rm0``. /// - /// - Parameter content: The input(s) given to the model as a prompt (see ``PartsRepresentable`` + /// - Parameter content: The input(s) given to the model as a prompt (see + /// ``ThrowingPartsRepresentable`` /// for conforming types). /// - Returns: The content generated by the model. /// - Throws: A ``GenerateContentError`` if the request failed. - public func generateContent(_ parts: any PartsRepresentable...) + public func generateContent(_ parts: any ThrowingPartsRepresentable...) async throws -> GenerateContentResponse { return try await generateContent([ModelContent(parts: parts)]) } @@ -134,12 +135,13 @@ public final class GenerativeModel { /// [few-shot](https://developers.google.com/machine-learning/glossary/generative#few-shot-prompting) /// prompts, see ``generateContent(_:)-58rm0``. /// - /// - Parameter content: The input(s) given to the model as a prompt (see ``PartsRepresentable`` + /// - Parameter content: The input(s) given to the model as a prompt (see + /// ``ThrowingPartsRepresentable`` /// for conforming types). /// - Returns: A stream wrapping content generated by the model or a ``GenerateContentError`` /// error if an error occurred. @available(macOS 12.0, *) - public func generateContentStream(_ parts: any PartsRepresentable...) + public func generateContentStream(_ parts: any ThrowingPartsRepresentable...) -> AsyncThrowingStream { return try generateContentStream([ModelContent(parts: parts)]) } @@ -212,12 +214,14 @@ public final class GenerativeModel { /// [few-shot](https://developers.google.com/machine-learning/glossary/generative#few-shot-prompting) /// input, see ``countTokens(_:)-9spwl``. /// - /// - Parameter content: The input(s) given to the model as a prompt (see ``PartsRepresentable`` + /// - Parameter content: The input(s) given to the model as a prompt (see + /// ``ThrowingPartsRepresentable`` /// for conforming types). /// - Returns: The results of running the model's tokenizer on the input; contains /// ``CountTokensResponse/totalTokens``. /// - Throws: A ``CountTokensError`` if the tokenization request failed. - public func countTokens(_ parts: any PartsRepresentable...) async throws -> CountTokensResponse { + public func countTokens(_ parts: any ThrowingPartsRepresentable...) async throws + -> CountTokensResponse { return try await countTokens([ModelContent(parts: parts)]) } diff --git a/Sources/GoogleAI/ModelContent.swift b/Sources/GoogleAI/ModelContent.swift index 05148bf..ba4960e 100644 --- a/Sources/GoogleAI/ModelContent.swift +++ b/Sources/GoogleAI/ModelContent.swift @@ -103,21 +103,38 @@ public struct ModelContent: Codable, Equatable { public let parts: [Part] /// Creates a new value from any data or `Array` of data interpretable as a - /// ``Part``. See ``PartsRepresentable`` for types that can be interpreted as `Part`s. - public init(role: String? = "user", parts: some PartsRepresentable) throws { + /// ``Part``. See ``ThrowingPartsRepresentable`` for types that can be interpreted as `Part`s. + public init(role: String? = "user", parts: some ThrowingPartsRepresentable) throws { self.role = role try self.parts = parts.tryPartsValue() } + /// Creates a new value from any data or `Array` of data interpretable as a + /// ``Part``. See ``ThrowingPartsRepresentable`` for types that can be interpreted as `Part`s. + public init(role: String? = "user", parts: some PartsRepresentable) { + self.role = role + self.parts = parts.toPartsValue() + } + /// Creates a new value from a list of ``Part``s. public init(role: String? = "user", parts: [Part]) { self.role = role self.parts = parts } - /// Creates a new value from any data interpretable as a ``Part``. See ``PartsRepresentable`` + /// Creates a new value from any data interpretable as a ``Part``. See + /// ``ThrowingPartsRepresentable`` + /// for types that can be interpreted as `Part`s. + public init(role: String? = "user", _ parts: any ThrowingPartsRepresentable...) throws { + let content = try parts.flatMap { try $0.tryPartsValue() } + self.init(role: role, parts: content) + } + + /// Creates a new value from any data interpretable as a ``Part``. See + /// ``ThrowingPartsRepresentable`` /// for types that can be interpreted as `Part`s. - public init(role: String? = "user", _ parts: any PartsRepresentable...) throws { - try self.init(role: role, parts: parts) + public init(role: String? = "user", _ parts: [PartsRepresentable]) { + let content = parts.flatMap { $0.toPartsValue() } + self.init(role: role, parts: content) } } diff --git a/Sources/GoogleAI/PartsRepresentable+Image.swift b/Sources/GoogleAI/PartsRepresentable+Image.swift index 3fe33a2..f9c699c 100644 --- a/Sources/GoogleAI/PartsRepresentable+Image.swift +++ b/Sources/GoogleAI/PartsRepresentable+Image.swift @@ -39,8 +39,8 @@ public enum ImageConversionError: Error { } #if canImport(UIKit) - /// Enables images to be representable as ``PartsRepresentable``. - extension UIImage: PartsRepresentable { + /// Enables images to be representable as ``ThrowingPartsRepresentable``. + extension UIImage: ThrowingPartsRepresentable { public func tryPartsValue() throws -> [ModelContent.Part] { guard let data = jpegData(compressionQuality: imageCompressionQuality) else { throw ImageConversionError.couldNotConvertToJPEG(self) @@ -50,8 +50,8 @@ public enum ImageConversionError: Error { } #elseif canImport(AppKit) - /// Enables images to be representable as ``PartsRepresentable``. - extension NSImage: PartsRepresentable { + /// Enables images to be representable as ``ThrowingPartsRepresentable``. + extension NSImage: ThrowingPartsRepresentable { public func tryPartsValue() throws -> [ModelContent.Part] { guard let cgImage = cgImage(forProposedRect: nil, context: nil, hints: nil) else { throw ImageConversionError.invalidUnderlyingImage @@ -66,7 +66,7 @@ public enum ImageConversionError: Error { } #endif -extension CGImage: PartsRepresentable { +extension CGImage: ThrowingPartsRepresentable { public func tryPartsValue() throws -> [ModelContent.Part] { let output = NSMutableData() guard let imageDestination = CGImageDestinationCreateWithData( @@ -85,7 +85,7 @@ extension CGImage: PartsRepresentable { } } -extension CIImage: PartsRepresentable { +extension CIImage: ThrowingPartsRepresentable { public func tryPartsValue() throws -> [ModelContent.Part] { let context = CIContext() let jpegData = (colorSpace ?? CGColorSpace(name: CGColorSpace.sRGB)) diff --git a/Sources/GoogleAI/PartsRepresentable.swift b/Sources/GoogleAI/PartsRepresentable.swift index cbdda35..a5afd80 100644 --- a/Sources/GoogleAI/PartsRepresentable.swift +++ b/Sources/GoogleAI/PartsRepresentable.swift @@ -16,21 +16,34 @@ import Foundation /// A protocol describing any data that could be serialized to model-interpretable input data, /// where the serialization process might fail with an error. -public protocol PartsRepresentable { +public protocol ThrowingPartsRepresentable { func tryPartsValue() throws -> [ModelContent.Part] } -/// Enables a ``ModelContent.Part`` to be passed in as ``PartsRepresentable``. -extension ModelContent.Part: PartsRepresentable { +/// A protocol describing any data that could be serialized to model-interpretable input data, +/// where the serialization process cannot fail with an error. For a failable conversion, see +/// ``ThrowingPartsRepresentable`` +public protocol PartsRepresentable: ThrowingPartsRepresentable { + func toPartsValue() -> [ModelContent.Part] +} + +public extension PartsRepresentable { + func tryPartsValue() throws -> [ModelContent.Part] { + return toPartsValue() + } +} + +/// Enables a ``ModelContent.Part`` to be passed in as ``ThrowingPartsRepresentable``. +extension ModelContent.Part: ThrowingPartsRepresentable { public typealias ErrorType = Never public func tryPartsValue() throws -> [ModelContent.Part] { return [self] } } -/// Enable an `Array` of ``PartsRepresentable`` values to be passed in as a single -/// ``PartsRepresentable``. -extension [PartsRepresentable]: PartsRepresentable { +/// Enable an `Array` of ``ThrowingPartsRepresentable`` values to be passed in as a single +/// ``ThrowingPartsRepresentable``. +extension [ThrowingPartsRepresentable]: ThrowingPartsRepresentable { public func tryPartsValue() throws -> [ModelContent.Part] { return try compactMap { element in try element.tryPartsValue() @@ -39,9 +52,9 @@ extension [PartsRepresentable]: PartsRepresentable { } } -/// Enables a `String` to be passed in as ``PartsRepresentable``. +/// Enables a `String` to be passed in as ``ThrowingPartsRepresentable``. extension String: PartsRepresentable { - public func tryPartsValue() throws -> [ModelContent.Part] { + public func toPartsValue() -> [ModelContent.Part] { return [.text(self)] } } diff --git a/Tests/GoogleAITests/ChatTests.swift b/Tests/GoogleAITests/ChatTests.swift index 0757327..4020d4b 100644 --- a/Tests/GoogleAITests/ChatTests.swift +++ b/Tests/GoogleAITests/ChatTests.swift @@ -60,7 +60,7 @@ final class ChatTests: XCTestCase { XCTAssertEqual(chat.history[0].parts[0].text, input) let finalText = "1 2 3 4 5 6 7 8 9 10" - let assembledExpectation = try ModelContent(role: "model", parts: finalText) + let assembledExpectation = ModelContent(role: "model", parts: finalText) XCTAssertEqual(chat.history[0].parts[0].text, input) XCTAssertEqual(chat.history[1], assembledExpectation) } diff --git a/Tests/GoogleAITests/GoogleAITests.swift b/Tests/GoogleAITests/GoogleAITests.swift index ea8250e..f063be3 100644 --- a/Tests/GoogleAITests/GoogleAITests.swift +++ b/Tests/GoogleAITests/GoogleAITests.swift @@ -78,44 +78,51 @@ final class GoogleGenerativeAITests: XCTestCase { _ = try await genAI.generateContent([str, NSImage(), "def", NSImage()]) #endif - // PartsRepresentable combinations. + // ThrowingPartsRepresentable combinations. let _ = ModelContent(parts: [.text(str)]) let _ = ModelContent(role: "model", parts: [.text(str)]) - let _ = try ModelContent(parts: "Constant String") - let _ = try ModelContent(parts: str) + let _ = ModelContent(parts: "Constant String") + let _ = ModelContent(parts: str) + // Note: This requires the `try` for some reason. Casting to explicit [PartsRepresentable] also + // doesn't work. let _ = try ModelContent(parts: [str]) - // Note: without `as [any PartsRepresentable]` this will fail to compile with "Cannot convert - // value of type `[Any]` to expected type `[any PartsRepresentable]`. Not sure if there's a - // way we can get it to work. + // Note: without `as [any ThrowingPartsRepresentable]` this will fail to compile with "Cannot + // convert value of type 'String' to expected element type + // 'Array.ArrayLiteralElement'. Not sure if there's a way we can get it to + // work. let _ = try ModelContent(parts: [str, ModelContent.Part.data( mimetype: "foo", Data() - )] as [any PartsRepresentable]) + )] as [any ThrowingPartsRepresentable]) #if canImport(UIKit) _ = try ModelContent(role: "user", parts: UIImage()) _ = try ModelContent(role: "user", parts: [UIImage()]) - // Note: without `as [any PartsRepresentable]` this will fail to compile with "Cannot convert - // value of type `[Any]` to expected type `[any PartsRepresentable]`. Not sure if there's a + // Note: without `as [any ThrowingPartsRepresentable]` this will fail to compile with "Cannot + // convert + // value of type `[Any]` to expected type `[any ThrowingPartsRepresentable]`. Not sure if + // there's a // way we can get it to work. - _ = try ModelContent(parts: [str, UIImage()] as [any PartsRepresentable]) + _ = try ModelContent(parts: [str, UIImage()] as [any ThrowingPartsRepresentable]) // Alternatively, you can explicitly declare the type in a variable and pass it in. - let representable2: [any PartsRepresentable] = [str, UIImage()] + let representable2: [any ThrowingPartsRepresentable] = [str, UIImage()] _ = try ModelContent(parts: representable2) _ = try ModelContent(parts: [str, UIImage(), - ModelContent.Part.text(str)] as [any PartsRepresentable]) + ModelContent.Part.text(str)] as [any ThrowingPartsRepresentable]) #elseif canImport(AppKit) _ = try ModelContent(role: "user", parts: NSImage()) _ = try ModelContent(role: "user", parts: [NSImage()]) - // Note: without `as [any PartsRepresentable]` this will fail to compile with "Cannot convert - // value of type `[Any]` to expected type `[any PartsRepresentable]`. Not sure if there's a + // Note: without `as [any ThrowingPartsRepresentable]` this will fail to compile with "Cannot + // convert + // value of type `[Any]` to expected type `[any ThrowingPartsRepresentable]`. Not sure if + // there's a // way we can get it to work. - _ = try ModelContent(parts: [str, NSImage()] as [any PartsRepresentable]) + _ = try ModelContent(parts: [str, NSImage()] as [any ThrowingPartsRepresentable]) // Alternatively, you can explicitly declare the type in a variable and pass it in. - let representable2: [any PartsRepresentable] = [str, NSImage()] + let representable2: [any ThrowingPartsRepresentable] = [str, NSImage()] _ = try ModelContent(parts: representable2) _ = try ModelContent(parts: [str, NSImage(), - ModelContent.Part.text(str)] as [any PartsRepresentable]) + ModelContent.Part.text(str)] as [any ThrowingPartsRepresentable]) #endif // countTokens API @@ -131,7 +138,7 @@ final class GoogleGenerativeAITests: XCTestCase { // Chat _ = genAI.startChat() - _ = try genAI.startChat(history: [ModelContent(parts: "abc")]) + _ = genAI.startChat(history: [ModelContent(parts: "abc")]) } // Result builder alternative diff --git a/Tests/GoogleAITests/PartsRepresentableTests.swift b/Tests/GoogleAITests/PartsRepresentableTests.swift index 5feb3f5..c0424d8 100644 --- a/Tests/GoogleAITests/PartsRepresentableTests.swift +++ b/Tests/GoogleAITests/PartsRepresentableTests.swift @@ -21,7 +21,7 @@ import XCTest import AppKit #endif -final class PartsRepresentableTests: XCTestCase { +final class ThrowingPartsRepresentableTests: XCTestCase { func testModelContentFromCGImageIsNotEmpty() throws { // adapted from https://forums.swift.org/t/creating-a-cgimage-from-color-array/18634/2 var srgbArray = [UInt32](repeating: 0xFFFF_FFFF, count: 8 * 8) From 5412c57a7329ba5c3068b48865ed21e8d6a78b9f Mon Sep 17 00:00:00 2001 From: Morgan Chen Date: Wed, 31 Jan 2024 15:31:09 -0800 Subject: [PATCH 10/15] api review feedback: use more specific error case and add failure tests --- Sources/GoogleAI/Chat.swift | 14 +++- Sources/GoogleAI/GenerateContentError.swift | 3 + Sources/GoogleAI/GenerativeModel.swift | 8 ++- .../GoogleAI/PartsRepresentable+Image.swift | 6 +- .../PartsRepresentableTests.swift | 66 +++++++++++++++++++ 5 files changed, 90 insertions(+), 7 deletions(-) diff --git a/Sources/GoogleAI/Chat.swift b/Sources/GoogleAI/Chat.swift index 1fc3a67..46ffe12 100644 --- a/Sources/GoogleAI/Chat.swift +++ b/Sources/GoogleAI/Chat.swift @@ -47,7 +47,11 @@ public class Chat { do { newContent = try content().map(populateContentRole(_:)) } catch let underlying { - throw GenerateContentError.internalError(underlying: underlying) + if let contentError = underlying as? ImageConversionError { + throw GenerateContentError.promptContentError(underlying: contentError) + } else { + throw GenerateContentError.internalError(underlying: underlying) + } } // Send the history alongside the new message as context. @@ -90,7 +94,13 @@ public class Chat { content = try contentClosure() } catch let underlying { return AsyncThrowingStream { continuation in - continuation.finish(throwing: GenerateContentError.internalError(underlying: underlying)) + let error: Error + if let contentError = underlying as? ImageConversionError { + error = GenerateContentError.promptContentError(underlying: contentError) + } else { + error = GenerateContentError.internalError(underlying: underlying) + } + continuation.finish(throwing: error) } } diff --git a/Sources/GoogleAI/GenerateContentError.swift b/Sources/GoogleAI/GenerateContentError.swift index 38e6b92..75ddff8 100644 --- a/Sources/GoogleAI/GenerateContentError.swift +++ b/Sources/GoogleAI/GenerateContentError.swift @@ -16,6 +16,9 @@ import Foundation /// Errors that occur when generating content from a model. public enum GenerateContentError: Error { + /// An error occurred when constructing the prompt. Examine the related error for details. + case promptContentError(underlying: ImageConversionError) + /// An internal error occurred. See the underlying error for more context. case internalError(underlying: Error) diff --git a/Sources/GoogleAI/GenerativeModel.swift b/Sources/GoogleAI/GenerativeModel.swift index d2c25bf..3c23586 100644 --- a/Sources/GoogleAI/GenerativeModel.swift +++ b/Sources/GoogleAI/GenerativeModel.swift @@ -159,7 +159,13 @@ public final class GenerativeModel { content = try contentClosure() } catch let underlying { return AsyncThrowingStream { continuation in - continuation.finish(throwing: GenerateContentError.internalError(underlying: underlying)) + let error: Error + if let contentError = underlying as? ImageConversionError { + error = GenerateContentError.promptContentError(underlying: contentError) + } else { + error = GenerateContentError.internalError(underlying: underlying) + } + continuation.finish(throwing: error) } } diff --git a/Sources/GoogleAI/PartsRepresentable+Image.swift b/Sources/GoogleAI/PartsRepresentable+Image.swift index f9c699c..50f9c1f 100644 --- a/Sources/GoogleAI/PartsRepresentable+Image.swift +++ b/Sources/GoogleAI/PartsRepresentable+Image.swift @@ -25,10 +25,8 @@ private let imageCompressionQuality: CGFloat = 0.8 /// For some image types like `CIImage`, creating valid model content requires creating a JPEG /// representation of the image that may not yet exist, which may be computationally expensive. public enum ImageConversionError: Error { - #if canImport(AppKit) - /// The image (the receiver of the call `toModelContentParts()`) was invalid. - case invalidUnderlyingImage - #endif + /// The image (the receiver of the call `toModelContentParts()`) was invalid. + case invalidUnderlyingImage /// A valid image destination could not be allocated. case couldNotAllocateDestination diff --git a/Tests/GoogleAITests/PartsRepresentableTests.swift b/Tests/GoogleAITests/PartsRepresentableTests.swift index c0424d8..24aef4a 100644 --- a/Tests/GoogleAITests/PartsRepresentableTests.swift +++ b/Tests/GoogleAITests/PartsRepresentableTests.swift @@ -14,6 +14,7 @@ import CoreGraphics import CoreImage +import GoogleGenerativeAI import XCTest #if canImport(UIKit) import UIKit @@ -49,7 +50,51 @@ final class ThrowingPartsRepresentableTests: XCTestCase { XCTAssert(modelContent.count > 0, "Expected non-empty model content for CGImage: \(image)") } + func testModelContentFromInvalidCIImageThrows() throws { + let image = CIImage.empty() + do { + let _ = try image.tryPartsValue() + } catch { + guard let imageError = (error as? ImageConversionError) else { + XCTFail("Got unexpected error type: \(error)") + return + } + switch imageError { + case let .couldNotConvertToJPEG(source): + // String(describing:) works around a type error. + XCTAssertEqual(String(describing: source), String(describing: image)) + return + case _: + XCTFail("Expected image conversion error, got \(imageError) instead") + return + } + } + XCTFail("Expected model content from invlaid image to error") + } + #if canImport(UIKit) + func testModelContentFromInvalidUIImageThrows() throws { + let image = UIImage() + do { + _ = try image.tryPartsValue() + } catch { + guard let imageError = (error as? ImageConversionError) else { + XCTFail("Got unexpected error type: \(error)") + return + } + switch imageError { + case let .couldNotConvertToJPEG(source): + // String(describing:) works around a type error. + XCTAssertEqual(String(describing: source), String(describing: image)) + return + case _: + XCTFail("Expected image conversion error, got \(imageError) instead") + return + } + } + XCTFail("Expected model content from invlaid image to error") + } + func testModelContentFromUIImageIsNotEmpty() throws { let coreImage = CIImage(color: CIColor.red) .cropped(to: CGRect(origin: CGPointZero, size: CGSize(width: 16, height: 16))) @@ -67,5 +112,26 @@ final class ThrowingPartsRepresentableTests: XCTestCase { let modelContent = try image.tryPartsValue() XCTAssert(modelContent.count > 0, "Expected non-empty model content for NSImage: \(image)") } + + func testModelContentFromInvalidNSImageThrows() throws { + let image = NSImage() + do { + _ = try image.tryPartsValue() + } catch { + guard let imageError = (error as? ImageConversionError) else { + XCTFail("Got unexpected error type: \(error)") + return + } + switch imageError { + case .invalidUnderlyingImage: + // Pass + return + case _: + XCTFail("Expected image conversion error, got \(imageError) instead") + return + } + } + XCTFail("Expected model content from invlaid image to error") + } #endif } From b744775fce82d424b9323939c496ba9a5ac5bed4 Mon Sep 17 00:00:00 2001 From: Morgan Chen Date: Thu, 15 Feb 2024 15:43:44 -0800 Subject: [PATCH 11/15] specialize error --- Sources/GoogleAI/Chat.swift | 4 ++-- Sources/GoogleAI/GenerateContentError.swift | 2 +- Sources/GoogleAI/GenerativeModel.swift | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Sources/GoogleAI/Chat.swift b/Sources/GoogleAI/Chat.swift index 46ffe12..1bce34f 100644 --- a/Sources/GoogleAI/Chat.swift +++ b/Sources/GoogleAI/Chat.swift @@ -48,7 +48,7 @@ public class Chat { newContent = try content().map(populateContentRole(_:)) } catch let underlying { if let contentError = underlying as? ImageConversionError { - throw GenerateContentError.promptContentError(underlying: contentError) + throw GenerateContentError.promptImageContentError(underlying: contentError) } else { throw GenerateContentError.internalError(underlying: underlying) } @@ -96,7 +96,7 @@ public class Chat { return AsyncThrowingStream { continuation in let error: Error if let contentError = underlying as? ImageConversionError { - error = GenerateContentError.promptContentError(underlying: contentError) + error = GenerateContentError.promptImageContentError(underlying: contentError) } else { error = GenerateContentError.internalError(underlying: underlying) } diff --git a/Sources/GoogleAI/GenerateContentError.swift b/Sources/GoogleAI/GenerateContentError.swift index 75ddff8..7f53aa6 100644 --- a/Sources/GoogleAI/GenerateContentError.swift +++ b/Sources/GoogleAI/GenerateContentError.swift @@ -17,7 +17,7 @@ import Foundation /// Errors that occur when generating content from a model. public enum GenerateContentError: Error { /// An error occurred when constructing the prompt. Examine the related error for details. - case promptContentError(underlying: ImageConversionError) + case promptImageContentError(underlying: ImageConversionError) /// An internal error occurred. See the underlying error for more context. case internalError(underlying: Error) diff --git a/Sources/GoogleAI/GenerativeModel.swift b/Sources/GoogleAI/GenerativeModel.swift index 3c23586..5d94621 100644 --- a/Sources/GoogleAI/GenerativeModel.swift +++ b/Sources/GoogleAI/GenerativeModel.swift @@ -161,7 +161,7 @@ public final class GenerativeModel { return AsyncThrowingStream { continuation in let error: Error if let contentError = underlying as? ImageConversionError { - error = GenerateContentError.promptContentError(underlying: contentError) + error = GenerateContentError.promptImageContentError(underlying: contentError) } else { error = GenerateContentError.internalError(underlying: underlying) } From de4292882d6b98299cec067857a22c9c4a51375c Mon Sep 17 00:00:00 2001 From: Morgan Chen Date: Thu, 15 Feb 2024 16:37:55 -0800 Subject: [PATCH 12/15] style --- Tests/GoogleAITests/PartsRepresentableTests.swift | 1 - 1 file changed, 1 deletion(-) diff --git a/Tests/GoogleAITests/PartsRepresentableTests.swift b/Tests/GoogleAITests/PartsRepresentableTests.swift index 0b3760f..da2f35f 100644 --- a/Tests/GoogleAITests/PartsRepresentableTests.swift +++ b/Tests/GoogleAITests/PartsRepresentableTests.swift @@ -24,7 +24,6 @@ import XCTest @available(iOS 15.0, macOS 11.0, macCatalyst 15.0, *) final class PartsRepresentableTests: XCTestCase { - func testModelContentFromCGImageIsNotEmpty() throws { // adapted from https://forums.swift.org/t/creating-a-cgimage-from-color-array/18634/2 var srgbArray = [UInt32](repeating: 0xFFFF_FFFF, count: 8 * 8) From 0adb052a7a32dea8fd8db2b1338f3c39cb517c60 Mon Sep 17 00:00:00 2001 From: Morgan Chen Date: Tue, 20 Feb 2024 15:38:37 -0800 Subject: [PATCH 13/15] code feedback changes --- Sources/GoogleAI/Chat.swift | 8 +++---- Sources/GoogleAI/GenerativeModel.swift | 33 ++++++++++++++------------ 2 files changed, 22 insertions(+), 19 deletions(-) diff --git a/Sources/GoogleAI/Chat.swift b/Sources/GoogleAI/Chat.swift index d0c6fff..c7cfb85 100644 --- a/Sources/GoogleAI/Chat.swift +++ b/Sources/GoogleAI/Chat.swift @@ -88,11 +88,11 @@ public class Chat { /// - Parameter content: The new content to send as a single chat message. /// - Returns: A stream containing the model's response or an error if an error occurred. @available(macOS 12.0, *) - public func sendMessageStream(_ contentClosure: @autoclosure () throws -> [ModelContent]) + public func sendMessageStream(_ content: @autoclosure () throws -> [ModelContent]) -> AsyncThrowingStream { - let content: [ModelContent] + let resolvedContent: [ModelContent] do { - content = try contentClosure() + resolvedContent = try content() } catch let underlying { return AsyncThrowingStream { continuation in let error: Error @@ -110,7 +110,7 @@ public class Chat { var aggregatedContent: [ModelContent] = [] // Ensure that the new content has the role set. - let newContent: [ModelContent] = content.map(populateContentRole(_:)) + let newContent: [ModelContent] = resolvedContent.map(populateContentRole(_:)) // Send the history alongside the new message as context. let request = history + newContent diff --git a/Sources/GoogleAI/GenerativeModel.swift b/Sources/GoogleAI/GenerativeModel.swift index 2a95112..2a963f7 100644 --- a/Sources/GoogleAI/GenerativeModel.swift +++ b/Sources/GoogleAI/GenerativeModel.swift @@ -111,18 +111,21 @@ public final class GenerativeModel { /// - Parameter content: The input(s) given to the model as a prompt. /// - Returns: The generated content response from the model. /// - Throws: A ``GenerateContentError`` if the request failed. - public func generateContent(_ content: [ModelContent]) async throws + public func generateContent(_ content: @autoclosure () throws -> [ModelContent]) async throws -> GenerateContentResponse { - let generateContentRequest = GenerateContentRequest(model: modelResourceName, - contents: content, - generationConfig: generationConfig, - safetySettings: safetySettings, - isStreaming: false, - options: requestOptions) let response: GenerateContentResponse do { + let generateContentRequest = try GenerateContentRequest(model: modelResourceName, + contents: content(), + generationConfig: generationConfig, + safetySettings: safetySettings, + isStreaming: false, + options: requestOptions) response = try await generativeAIService.loadRequest(request: generateContentRequest) } catch { + if let imageError = error as? ImageConversionError { + throw GenerateContentError.promptImageContentError(underlying: imageError) + } throw GenerativeModel.generateContentError(from: error) } @@ -251,16 +254,16 @@ public final class GenerativeModel { /// - Parameter content: The input given to the model as a prompt. /// - Returns: The results of running the model's tokenizer on the input; contains /// ``CountTokensResponse/totalTokens``. - /// - Throws: A ``CountTokensError`` if the tokenization request failed. - public func countTokens(_ content: [ModelContent]) async throws + /// - Throws: A ``CountTokensError`` if the tokenization request failed or the input content was + /// invalid. + public func countTokens(_ content: @autoclosure () throws -> [ModelContent]) async throws -> CountTokensResponse { - let countTokensRequest = CountTokensRequest( - model: modelResourceName, - contents: content, - options: requestOptions - ) - do { + let countTokensRequest = try CountTokensRequest( + model: modelResourceName, + contents: content(), + options: requestOptions + ) return try await generativeAIService.loadRequest(request: countTokensRequest) } catch { throw CountTokensError.internalError(underlying: error) From d962763ca1095879b54f2c12a9c0d3e708358aac Mon Sep 17 00:00:00 2001 From: Morgan Chen Date: Tue, 20 Feb 2024 15:45:15 -0800 Subject: [PATCH 14/15] use partsValue --- Sources/GoogleAI/ModelContent.swift | 4 ++-- Sources/GoogleAI/PartsRepresentable.swift | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Sources/GoogleAI/ModelContent.swift b/Sources/GoogleAI/ModelContent.swift index edfead4..172fc80 100644 --- a/Sources/GoogleAI/ModelContent.swift +++ b/Sources/GoogleAI/ModelContent.swift @@ -114,7 +114,7 @@ public struct ModelContent: Codable, Equatable { /// ``Part``. See ``ThrowingPartsRepresentable`` for types that can be interpreted as `Part`s. public init(role: String? = "user", parts: some PartsRepresentable) { self.role = role - self.parts = parts.toPartsValue() + self.parts = parts.partsValue } /// Creates a new value from a list of ``Part``s. @@ -135,7 +135,7 @@ public struct ModelContent: Codable, Equatable { /// ``ThrowingPartsRepresentable`` /// for types that can be interpreted as `Part`s. public init(role: String? = "user", _ parts: [PartsRepresentable]) { - let content = parts.flatMap { $0.toPartsValue() } + let content = parts.flatMap { $0.partsValue } self.init(role: role, parts: content) } } diff --git a/Sources/GoogleAI/PartsRepresentable.swift b/Sources/GoogleAI/PartsRepresentable.swift index 5d281ef..05ba0d9 100644 --- a/Sources/GoogleAI/PartsRepresentable.swift +++ b/Sources/GoogleAI/PartsRepresentable.swift @@ -26,13 +26,13 @@ public protocol ThrowingPartsRepresentable { /// ``ThrowingPartsRepresentable`` @available(iOS 15.0, macOS 11.0, macCatalyst 15.0, *) public protocol PartsRepresentable: ThrowingPartsRepresentable { - func toPartsValue() -> [ModelContent.Part] + var partsValue: [ModelContent.Part] { get } } @available(iOS 15.0, macOS 11.0, macCatalyst 15.0, *) public extension PartsRepresentable { func tryPartsValue() throws -> [ModelContent.Part] { - return toPartsValue() + return partsValue } } @@ -60,7 +60,7 @@ extension [ThrowingPartsRepresentable]: ThrowingPartsRepresentable { /// Enables a `String` to be passed in as ``ThrowingPartsRepresentable``. @available(iOS 15.0, macOS 11.0, macCatalyst 15.0, *) extension String: PartsRepresentable { - public func toPartsValue() -> [ModelContent.Part] { + public var partsValue: [ModelContent.Part] { return [.text(self)] } } From 183f14ede5c13441be36885c212b03dbfcdd74a1 Mon Sep 17 00:00:00 2001 From: Morgan Chen Date: Wed, 21 Feb 2024 13:43:08 -0800 Subject: [PATCH 15/15] use consistent closure name --- Sources/GoogleAI/GenerativeModel.swift | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Sources/GoogleAI/GenerativeModel.swift b/Sources/GoogleAI/GenerativeModel.swift index 2a963f7..03e0191 100644 --- a/Sources/GoogleAI/GenerativeModel.swift +++ b/Sources/GoogleAI/GenerativeModel.swift @@ -169,11 +169,11 @@ public final class GenerativeModel { /// - Returns: A stream wrapping content generated by the model or a ``GenerateContentError`` /// error if an error occurred. @available(macOS 12.0, *) - public func generateContentStream(_ contentClosure: @autoclosure () throws -> [ModelContent]) + public func generateContentStream(_ content: @autoclosure () throws -> [ModelContent]) -> AsyncThrowingStream { - let content: [ModelContent] + let evaluatedContent: [ModelContent] do { - content = try contentClosure() + evaluatedContent = try content() } catch let underlying { return AsyncThrowingStream { continuation in let error: Error @@ -187,7 +187,7 @@ public final class GenerativeModel { } let generateContentRequest = GenerateContentRequest(model: modelResourceName, - contents: content, + contents: evaluatedContent, generationConfig: generationConfig, safetySettings: safetySettings, isStreaming: true,