Lift to SpeziLLM

LLMonFHIR.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved

@@ -23,17 +23,25 @@
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/StanfordBDHG/llama.cpp",
       "state" : {
-        "revision" : "bcbf5bf9f677b92262aa28a2849defeddc00505d",
-        "version" : "0.1.6"
+        "revision" : "b0611c7d3cb049822f9911878514e4706b80e2ac",
+        "version" : "0.1.8"
       }
     },
     {
       "identity" : "openai",
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/MacPaw/OpenAI",
       "state" : {
-        "revision" : "ac5892fd0de8d283362ddc30f8e9f1a0eaba8cc0",
-        "version" : "0.2.5"
+        "revision" : "35afc9a6ee127b8f22a85a31aec2036a987478af"
+      }
+    },
+    {
+      "identity" : "semaphore",
+      "kind" : "remoteSourceControl",
+      "location" : "https://github.com/groue/Semaphore.git",
+      "state" : {
+        "revision" : "f1c4a0acabeb591068dea6cffdd39660b86dec28",
+        "version" : "0.0.8"
       }
     },
     {
@@ -48,10 +56,10 @@
     {
       "identity" : "spezichat",
       "kind" : "remoteSourceControl",
-      "location" : "https://github.com/StanfordSpezi/SpeziChat.git",
+      "location" : "https://github.com/StanfordSpezi/SpeziChat",
       "state" : {
-        "revision" : "9d45c10bcf859c98f2998ecd4f6a80f31894fe2c",
-        "version" : "0.1.4"
+        "revision" : "ea5e21b4f42d99a5549dd7a7033e2a3efeb5fd36",
+        "version" : "0.1.5"
       }
     },
     {
@@ -60,7 +68,7 @@
       "location" : "https://github.com/StanfordSpezi/SpeziFHIR",
       "state" : {
         "branch" : "feat/lift-to-spezi-llm",
-        "revision" : "28953591898c4b448028944f52cf4998895866ed"
+        "revision" : "b58ba7afc34ff1b83d4f4ab3525d3d13f372b080"
       }
     },
     {
@@ -84,19 +92,19 @@
     {
       "identity" : "spezillm",
       "kind" : "remoteSourceControl",
-      "location" : "https://github.com/StanfordSpezi/SpeziLLM.git",
+      "location" : "https://github.com/StanfordSpezi/SpeziLLM",
       "state" : {
-        "revision" : "24d6c197f1821925e3fc1ee9589859b6853aee01",
-        "version" : "0.6.0"
+        "branch" : "feat/structural-improvments",
+        "revision" : "f30f4d311cf99a396e76e5b732dca6462463f663"
       }
     },
     {
       "identity" : "spezionboarding",
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/StanfordSpezi/SpeziOnboarding.git",
       "state" : {
-        "revision" : "3ee713576eaeaa03200ba26bbc1269ceeb6abb25",
-        "version" : "1.0.1"
+        "revision" : "8fb6d9f1a080661c0cc564a93b82ead3c8d44d4f",
+        "version" : "1.0.2"
       }
     },
     {
@@ -111,7 +119,7 @@
     {
       "identity" : "spezistorage",
       "kind" : "remoteSourceControl",
-      "location" : "https://github.com/StanfordSpezi/SpeziStorage.git",
+      "location" : "https://github.com/StanfordSpezi/SpeziStorage",
       "state" : {
         "revision" : "eaed2220375c35400aa69d1f96a8d32b7e66b1c7",
         "version" : "1.0.0"
@@ -131,17 +139,17 @@
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/apple/swift-collections.git",
       "state" : {
-        "revision" : "a902f1823a7ff3c9ab2fba0f992396b948eda307",
-        "version" : "1.0.5"
+        "revision" : "d029d9d39c87bed85b1c50adee7c41795261a192",
+        "version" : "1.0.6"
       }
     },
     {
       "identity" : "xctestextensions",
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/StanfordBDHG/XCTestExtensions.git",
       "state" : {
-        "revision" : "388a6d6a5be48eff5d98a2c45e0b50f30ed21dc3",
-        "version" : "0.4.7"
+        "revision" : "fb7fcee97c574b950e03b0a53874e26db27db2fe",
+        "version" : "0.4.8"
       }
     },
     {

LLMonFHIR.xcodeproj/xcshareddata/xcschemes/LLMonFHIR.xcscheme

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <Scheme
-   LastUpgradeVersion = "1500"
+   LastUpgradeVersion = "1510"
    version = "1.7">
    <BuildAction
       parallelizeBuildables = "YES"

LLMonFHIR/FHIR Display/FHIRResourcesView.swift

@@ -7,11 +7,10 @@
 //
 
 import ModelsR4
-import OpenAI
 import SpeziFHIR
 import SpeziFHIRInterpretation
-import SpeziOnboarding
 import SpeziLLMOpenAI
+import SpeziOnboarding
 import SpeziViews
 import SwiftUI
 

LLMonFHIR/FHIR Display/InspectResourceView.swift

@@ -6,9 +6,9 @@
 // SPDX-License-Identifier: MIT
 //
 
-import OpenAI
 import SpeziFHIR
 import SpeziFHIRInterpretation
+import SpeziLLM
 import SpeziViews
 import SwiftUI
 
@@ -115,10 +115,10 @@ struct InspectResourceView: View {
             do {
                 try await fhirResourceSummary.summarize(resource: resource, forceReload: forceReload)
                 loadingSummary = .idle
-            } catch let error as APIErrorResponse {
+            } catch let error as LLMError {
                 loadingSummary = .error(error)
             } catch {
-                loadingSummary = .error("Unknown error")
+                loadingSummary = .error("Unknown LLM processing error")
             }
         }
     }
@@ -130,10 +130,10 @@ struct InspectResourceView: View {
             do {
                 try await fhirResourceInterpreter.interpret(resource: resource, forceReload: forceReload)
                 interpreting = .idle
-            } catch let error as APIErrorResponse {
-                loadingSummary = .error(error)
+            } catch let error as LLMError {
+                interpreting = .error(error)
             } catch {
-                loadingSummary = .error("Unknown error")
+                interpreting = .error("Unknown LLM processing error")
             }
         }
     }

LLMonFHIR/FHIR Display/MultipleResourcesChatView.swift

@@ -6,7 +6,6 @@
 // SPDX-License-Identifier: MIT
 //
 
-import OpenAI
 import SpeziChat
 import SpeziFHIR
 import SpeziFHIRInterpretation
@@ -25,27 +24,30 @@ struct MultipleResourcesChatView: View {
     @AppStorage(StorageKeys.enableTextToSpeech) private var textToSpeech = StorageKeys.Defaults.enableTextToSpeech
 
 
-    private var disableInput: Binding<Bool> {
-        Binding(
-            get: {
-                multipleResourceInterpreter.viewState == .processing
-            },
-            set: { _ in }
-        )
-    }
-
     var body: some View {
         @Bindable var multipleResourceInterpreter = multipleResourceInterpreter
         NavigationStack {
-            ChatView($multipleResourceInterpreter.chat, disableInput: disableInput)
+            Group {
+                if let llm = multipleResourceInterpreter.llm {
+                    let contextBinding = Binding { llm.context } set: { llm.context = $0 }
+                    ChatView(
+                        contextBinding,
+                        disableInput: multipleResourceInterpreter.viewState == .processing
+                    )
+                        .onChange(of: llm.context) {
+                            multipleResourceInterpreter.queryLLM()
+                        }
+                } else {
+                    ChatView(
+                        .constant([])
+                    )
+                }
+            }
                 .navigationTitle("LLM on FHIR")
                 .toolbar {
                     toolbar
                 }
                 .viewStateAlert(state: $multipleResourceInterpreter.viewState)
-                .onChange(of: multipleResourceInterpreter.chat) {
-                    multipleResourceInterpreter.queryLLM()
-                }
                 .onAppear {
                     multipleResourceInterpreter.queryLLM()
                 }
@@ -54,7 +56,7 @@ struct MultipleResourcesChatView: View {
     }
 
 
-    @ToolbarContentBuilder private var toolbar: some ToolbarContent {
+    @MainActor @ToolbarContentBuilder private var toolbar: some ToolbarContent {
         ToolbarItem(placement: .cancellationAction) {
             if multipleResourceInterpreter.viewState == .processing {
                 ProgressView()

LLMonFHIR/FHIR Interpretation/FHIRInterpretationFunction.swift

@@ -0,0 +1,110 @@
+//
+// This source file is part of the Stanford LLM on FHIR project
+//
+// SPDX-FileCopyrightText: 2023 Stanford University
+//
+// SPDX-License-Identifier: MIT
+//
+
+import os
+import SpeziFHIR
+import SpeziFHIRInterpretation
+import SpeziLLMOpenAI
+
+
+struct FHIRInterpretationFunction: LLMFunction {
+    static let logger = Logger(subsystem: "edu.stanford.bdhg", category: "LLMonFHIR")
+
+    static let name = "get_resources"
+    static let description = String(localized: "FUNCTION_DESCRIPTION")
+
+    private let fhirStore: FHIRStore
+    private let resourceSummary: FHIRResourceSummary
+    private let allResourcesFunctionCallIdentifier: [String]
+
+
+    @Parameter var resources: [String]
+
+
+    init(fhirStore: FHIRStore, resourceSummary: FHIRResourceSummary, allResourcesFunctionCallIdentifier: [String]) {
+        self.fhirStore = fhirStore
+        self.resourceSummary = resourceSummary
+        self.allResourcesFunctionCallIdentifier = allResourcesFunctionCallIdentifier
+
+        _resources = Parameter(description: String(localized: "PARAMETER_DESCRIPTION"), enumValues: allResourcesFunctionCallIdentifier)
+    }
+
+
+    func execute() async throws -> String? {
+        var functionOutput: [String] = []
+
+        try await withThrowingTaskGroup(of: [String].self) { outerGroup in
+            // Iterate over all requested resources by the LLM
+            for requestedResource in resources {
+                outerGroup.addTask {
+                    // Fetch relevant FHIR resources matching the resources requested by the LLM
+                    var fittingResources = fhirStore.llmRelevantResources.filter { $0.functionCallIdentifier.contains(requestedResource) }
+
+                    // Stores output of nested task group summarizing fitting resources
+                    var nestedFunctionOutputResults = [String]()
+
+                    guard !fittingResources.isEmpty else {
+                        nestedFunctionOutputResults.append(
+                            String(
+                                localized: "The medical record does not include any FHIR resources for the search term \(requestedResource)."
+                            )
+                        )
+                        return []
+                    }
+
+                    // Filter out fitting resources (if greater than 64 entries)
+                    fittingResources = filterFittingResources(fittingResources)
+
+                    try await withThrowingTaskGroup(of: String.self) { innerGroup in
+                        // Iterate over fitting resources and summarizing them
+                        for resource in fittingResources {
+                            innerGroup.addTask {
+                                try await summarizeResource(fhirResource: resource, resourceType: requestedResource)
+                            }
+                        }
+
+                        for try await nestedResult in innerGroup {
+                            nestedFunctionOutputResults.append(nestedResult)
+                        }
+                    }
+
+                    return nestedFunctionOutputResults
+                }
+            }
+
+            for try await result in outerGroup {
+                functionOutput.append(contentsOf: result)
+            }
+        }
+
+        return functionOutput.joined(separator: "\n\n")
+    }
+
+    private func summarizeResource(fhirResource: FHIRResource, resourceType: String) async throws -> String {
+        let summary = try await resourceSummary.summarize(resource: fhirResource)
+        Self.logger.debug("Summary of appended FHIR resource \(resourceType): \(summary.description)")
+        return String(localized: "This is the summary of the requested \(resourceType):\n\n\(summary.description)")
+    }
+
+    private func filterFittingResources(_ fittingResources: [FHIRResource]) -> [FHIRResource] {
+        Self.logger.debug("Overall fitting Resources: \(fittingResources.count)")
+
+        var fittingResources = fittingResources
+
+        if fittingResources.count > 64 {
+            fittingResources = fittingResources.lazy.sorted(by: { $0.date ?? .distantPast < $1.date ?? .distantPast }).suffix(64)
+            Self.logger.debug(
+                """
+                Reduced to the following 64 resources: \(fittingResources.map { $0.functionCallIdentifier }.joined(separator: ","))
+                """
+            )
+        }
+
+        return fittingResources
+    }
+}