diff --git a/spring-ai-docs/src/main/antora/modules/ROOT/nav.adoc b/spring-ai-docs/src/main/antora/modules/ROOT/nav.adoc index 93940645ae7..421aa760eab 100644 --- a/spring-ai-docs/src/main/antora/modules/ROOT/nav.adoc +++ b/spring-ai-docs/src/main/antora/modules/ROOT/nav.adoc @@ -1,10 +1,11 @@ * xref:index.adoc[Overview] -* xref:concepts.adoc[AI Concepts] +** xref:concepts.adoc[AI Concepts] * xref:getting-started.adoc[Getting Started] -* xref:api/index.adoc[] -** xref:api/chatclient.adoc[] -*** xref:api/advisors.adoc[Advisors] -** xref:api/chatmodel.adoc[] +* xref:api/chatclient.adoc[] +** xref:api/advisors.adoc[Advisors] +* xref:api/index.adoc[AI Models] +** xref:api/chatmodel.adoc[Chat Models] +*** xref:api/prompt.adoc[] *** xref:api/bedrock-chat.adoc[Amazon Bedrock] **** xref:api/chat/bedrock/bedrock-anthropic3.adoc[Anthropic3] **** xref:api/chat/bedrock/bedrock-anthropic.adoc[Anthropic2] @@ -37,7 +38,7 @@ *** xref:api/chat/zhipuai-chat.adoc[ZhiPu AI] // **** xref:api/chat/functions/zhipuai-chat-functions.adoc[Function Calling] *** xref:api/chat/watsonx-ai-chat.adoc[watsonx.AI] -** xref:api/embeddings.adoc[] +** xref:api/embeddings.adoc[Embedding Models] *** xref:api/bedrock.adoc[Amazon Bedrock] **** xref:api/embeddings/bedrock-cohere-embedding.adoc[Cohere] **** xref:api/embeddings/bedrock-titan-embedding.adoc[Titan] @@ -56,49 +57,48 @@ **** xref:api/embeddings/vertexai-embeddings-palm2.adoc[PaLM2 Embedding] *** xref:api/embeddings/watsonx-ai-embeddings.adoc[watsonx.AI] *** xref:api/embeddings/zhipuai-embeddings.adoc[ZhiPu AI] -** xref:api/imageclient.adoc[] +** xref:api/imageclient.adoc[Image Models] *** xref:api/image/azure-openai-image.adoc[Azure OpenAI] *** xref:api/image/openai-image.adoc[OpenAI] *** xref:api/image/stabilityai-image.adoc[Stability] *** xref:api/image/zhipuai-image.adoc[ZhiPuAI] *** xref:api/image/qianfan-image.adoc[QianFan] -** xref:api/audio[Audio Model API] +** xref:api/audio[Audio Models] *** xref:api/audio/transcriptions.adoc[] **** xref:api/audio/transcriptions/azure-openai-transcriptions.adoc[Azure OpenAI] **** xref:api/audio/transcriptions/openai-transcriptions.adoc[OpenAI] *** xref:api/audio/speech.adoc[] **** xref:api/audio/speech/openai-speech.adoc[OpenAI] -** xref:api/moderation[Moderation Model API] +** xref:api/moderation[Moderation Models] *** xref:api/moderation/openai-moderation.adoc[OpenAI] +// ** xref:api/generic-model.adoc[] -** xref:api/vectordbs.adoc[] -*** xref:api/vectordbs/azure.adoc[] -*** xref:api/vectordbs/apache-cassandra.adoc[] -*** xref:api/vectordbs/chroma.adoc[] -*** xref:api/vectordbs/elasticsearch.adoc[] -*** xref:api/vectordbs/gemfire.adoc[GemFire] -*** xref:api/vectordbs/milvus.adoc[] -*** xref:api/vectordbs/mongodb.adoc[] -*** xref:api/vectordbs/neo4j.adoc[] -*** xref:api/vectordbs/opensearch.adoc[] -*** xref:api/vectordbs/oracle.adoc[Oracle] -*** xref:api/vectordbs/pgvector.adoc[] -*** xref:api/vectordbs/pinecone.adoc[] -*** xref:api/vectordbs/qdrant.adoc[] -*** xref:api/vectordbs/redis.adoc[] -*** xref:api/vectordbs/hana.adoc[SAP Hana] -*** xref:api/vectordbs/typesense.adoc[] -*** xref:api/vectordbs/weaviate.adoc[] - -** xref:api/functions.adoc[Function Calling] -** xref:api/multimodality.adoc[Multimodality] -** xref:api/prompt.adoc[] -** xref:api/structured-output-converter.adoc[Structured Output] +* xref:api/vectordbs.adoc[] ** xref:api/etl-pipeline.adoc[] -** xref:api/testing.adoc[] -** xref:api/generic-model.adoc[] +** xref:api/vectordbs/azure.adoc[] +** xref:api/vectordbs/apache-cassandra.adoc[] +** xref:api/vectordbs/chroma.adoc[] +** xref:api/vectordbs/elasticsearch.adoc[] +** xref:api/vectordbs/gemfire.adoc[GemFire] +** xref:api/vectordbs/milvus.adoc[] +** xref:api/vectordbs/mongodb.adoc[] +** xref:api/vectordbs/neo4j.adoc[] +** xref:api/vectordbs/opensearch.adoc[] +** xref:api/vectordbs/oracle.adoc[Oracle] +** xref:api/vectordbs/pgvector.adoc[] +** xref:api/vectordbs/pinecone.adoc[] +** xref:api/vectordbs/qdrant.adoc[] +** xref:api/vectordbs/redis.adoc[] +** xref:api/vectordbs/hana.adoc[SAP Hana] +** xref:api/vectordbs/typesense.adoc[] +** xref:api/vectordbs/weaviate.adoc[] + * xref:observabilty/index.adoc[] +* xref:api/functions.adoc[Function Calling] +* xref:api/multimodality.adoc[Multimodality] +* xref:api/testing.adoc[LLM Evaluation] +* xref:api/structured-output-converter.adoc[Structured Output] * Service Connections ** xref:api/docker-compose.adoc[Docker Compose] diff --git a/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/multimodality.adoc b/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/multimodality.adoc index 17d9ae7d5ba..bbcab25b3b9 100644 --- a/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/multimodality.adoc +++ b/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/multimodality.adoc @@ -1,23 +1,21 @@ [[Multimodality]] = Multimodality API +// image::orbis-sensualium-pictus2.jpg[Orbis Sensualium Pictus, align="center"] + +> "All things that are naturally connected ought to be taught in combination" - John Amos Comenius, "Orbis Sensualium Pictus", 1658 + Humans process knowledge, simultaneously across multiple modes of data inputs. The way we learn, our experiences are all multimodal. We don't have just vision, just audio and just text. -These foundational principles of learning were articulated by the father of modern education link:https://en.wikipedia.org/wiki/John_Amos_Comenius[John Amos Comenius], in his work, "Orbis Sensualium Pictus", dating back to 1658. - -image::orbis-sensualium-pictus2.jpg[Orbis Sensualium Pictus, align="center"] - -> "All things that are naturally connected ought to be taught in combination" - -Contrary to those principles, in the past, our approach to Machine Learning was often focused on specialized models tailored to process a single modality. +Contrary to those principles, the Machine Learning was often focused on specialized models tailored to process a single modality. For instance, we developed audio models for tasks like text-to-speech or speech-to-text, and computer vision models for tasks such as object detection and classification. However, a new wave of multimodal large language models starts to emerge. -Examples include OpenAI's GPT-4 Vision, Google's Vertex AI Gemini Pro Vision, Anthropic's Claude3, and open source offerings LLaVA and balklava are able to accept multiple inputs, including text images, audio and video and generate text responses by integrating these inputs. +Examples include OpenAI's GPT-4o , Google's Vertex AI Gemini 1.5, Anthropic's Claude3, and open source offerings Llama3.2, LLaVA and Balklava are able to accept multiple inputs, including text images, audio and video and generate text responses by integrating these inputs. -The multimodal large language model (LLM) features enable the models to process and generate text in conjunction with other modalities such as images, audio, or video. +NOTE: The multimodal large language model (LLM) features enable the models to process and generate text in conjunction with other modalities such as images, audio, or video. == Spring AI Multimodality @@ -68,7 +66,7 @@ and produce a response like: Spring AI provides multimodal support for the following chat models: * xref:api/chat/openai-chat.adoc#_multimodal[OpenAI (e.g. GPT-4 and GPT-4o models)] -* xref:api/chat/ollama-chat.adoc#_multimodal[Ollama (e.g. LlaVa and Baklava models)] +* xref:api/chat/ollama-chat.adoc#_multimodal[Ollama (e.g. LlaVa, Baklava, Llama3.2 models)] * xref:api/chat/vertexai-gemini-chat.adoc#_multimodal[Vertex AI Gemini (e.g. gemini-1.5-pro-001, gemini-1.5-flash-001 models)] * xref:api/chat/anthropic-chat.adoc#_multimodal[Anthropic Claude 3] * xref:api/chat/bedrock/bedrock-anthropic3.adoc#_multimodal[AWS Bedrock Anthropic Claude 3]