Skip to content

Commit

Permalink
feat: image embeddings
Browse files Browse the repository at this point in the history
  • Loading branch information
@santiagxf
santiagxf committed Jan 15, 2025
1 parent 3d6b1da commit 7f434e3
Show file tree
Hide file tree
Showing 10 changed files with 504 additions and 0 deletions.
1 change: 1 addition & 0 deletions ...tion/ai/ModelInference/examples/2024-05-01-preview/GetChatCompletions_MaximumSet_Gen.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
"api-version": "2024-05-01-preview",
"extra-parameters": "error",
"body": {
"modalities": [ "text" ],
"messages": [
{
"role": "system",
Expand Down
53 changes: 53 additions & 0 deletions ...tion/ai/ModelInference/examples/2024-05-01-preview/GetImageEmbeddings_MaximumSet_Gen.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
{
"title": "maximum set image embeddings",
"operationId": "GetImageEmbeddings",
"parameters": {
"api-version": "2024-05-01-preview",
"extra-parameters": "error",
"body": {
"input": [
{
"image": "puqkvvlvgcjyzughesnkena",
"text": "azrzyjsmnuefqpowpvfmyobeehqsni"
}
],
"dimensions": 1024,
"encoding_format": "float",
"input_type": "text",
"model": "my-model-name"
}
},
"responses": {
"200": {
"body": {
"id": "cknxthfa",
"data": [
{
"index": 0,
"object": "embedding",
"embedding": [
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0
]
}
],
"object": "list",
"model": "my-model-name",
"usage": {
"prompt_tokens": 15,
"total_tokens": 15
}
}
}
}
}
47 changes: 47 additions & 0 deletions ...tion/ai/ModelInference/examples/2024-05-01-preview/GetImageEmbeddings_MinimumSet_Gen.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
{
"title": "minimum set image embeddings",
"operationId": "GetImageEmbeddings",
"parameters": {
"api-version": "2024-05-01-preview",
"body": {
"input": [
{
"image": "gvmojtfooxixxzayrditjlyymg"
}
]
}
},
"responses": {
"200": {
"body": {
"id": "cknxthfa",
"data": [
{
"index": 0,
"object": "embedding",
"embedding": [
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0
]
}
],
"object": "list",
"model": "my-model-name",
"usage": {
"prompt_tokens": 15,
"total_tokens": 15
}
}
}
}
}
61 changes: 61 additions & 0 deletions specification/ai/ModelInference/models/image_embeddings.tsp
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
import "@typespec/rest";
import "@typespec/http";

import "./embeddings.tsp";

using TypeSpec.Rest;
using TypeSpec.Http;

namespace ModelInference;

@doc("""
The configuration information for an image embeddings request.
""")
model ImageEmbeddingsOptions {
@doc("""
Input image to embed. To embed multiple inputs in a single request, pass an array.
The input must not exceed the max input tokens for the model.
""")
input: ImageEmbeddingInput[];

@doc("""
Optional. The number of dimensions the resulting output embeddings should have.
Passing null causes the model to use its default value.
Returns a 422 error if the model doesn't support the value or parameter.
""")
dimensions?: int32;

@doc("""
Optional. The number of dimensions the resulting output embeddings should have.
Passing null causes the model to use its default value.
Returns a 422 error if the model doesn't support the value or parameter.
""")
encoding_format?: EmbeddingEncodingFormat;

@doc("""
Optional. The type of the input.
Returns a 422 error if the model doesn't support the value or parameter.
""")
input_type?: EmbeddingInputType;

@doc("""
ID of the specific AI model to use, if more than one model is available on the endpoint.
""")
`model`?: string;

...Record<unknown>;
}

@doc("Represents an image with optional text.")
model ImageEmbeddingInput {
@doc("""
The input image encoded in base64 string as a data URL. Example: `data:image/{format};base64,{data}`.
""")
image: string;

@doc("""
Optional. The text input to feed into the model (like DINO, CLIP).
Returns a 422 error if the model doesn't support the value or parameter.
""")
text?: string;
}
15 changes: 15 additions & 0 deletions specification/ai/ModelInference/routes.tsp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import "@typespec/versioning";
import "./models/common.tsp";
import "./models/chat_completions.tsp";
import "./models/embeddings.tsp";
import "./models/image_embeddings.tsp";

using TypeSpec.Rest;
using TypeSpec.Http;
Expand Down Expand Up @@ -44,6 +45,20 @@ op getEmbeddings is Azure.Core.RpcOperation<
EmbeddingsResult
>;

@doc("""
Return the embedding vectors for given images.
The method makes a REST API call to the `/images/embeddings` route on the given endpoint.
""")
@actionSeparator("/")
@route("images/embeddings")
op getImageEmbeddings is Azure.Core.RpcOperation<
{
...ImageEmbeddingsOptions;
...AdditionalRequestHeaders;
},
EmbeddingsResult
>;

@doc("""
Returns information about the AI model.
The method makes a REST API call to the `/info` route on the given endpoint.
Expand Down
1 change: 1 addition & 0 deletions ...ModelInference/preview/2024-05-01-preview/examples/GetChatCompletions_MaximumSet_Gen.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
"api-version": "2024-05-01-preview",
"extra-parameters": "error",
"body": {
"modalities": [ "text" ],
"messages": [
{
"role": "system",
Expand Down
53 changes: 53 additions & 0 deletions ...ModelInference/preview/2024-05-01-preview/examples/GetImageEmbeddings_MaximumSet_Gen.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
{
"title": "maximum set image embeddings",
"operationId": "GetImageEmbeddings",
"parameters": {
"api-version": "2024-05-01-preview",
"extra-parameters": "error",
"body": {
"input": [
{
"image": "puqkvvlvgcjyzughesnkena",
"text": "azrzyjsmnuefqpowpvfmyobeehqsni"
}
],
"dimensions": 1024,
"encoding_format": "float",
"input_type": "text",
"model": "my-model-name"
}
},
"responses": {
"200": {
"body": {
"id": "cknxthfa",
"data": [
{
"index": 0,
"object": "embedding",
"embedding": [
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0
]
}
],
"object": "list",
"model": "my-model-name",
"usage": {
"prompt_tokens": 15,
"total_tokens": 15
}
}
}
}
}
47 changes: 47 additions & 0 deletions ...ModelInference/preview/2024-05-01-preview/examples/GetImageEmbeddings_MinimumSet_Gen.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
{
"title": "minimum set image embeddings",
"operationId": "GetImageEmbeddings",
"parameters": {
"api-version": "2024-05-01-preview",
"body": {
"input": [
{
"image": "gvmojtfooxixxzayrditjlyymg"
}
]
}
},
"responses": {
"200": {
"body": {
"id": "cknxthfa",
"data": [
{
"index": 0,
"object": "embedding",
"embedding": [
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0
]
}
],
"object": "list",
"model": "my-model-name",
"usage": {
"prompt_tokens": 15,
"total_tokens": 15
}
}
}
}
}
Loading

0 comments on commit 7f434e3

Please sign in to comment.