feat (provider/perplexity): add support for return_images (#4651)

Co-authored-by: Kevin Wolf <[email protected]>

.changeset/large-tables-jam.md

@@ -0,0 +1,5 @@
+---
+'@ai-sdk/perplexity': patch
+---
+
+feat (provider/perplexity): add support for return_images

content/providers/01-ai-sdk-providers/70-perplexity.mdx

@@ -88,6 +88,11 @@ The Perplexity provider includes additional experimental metadata in the respons
 const result = await generateText({
   model: perplexity('sonar-pro'),
   prompt: 'What are the latest developments in quantum computing?',
+  providerOptions: {
+    perplexity: {
+      return_images: true, // Enable image responses (Tier-2 Perplexity users only)
+    },
+  },
 });
 
 console.log(result.experimental_providerMetadata);
@@ -99,6 +104,10 @@ console.log(result.experimental_providerMetadata);
 //       'https://www.cbsnews.com/sanfrancisco/',
 //     ],
 //     usage: { citationTokens: 5286, numSearchQueries: 1 },
+//     images: [
+//       { imageUrl: "https://example.com/image1.jpg", originUrl: "https://elsewhere.com/page1", height: 1280, width: 720 },
+//       { imageUrl: "https://example.com/image2.jpg", originUrl: "https://elsewhere.com/page2", height: 1280, width: 720 }
+//     ]
 //   },
 // }
 ```
@@ -107,9 +116,12 @@ The metadata includes:
 
 - `citations`: Array of URLs used as sources for the response
 - `usage`: Object containing `citationTokens` and `numSearchQueries` metrics
+- `images`: Array of image URLs when `return_images` is enabled (Tier-2 users only)
+
+You can enable image responses by setting `return_images: true` in the provider options. This feature is only available to Perplexity Tier-2 users and above.
 
 <Note>
-  For more details about Perplexity's citations see the [Perplexity chat
+  For more details about Perplexity's capabilities, see the [Perplexity chat
   completion docs](https://docs.perplexity.ai/api-reference/chat-completions).
 </Note>
 

examples/ai-core/src/generate-text/perplexity-images.ts

@@ -0,0 +1,24 @@
+import 'dotenv/config';
+import { perplexity } from '@ai-sdk/perplexity';
+import { generateText } from 'ai';
+
+async function main() {
+  const result = await generateText({
+    model: perplexity('sonar-pro'),
+    prompt:
+      'Tell me about the earliest cave drawings known and include images.',
+    providerOptions: {
+      perplexity: {
+        return_images: true,
+      },
+    },
+  });
+
+  console.log(result.text);
+  console.log();
+  console.log('Token usage:', result.usage);
+  console.log('Finish reason:', result.finishReason);
+  console.log('Metadata:', result.experimental_providerMetadata);
+}
+
+main().catch(console.error);

examples/ai-core/src/stream-text/perplexity-images.ts

@@ -0,0 +1,30 @@
+import { perplexity } from '@ai-sdk/perplexity';
+import { streamText } from 'ai';
+import 'dotenv/config';
+
+async function main() {
+  const result = streamText({
+    model: perplexity('sonar-pro'),
+    prompt:
+      'Tell me about the earliest cave drawings known and include images.',
+    providerOptions: {
+      perplexity: {
+        return_images: true,
+      },
+    },
+  });
+
+  for await (const textPart of result.textStream) {
+    process.stdout.write(textPart);
+  }
+
+  console.log();
+  console.log('Token usage:', await result.usage);
+  console.log('Finish reason:', await result.finishReason);
+  console.log(
+    'Metadata:',
+    JSON.stringify(await result.experimental_providerMetadata, null, 2),
+  );
+}
+
+main().catch(console.error);

examples/ai-core/src/stream-text/perplexity.ts

@@ -17,7 +17,10 @@ async function main() {
   console.log();
   console.log('Token usage:', await result.usage);
   console.log('Finish reason:', await result.finishReason);
-  console.log('Metadata:', await result.experimental_providerMetadata);
+  console.log(
+    'Metadata:',
+    JSON.stringify(await result.experimental_providerMetadata, null, 2),
+  );
 }
 
 main().catch(console.error);

packages/perplexity/src/perplexity-metadata-extractor.test.ts

@@ -1,9 +1,17 @@
 import { perplexityMetadataExtractor } from './perplexity-metadata-extractor';
 
 describe('buildMetadataFromResponse', () => {
-  it('should extract metadata from complete response with citations and usage', () => {
+  it('should extract metadata from complete response with citations, images and usage', () => {
     const response = {
       citations: ['source1', 'source2'],
+      images: [
+        {
+          image_url: 'https://images.com/image1.jpg',
+          origin_url: 'https://elsewhere.com/page1',
+          height: 100,
+          width: 100,
+        },
+      ],
       usage: {
         citation_tokens: 100,
         num_search_queries: 5,
@@ -17,6 +25,14 @@ describe('buildMetadataFromResponse', () => {
     expect(metadata).toEqual({
       perplexity: {
         citations: ['source1', 'source2'],
+        images: [
+          {
+            imageUrl: 'https://images.com/image1.jpg',
+            originUrl: 'https://elsewhere.com/page1',
+            height: 100,
+            width: 100,
+          },
+        ],
         usage: {
           citationTokens: 100,
           numSearchQueries: 5,
@@ -41,6 +57,36 @@ describe('buildMetadataFromResponse', () => {
     });
   });
 
+  it('should extract metadata with only images', () => {
+    const response = {
+      images: [
+        {
+          image_url: 'https://images.com/image1.jpg',
+          origin_url: 'https://elsewhere.com/page1',
+          height: 100,
+          width: 100,
+        },
+      ],
+    };
+
+    const metadata = perplexityMetadataExtractor.extractMetadata({
+      parsedBody: response,
+    });
+
+    expect(metadata).toEqual({
+      perplexity: {
+        images: [
+          {
+            imageUrl: 'https://images.com/image1.jpg',
+            originUrl: 'https://elsewhere.com/page1',
+            height: 100,
+            width: 100,
+          },
+        ],
+      },
+    });
+  });
+
   it('should extract metadata with only usage', () => {
     const response = {
       usage: {
@@ -97,6 +143,19 @@ describe('streaming metadata extractor', () => {
       citations: ['source1', 'source2'],
     });
 
+    // Process chunk with images
+    extractor.processChunk({
+      choices: [{ delta: { role: 'assistant', content: 'content' } }],
+      images: [
+        {
+          image_url: 'https://images.com/image1.jpg',
+          origin_url: 'https://elsewhere.com/page1',
+          height: 100,
+          width: 100,
+        },
+      ],
+    });
+
     // Process chunk with usage
     extractor.processChunk({
       choices: [{ delta: { role: 'assistant', content: 'content' } }],
@@ -111,6 +170,14 @@ describe('streaming metadata extractor', () => {
     expect(finalMetadata).toEqual({
       perplexity: {
         citations: ['source1', 'source2'],
+        images: [
+          {
+            imageUrl: 'https://images.com/image1.jpg',
+            originUrl: 'https://elsewhere.com/page1',
+            height: 100,
+            width: 100,
+          },
+        ],
         usage: {
           citationTokens: 100,
           numSearchQueries: 5,
@@ -125,6 +192,14 @@ describe('streaming metadata extractor', () => {
     // Process initial chunk
     extractor.processChunk({
       citations: ['source1'],
+      images: [
+        {
+          image_url: 'https://images.com/image1.jpg',
+          origin_url: 'https://elsewhere.com/page1',
+          height: 100,
+          width: 100,
+        },
+      ],
       usage: {
         citation_tokens: 50,
         num_search_queries: 2,
@@ -134,6 +209,20 @@ describe('streaming metadata extractor', () => {
     // Process chunk with updated data
     extractor.processChunk({
       citations: ['source1', 'source2'],
+      images: [
+        {
+          image_url: 'https://images.com/image1.jpg',
+          origin_url: 'https://elsewhere.com/page1',
+          height: 100,
+          width: 100,
+        },
+        {
+          image_url: 'https://images.com/image2.jpg',
+          origin_url: 'https://elsewhere.com/page2',
+          height: 200,
+          width: 200,
+        },
+      ],
       usage: {
         citation_tokens: 100,
         num_search_queries: 5,
@@ -145,6 +234,20 @@ describe('streaming metadata extractor', () => {
     expect(finalMetadata).toEqual({
       perplexity: {
         citations: ['source1', 'source2'],
+        images: [
+          {
+            imageUrl: 'https://images.com/image1.jpg',
+            originUrl: 'https://elsewhere.com/page1',
+            height: 100,
+            width: 100,
+          },
+          {
+            imageUrl: 'https://images.com/image2.jpg',
+            originUrl: 'https://elsewhere.com/page2',
+            height: 200,
+            width: 200,
+          },
+        ],
         usage: {
           citationTokens: 100,
           numSearchQueries: 5,