feat: update theme, add diarization (#81)

* feat: update theme, add diarization

* fix: navigation test

* fix: Broken test during to locals change

* fix: timeout in navigation test not long enough

.env.example

@@ -1,8 +1,14 @@
-AZURE_OPENAI_ENDPOINT=url
-AZURE_OPENAI_API_KEY=key
-AZURE_OPENAI_GPT_DEPLOYMENT_NAME=name
-AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME=name
-AZURE_DOCUMENT_INTELLIGENCE_KEY=key
-AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT=endpoint
-
-OPENAI_API_KEY=key
+
+AZURE_OPENAI_ENDPOINT=secret
+AZURE_OPENAI_API_KEY=secret
+AZURE_OPENAI_GPT_DEPLOYMENT_NAME=secret
+AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME=secret
+AZURE_OPENAI_WHISPER_DEPLOYMENT_NAME=secret
+
+AZURE_CREATE_DIARIZATION_ENDPOINT=secret
+
+AZURE_DOCUMENT_INTELLIGENCE_KEY=secret
+AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT=secret
+
+AWS_ACCESS_KEY=secret
+AWS_SECRET_ACCESS_KEY=secret

.gitignore

@@ -10,3 +10,4 @@ node_modules
 vite.config.js.timestamp-*
 vite.config.ts.timestamp-*
 .vscode
+test-results

package.json

@@ -53,7 +53,7 @@
 	"dependencies": {
 		"@aws-sdk/client-bedrock-runtime": "^3.624.0",
 		"@azure/openai": "^1.0.0-beta.12",
-		"@cmi-dair/skeleton-themes": "^0.1.1",
+		"@cmi-dair/skeleton-themes": "^0.2.0",
 		"@floating-ui/dom": "^1.6.7",
 		"@sveltejs/adapter-node": "^5.2.0",
 		"fluent-ffmpeg": "^2.1.3",

src/app.d.ts

@@ -2,7 +2,9 @@
 // for information about these interfaces
 // and what to do when importing types
 declare namespace App {
-	// interface Locals {}
+	interface Locals {
+		user: string;
+	}
 	// interface PageData {}
 	// interface Error {}
 	// interface Platform {}

src/hooks.server.test.ts

@@ -24,13 +24,17 @@ function createResolve() {
 
 describe('handle requests', () => {
 	it('should set request headers and log request information', async () => {
-		const event = createEvent();
+		const event = {
+			locals: {},
+			request: new Request('https://example.com'),
+			resolve: async () => new Response()
+		};
+
 		const resolve = createResolve();
 
 		const response = await handle({ event, resolve });
 
 		expect(event.request.headers.get('X-Request-ID')).toBeDefined();
-		expect(event.request.headers.get('X-User')).toBe('[email protected]');
 		expect(resolve).toHaveBeenCalledWith(event);
 		expect(response.headers.append).toHaveBeenCalledWith('X-Request-ID', expect.any(String));
 	});

src/hooks.server.ts

@@ -6,17 +6,16 @@ export async function handle({ event, resolve }) {
 	const requestId = randomUUID();
 	const startTime = performance.now();
 
-	const user =
+	event.locals.user =
 		event.request.headers.get('X-MS-CLIENT-PRINCIPAL-NAME') || '[email protected]';
 	logger.info({
 		type: `Request`,
 		method: event.request.method,
 		url: event.request.url,
-		user,
+		user: event.locals.user,
 		requestId
 	});
 	event.request.headers.set('X-Request-ID', requestId);
-	event.request.headers.set('X-User', user);
 
 	const response = await resolve(event);
 
@@ -27,7 +26,7 @@ export async function handle({ event, resolve }) {
 		statusCode: response.status,
 		method: event.request.method,
 		url: event.request.url,
-		user,
+		user: event.locals.user,
 		requestId,
 		responseTime
 	};

src/lib/components/Navigation.svelte

@@ -6,6 +6,8 @@
 	const pages = [
 		{ name: 'Chatbot', href: '/gpt' },
 		{ name: 'Document Intelligence', href: '/document-intelligence' },
+		{ name: 'Diarize (Submit)', href: '/diarize-submit' },
+		{ name: 'Diarize (Retrieve)', href: '/diarize-retrieve' },
 		{ name: 'Embedding', href: '/embedding' },
 		{ name: 'Image Generation', href: '/dalle' },
 		{ name: 'Text To Speech', href: '/text-to-speech' },

src/lib/components/PageTemplates/FormBasePage.svelte

@@ -2,8 +2,8 @@
 	import { page } from '$app/stores';
 	import { getToastStore, type ToastSettings } from '@skeletonlabs/skeleton';
 	import LoadingBar from '$lib/components/LoadingBar.svelte';
-	import NoBaaBanner from '$lib/components/BAA/NoBaaBanner.svelte';
-	import HasBaaBanner from '$lib/components/BAA/HasBaaBanner.svelte';
+	import NoBaaBanner from '$lib/components/banners/NoBaaBanner.svelte';
+	import HasBaaBanner from '$lib/components/banners/HasBaaBanner.svelte';
 
 	export let title: string;
 	export let description: string;

src/lib/components/banners/BetaBanner.svelte

@@ -0,0 +1,9 @@
+<script lang="ts">
+	import BaseBanner from './BaseBanner.svelte';
+</script>
+
+<BaseBanner
+	title="Beta Feature"
+	message="This feature is currently in beta. If you encounter any issues, please contact [email protected]."
+	variant="warning"
+/>

src/lib/icons/DownloadIcon.svelte

@@ -0,0 +1 @@
+<i class={`fas fa-download ${$$props.class}`} />

src/lib/server/secrets.ts

@@ -8,6 +8,11 @@ export const AZURE_OPENAI_GPT_DEPLOYMENT_NAME = env.AZURE_OPENAI_GPT_DEPLOYMENT_
 export const AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME =
 	env.AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME || '';
 export const AZURE_OPENAI_WHISPER_DEPLOYMENT_NAME = env.AZURE_OPENAI_WHISPER_DEPLOYMENT_NAME || '';
+
+export const AZURE_SPEECH_SERVICES_ENDPOINT = env.AZURE_SPEECH_SERVICES_ENDPOINT || '';
+export const AZURE_SPEECH_SERVICES_KEY = env.AZURE_SPEECH_SERVICES_KEY || '';
+export const AZURE_CREATE_DIARIZATION_ENDPOINT = env.AZURE_CREATE_DIARIZATION_ENDPOINT || '';
+
 export const AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT = env.AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT || '';
 export const AZURE_DOCUMENT_INTELLIGENCE_KEY = env.AZURE_DOCUMENT_INTELLIGENCE_KEY || '';
 

src/lib/server/utils.ts

@@ -1,4 +1,7 @@
 import winston from 'winston';
+import ffmpeg from 'fluent-ffmpeg';
+import { diskFileToMemoryFile, memoryFileToDiskFile } from '$lib/fileHandling';
+import fs from 'fs';
 
 const { combine, timestamp, json } = winston.format;
 export const logger = winston.createLogger({
@@ -11,3 +14,33 @@ export const logger = winston.createLogger({
 	),
 	transports: [new winston.transports.Console()]
 });
+
+export async function convertToMp3(file: File): Promise<File> {
+	const fileExtension = file.name.split('.').pop();
+	if (fileExtension == 'mp3') return file;
+
+	const timestamp = new Date().getTime();
+
+	const inputName = `tempInputFile_${timestamp}.${fileExtension}`;
+	const convertedFileName = `temp_${timestamp}.mp3`;
+
+	try {
+		await memoryFileToDiskFile(file, inputName);
+		await new Promise<void>((resolve, reject) => {
+			ffmpeg(inputName)
+				.setStartTime(0)
+				.output(convertedFileName)
+				.on('error', (err) => {
+					logger.error(err);
+					reject();
+				})
+				.on('end', () => resolve())
+				.run();
+		});
+		const outputFile = diskFileToMemoryFile(convertedFileName, 'audio/mp3');
+		return outputFile;
+	} finally {
+		fs.unlinkSync(inputName);
+		fs.unlinkSync(convertedFileName);
+	}
+}

src/routes/api/diarize-retrieve/+server.ts

@@ -0,0 +1,48 @@
+import { AZURE_SPEECH_SERVICES_KEY } from '$lib/server/secrets';
+
+type TranscriptionLocationResponse = {
+	values: {
+		self: string;
+		name: string;
+		kind: 'Transcription' | 'TranscriptionReport';
+		properties: {
+			size: number;
+		};
+		createdDateTime: string;
+		links: {
+			contentUrl: string;
+		};
+	}[];
+};
+
+export async function GET({ request }) {
+	const url = request.headers.get('X-Transcript-Url');
+	if (url === null) {
+		return new Response('No valid url.', { status: 400 });
+	}
+
+	const transcriptLocationJson: TranscriptionLocationResponse = await (
+		await fetch(url, {
+			headers: {
+				'Ocp-Apim-Subscription-Key': AZURE_SPEECH_SERVICES_KEY
+			}
+		})
+	).json();
+
+	const urls = transcriptLocationJson.values
+		.filter((json) => json.kind === 'Transcription')
+		.map((json) => {
+			return { name: json.name, link: json.links.contentUrl };
+		});
+	const responses = await Promise.all(
+		urls.map(async (url) => {
+			return await fetch(url.link, {
+				headers: {
+					'Ocp-Apim-Subscription-Key': AZURE_SPEECH_SERVICES_KEY
+				}
+			});
+		})
+	);
+
+	return new Response(responses[0].body);
+}

src/routes/api/gpt/+server.ts

@@ -3,7 +3,7 @@ import type { Message } from '$lib/types';
 import { logger } from '$lib/server/utils';
 import { AZURE_OPENAI_GPT_DEPLOYMENT_NAME } from '$lib/server/secrets';
 
-export async function POST({ request }) {
+export async function POST({ request, locals }) {
 	const data = await request.json();
 	const messages = data.messages as Message[];
 	const azureOpenai = getAzureOpenAiClient();
@@ -16,12 +16,11 @@ export async function POST({ request }) {
 	}
 
 	const requestId = request.headers.get('X-Request-ID');
-	const user = request.headers.get('X-User');
 	logger.info({
 		type: 'OpenAI Request',
 		AZURE_OPENAI_GPT_DEPLOYMENT_NAME,
 		requestId,
-		user
+		user: locals.user
 	});
 
 	const completion = await azureOpenai.streamChatCompletions(

src/routes/api/text-to-speech/+server.ts

@@ -2,7 +2,7 @@ import OpenAI from 'openai';
 import { logger } from '$lib/server/utils';
 import { OPENAI_API_KEY } from '$lib/server/secrets';
 
-export async function POST({ request }) {
+export async function POST({ request, locals }) {
 	const openai = new OpenAI({ apiKey: OPENAI_API_KEY });
 
 	const formData = await request.formData();
@@ -16,12 +16,11 @@ export async function POST({ request }) {
 	}
 
 	const requestId = request.headers.get('X-Request-ID');
-	const user = request.headers.get('X-User');
 	logger.info({
 		type: 'OpenAI Request',
 		model,
 		requestId,
-		user
+		user: locals.user
 	});
 	const audioResponse = await openai.audio.speech.create({
 		input,

src/routes/dalle/+page.server.ts

@@ -21,12 +21,11 @@ export const actions = {
 
 		const model = 'dall-e-3';
 		const requestId = event.request.headers.get('X-Request-ID');
-		const user = event.request.headers.get('X-User');
 		logger.info({
 			type: 'OpenAI Request',
 			model,
 			requestId,
-			user
+			user: event.locals.user
 		});
 		const responses: APIPromise<ImagesResponse>[] = [];
 		for (let i = 0; i < number; i++) {

src/routes/dalle/+page.svelte

@@ -6,10 +6,7 @@
 
 	const maxImages = 4;
 	const title = 'Image Generation';
-	const description = `
-		This tool lets you create images with DALL-E 3. Simply write what you want to see and select the
-		desired size and number of images.
-	`;
+	const description = `This tool lets you create images with DALL-E 3. Simply write what you want to see and select the desired size and number of images.`;
 	const enhancer: SubmitFunction = () => {
 		return async ({ update }) => {
 			await update({ reset: false });

src/routes/diarize-retrieve/+page.server.ts

@@ -0,0 +1,24 @@
+import { AZURE_SPEECH_SERVICES_ENDPOINT, AZURE_SPEECH_SERVICES_KEY } from '$lib/server/secrets';
+import type { TranscriptionResponse } from './types';
+
+export async function load({ fetch, locals }) {
+	const transcriptions: TranscriptionResponse = await fetch(
+		AZURE_SPEECH_SERVICES_ENDPOINT + '/speechtotext/v3.2/transcriptions',
+		{
+			headers: {
+				'Ocp-Apim-Subscription-Key': AZURE_SPEECH_SERVICES_KEY
+			}
+		}
+	).then(async (resp) => await resp.json());
+	const userTranscriptions = transcriptions.values
+		.filter((transcript) => {
+			// As a bit of a user-management hack, the owner is stored in the display name.
+			return transcript.displayName.split('-!-')[0] === locals.user;
+		})
+		.map((transcript) => {
+			transcript.displayName = transcript.displayName.split('-!-')[1] ?? transcript.displayName;
+			return transcript;
+		})
+		.sort((a, b) => b.createdDateTime.localeCompare(a.createdDateTime));
+	return { values: userTranscriptions };
+}

src/routes/diarize-retrieve/+page.svelte

@@ -0,0 +1,62 @@
+<script lang="ts">
+	import BetaBanner from '$lib/components/banners/BetaBanner.svelte';
+	import HasBaaBanner from '$lib/components/banners/HasBaaBanner.svelte';
+	import { downloadBlob } from '$lib/fileHandling';
+	import DownloadIcon from '$lib/icons/DownloadIcon.svelte';
+	import type { Transcription, TranscriptionResponse } from './types';
+
+	export let data: TranscriptionResponse;
+	async function downloadFile(transcription: Transcription) {
+		const response = await fetch('/api/diarize-retrieve', {
+			headers: {
+				'X-Transcript-Url': transcription.links.files
+			}
+		});
+		const blob = await response.blob();
+		const fileparts = transcription.displayName.split('.');
+		fileparts.pop();
+		const filename = fileparts.join('') + '.json';
+		downloadBlob(blob, filename);
+	}
+</script>
+
+<div class="space-y-2">
+	<BetaBanner />
+	<HasBaaBanner />
+</div>
+<span>
+	Here you may find the results of your submitted diarizations. Please note that it can take up to a
+	few hours before your diarizations are completed. Diarization results are deleted after a week.
+</span>
+
+<div class="table-container">
+	<table class="table table-hover">
+		<thead>
+			<tr>
+				<th>Name</th>
+				<th>Created at (UTC)</th>
+				<th>Status</th>
+			</tr>
+		</thead>
+		<tbody>
+			{#each data.values as transcription}
+				<tr>
+					<td>{transcription.displayName}</td>
+					<td>{transcription.createdDateTime}</td>
+					<td>
+						{#if transcription.status === 'Succeeded'}
+							<button
+								class="btn btn-sm hover:variant-ghost-primary"
+								on:click={() => downloadFile(transcription)}
+							>
+								<DownloadIcon class="text-lg" />
+							</button>
+						{:else}
+							{transcription.status}
+						{/if}
+					</td>
+				</tr>
+			{/each}
+		</tbody>
+	</table>
+</div>