Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add ai knowledge base table and entity #215

Open
wants to merge 4 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
CREATE TABLE IF NOT EXISTS number_pool(
`id` INT PRIMARY KEY AUTO_INCREMENT,
`number` VARCHAR(48) NOT NULL,
`is_used` TINYINT(1) Default 1 NOT NULL,
`created_date` DATETIME(3) NOT NULL
)CHARSET=utf8mb4;

CREATE TABLE IF NOT EXISTS ai_speech_assistant_knowledge(
`id` INT PRIMARY KEY AUTO_INCREMENT,
`assistant_id` INT NOT NULL,
`json` TEXT NOT NULL,
`prompt` TEXT NOT NULL,
`version` VARCHAR(128) NOT NULL,
`is_active` TINYINT(1) Default 1 NOT NULL,
`created_date` DATETIME(3) NOT NULL,
`created_by` VARCHAR(255) NOT NULL
)CHARSET=utf8mb4;

CREATE TABLE IF NOT EXISTS ai_speech_assistant_greeting(
`id` int PRIMARY KEY AUTO_INCREMENT,
`assistant_id` int NOT NULL,
`text` text NOT NULL,
`version` varchar(128) NOT NULL,
`is_active` tinyint(1) DEFAULT 1 NOT NULL,
`created_date` datetime(3) NOT NULL,
`created_by` varchar(255) NOT NULL
) CHARSET=utf8mb4;
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
ALTER TABLE ai_speech_assistant
CHANGE COLUMN did_number answering_number VARCHAR(48) NULL,
CHANGE COLUMN url model_url VARCHAR(255) NULL,
CHANGE COLUMN voice model_voice VARCHAR(36) NULL,
CHANGE COLUMN provider model_provider VARCHAR(255) NULL,
ADD COLUMN answering_number_id INT NOT NULL,
ADD COLUMN created_by VARCHAR(255) NULL;

ALTER TABLE ai_speech_assistant DROP COLUMN greetings, DROP COLUMN custom_record_analyze_prompt;
30 changes: 15 additions & 15 deletions src/SmartTalk.Core/Domain/AISpeechAssistant/AiSpeechAssistant.cs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
using System.ComponentModel.DataAnnotations;
using SmartTalk.Messages.Enums.AiSpeechAssistant;
using System.ComponentModel.DataAnnotations.Schema;
using SmartTalk.Messages.Enums.AiSpeechAssistant;

namespace SmartTalk.Core.Domain.AISpeechAssistant;

Expand All @@ -15,27 +15,27 @@ public class AiSpeechAssistant : IEntity, IHasCreatedFields
[Column("name"), StringLength(255)]
public string Name { get; set; }

[Column("did_number"), StringLength(32)]
public string DidNumber { get; set; }
[Column("answering_number_id")]
public int AnsweringNumberId { get; set; }

[Column("answering_number")]
public string AnsweringNumber { get; set; }

[Column("url"), StringLength(512)]
public string Url { get; set; }
[Column("model_url")]
public string ModelUrl { get; set; }

[Column("voice"), StringLength(36)]
public string Voice { get; set; }
[Column("model_provider")]
public AiSpeechAssistantProvider ModelProvider { get; set; }

[Column("provider")]
public AiSpeechAssistantProvider Provider { get; set; }
[Column("model_voice")]
public string ModelVoice { get; set; }

[Column("agent_id")]
public int AgentId { get; set; }

[Column("greetings"), StringLength(1024)]
public string Greetings { get; set; }

[Column("custom_record_analyze_prompt")]
public string CustomRecordAnalyzePrompt { get; set; }

[Column("created_date")]
public DateTimeOffset CreatedDate { get; set; }

[Column("created_by")]
public string CreatedBy { get; set; }
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
using System.ComponentModel.DataAnnotations;
using System.ComponentModel.DataAnnotations.Schema;

namespace SmartTalk.Core.Domain.AIKnowledgeBase;

[Table("ai_speech_assistant_greeting")]
public class AiSpeechAssistantGreeting : IEntity, IHasCreatedFields
{
[Key]
[Column("id")]
[DatabaseGenerated(DatabaseGeneratedOption.Identity)]
public int Id { get; set; }

[Column("assistant_id")]
public int AssistantId { get; set; }

[Column("text")]
public string Text { get; set; }

[Column("version")]
public string Version { get; set; }

[Column("is_active", TypeName = "tinyint(1)")]
public bool IsActive { get; set; }

[Column("created_date")]
public DateTimeOffset CreatedDate { get; set; }

[Column("created_by")]
public string CreatedBy { get; set; }
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
using System.ComponentModel.DataAnnotations;
using System.ComponentModel.DataAnnotations.Schema;

namespace SmartTalk.Core.Domain.AIKnowledgeBase;

[Table("ai_speech_assistant_knowledge")]
public class AiSpeechAssistantKnowledge : IEntity, IHasCreatedFields
{
[Key]
[Column("id")]
[DatabaseGenerated(DatabaseGeneratedOption.Identity)]
public int Id { get; set; }

[Column("assistant_id")]
public int AssistantId { get; set; }

[Column("json")]
public string Json { get; set; }

[Column("prompt")]
public string Prompt { get; set; }

[Column("version")]
public string Version { get; set; }

[Column("is_active", TypeName = "tinyint(1)")]
public bool IsActive { get; set; }

[Column("created_date")]
public DateTimeOffset CreatedDate { get; set; }

[Column("created_by")]
public string CreatedBy { get; set; }
}
22 changes: 22 additions & 0 deletions src/SmartTalk.Core/Domain/AISpeechAssistant/NumberPool.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
using System.ComponentModel.DataAnnotations;
using System.ComponentModel.DataAnnotations.Schema;

namespace SmartTalk.Core.Domain.AIKnowledgeBase;

[Table("number_pool")]
public class NumberPool : IEntity, IHasCreatedFields
{
[Key]
[Column("id")]
[DatabaseGenerated(DatabaseGeneratedOption.Identity)]
public int Id { get; set; }

[Column("number")]
public string Number { get; set; }

[Column("is_used", TypeName = "tinyint(1)")]
public bool IsUsed { get; set; }

[Column("created_date")]
public DateTimeOffset CreatedDate { get; set; }
}
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ from userProfile in userProfileGroup.DefaultIfEmpty()
assistant, promptTemplate, userProfile
};

assistantInfo = assistantInfo.Where(x => assistantId.HasValue ? x.assistant.Id == assistantId.Value : x.assistant.DidNumber == didNumber);
assistantInfo = assistantInfo.Where(x => assistantId.HasValue ? x.assistant.Id == assistantId.Value : x.assistant.AnsweringNumber == didNumber);

var result = await assistantInfo.FirstOrDefaultAsync(cancellationToken: cancellationToken).ConfigureAwait(false);

Expand All @@ -52,7 +52,7 @@ from userProfile in userProfileGroup.DefaultIfEmpty()

public async Task<Domain.AISpeechAssistant.AiSpeechAssistant> GetAiSpeechAssistantByNumbersAsync(string didNumber, CancellationToken cancellationToken)
{
return await _repository.Query<Domain.AISpeechAssistant.AiSpeechAssistant>().Where(x => x.DidNumber == didNumber)
return await _repository.Query<Domain.AISpeechAssistant.AiSpeechAssistant>().Where(x => x.AnsweringNumber == didNumber)
.FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@
using SmartTalk.Core.Constants;
using Microsoft.AspNetCore.Http;
using OpenAI.Chat;
using SmartTalk.Core.Services.Agents;
using SmartTalk.Core.Services.Http;
using SmartTalk.Core.Services.Http.Clients;
using SmartTalk.Messages.Constants;
Expand All @@ -29,8 +28,6 @@
using SmartTalk.Messages.Enums.AiSpeechAssistant;
using SmartTalk.Messages.Events.AiSpeechAssistant;
using SmartTalk.Messages.Commands.AiSpeechAssistant;
using SmartTalk.Messages.Commands.PhoneOrder;
using SmartTalk.Messages.Dto.Agent;
using SmartTalk.Messages.Enums.Agent;
using SmartTalk.Messages.Enums.PhoneOrder;
using JsonSerializer = System.Text.Json.JsonSerializer;
Expand Down Expand Up @@ -157,6 +154,8 @@ public async Task ReceivePhoneRecordingStatusCallbackAsync(ReceivePhoneRecording

var (record, agent, aiSpeechAssistant) = await _phoneOrderDataProvider.GetRecordWithAgentAndAssistantAsync(command.CallSid, cancellationToken).ConfigureAwait(false);

var knowledge = await _phoneOrderDataProvider.GetKnowledgePromptByAssistantIdAsync(aiSpeechAssistant.Id, cancellationToken).ConfigureAwait(false);

Log.Information("Get phone order record: {@record}", record);

record.Url = command.RecordingUrl;
Expand All @@ -167,9 +166,9 @@ public async Task ReceivePhoneRecordingStatusCallbackAsync(ReceivePhoneRecording
var audioData = BinaryData.FromBytes(audioFileRawBytes);
List<ChatMessage> messages =
[
new SystemChatMessage(string.IsNullOrEmpty(aiSpeechAssistant?.CustomRecordAnalyzePrompt)
new SystemChatMessage(string.IsNullOrEmpty(knowledge?.Prompt)
? "你是一名電話錄音的分析員,通過聽取錄音內容和語氣情緒作出精確分析,冩出一份分析報告。\n\n分析報告的格式:交談主題:xxx\n\n 內容摘要:xxx \n\n 客人情感與情緒: xxx \n\n 待辦事件: \n1.xxx\n2.xxx \n\n 客人下單內容(如果沒有則忽略):1. 牛肉(1箱)\n2.雞腿肉(1箱)"
: aiSpeechAssistant.CustomRecordAnalyzePrompt),
: knowledge.Prompt),
new UserChatMessage(ChatMessageContentPart.CreateInputAudioPart(audioData, ChatInputAudioFormat.Wav)),
new UserChatMessage("幫我根據錄音生成分析報告:")
];
Expand Down Expand Up @@ -237,7 +236,7 @@ private async Task<WebSocket> ConnectOpenAiRealTimeSocketAsync(Domain.AISpeechAs
openAiWebSocket.Options.SetRequestHeader("Authorization", GetAuthorizationHeader(assistant));
openAiWebSocket.Options.SetRequestHeader("OpenAI-Beta", "realtime=v1");

var url = string.IsNullOrEmpty(assistant.Url) ? AiSpeechAssistantStore.DefaultUrl : assistant.Url;
var url = string.IsNullOrEmpty(assistant.ModelUrl) ? AiSpeechAssistantStore.DefaultUrl : assistant.ModelUrl;

await openAiWebSocket.ConnectAsync(new Uri(url), cancellationToken).ConfigureAwait(false);

Expand All @@ -248,11 +247,11 @@ private async Task<WebSocket> ConnectOpenAiRealTimeSocketAsync(Domain.AISpeechAs

private string GetAuthorizationHeader(Domain.AISpeechAssistant.AiSpeechAssistant assistant)
{
return assistant.Provider switch
return assistant.ModelProvider switch
{
AiSpeechAssistantProvider.OpenAi => $"Bearer {_openAiSettings.ApiKey}",
AiSpeechAssistantProvider.ZhiPuAi => $"Bearer {_zhiPuAiSettings.ApiKey}",
_ => throw new NotSupportedException(nameof(assistant.Provider))
_ => throw new NotSupportedException(nameof(assistant.ModelProvider))
};
}

Expand Down Expand Up @@ -739,7 +738,7 @@ private async Task SendSessionUpdateAsync(WebSocket openAiWebSocket, Domain.AISp
turn_detection = new { type = "server_vad" },
input_audio_format = "g711_ulaw",
output_audio_format = "g711_ulaw",
voice = string.IsNullOrEmpty(assistant.Voice) ? "alloy" : assistant.Voice,
voice = string.IsNullOrEmpty(assistant.ModelVoice) ? "alloy" : assistant.ModelVoice,
instructions = prompt,
modalities = new[] { "text", "audio" },
temperature = 0.8,
Expand Down
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
using Microsoft.EntityFrameworkCore;
using SmartTalk.Core.Domain.Account;
using SmartTalk.Core.Domain.AIKnowledgeBase;
using SmartTalk.Core.Domain.PhoneOrder;
using SmartTalk.Core.Domain.Restaurants;
using SmartTalk.Core.Domain.System;
using SmartTalk.Messages.Dto.Agent;
using SmartTalk.Messages.Dto.PhoneOrder;
using SmartTalk.Messages.Dto.Restaurant;
using SmartTalk.Messages.Enums.PhoneOrder;
Expand Down Expand Up @@ -35,6 +35,8 @@ Task<List<GetPhoneOrderRecordsWithUserCountDto>> GetPhoneOrderRecordsWithUserCou
Task<PhoneOrderRecord> GetPhoneOrderRecordBySessionIdAsync(string sessionId, CancellationToken cancellationToken);

Task<(PhoneOrderRecord, Agent, Domain.AISpeechAssistant.AiSpeechAssistant)> GetRecordWithAgentAndAssistantAsync(string sessionId, CancellationToken cancellationToken);

Task<AiSpeechAssistantKnowledge> GetKnowledgePromptByAssistantIdAsync(int assistantId, CancellationToken cancellationToken);
}

public partial class PhoneOrderDataProvider
Expand Down Expand Up @@ -194,4 +196,9 @@ from assistant in assistantGroup.DefaultIfEmpty()

return (result?.record, result?.agent, result?.assistant);
}

public async Task<AiSpeechAssistantKnowledge> GetKnowledgePromptByAssistantIdAsync(int assistantId, CancellationToken cancellationToken)
{
return await _repository.Query<AiSpeechAssistantKnowledge>().Where(x=>x.AssistantId == assistantId).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false);
}
}