Skip to content

Commit

Permalink
feat(core): AWEL flow 2.0 backend code (#1879)
Browse files Browse the repository at this point in the history 
Co-authored-by: yhjun1026 <[email protected]>
  • Loading branch information
@fangyinc @yhjun1026
fangyinc and yhjun1026 authored Aug 23, 2024
1 parent 3a32344 commit 9502251
Show file tree
Hide file tree
Showing 67 changed files with 8,289 additions and 190 deletions.
11 changes: 11 additions & 0 deletions .env.template
View file
Original file line number Diff line number Diff line change
Expand Up @@ -272,6 +272,17 @@ DBGPT_LOG_LEVEL=INFO
# API_KEYS - The list of API keys that are allowed to access the API. Each of the below are an option, separated by commas.
# API_KEYS=dbgpt

#*******************************************************************#
#** ENCRYPT **#
#*******************************************************************#
# ENCRYPT KEY - The key used to encrypt and decrypt the data
# ENCRYPT_KEY=your_secret_key

#*******************************************************************#
#** File Server **#
#*******************************************************************#
## The local storage path of the file server, the default is pilot/data/file_server
# FILE_SERVER_LOCAL_STORAGE_PATH =

#*******************************************************************#
#** Application Config **#
Expand Down
3 changes: 3 additions & 0 deletions .mypy.ini
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,3 +119,6 @@ ignore_missing_imports = True

[mypy-networkx.*]
ignore_missing_imports = True

[mypy-pypdf.*]
ignore_missing_imports = True
129 changes: 116 additions & 13 deletions assets/schema/dbgpt.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,12 +32,14 @@ CREATE TABLE IF NOT EXISTS `knowledge_document`
`id` int NOT NULL AUTO_INCREMENT COMMENT 'auto increment id',
`doc_name` varchar(100) NOT NULL COMMENT 'document path name',
`doc_type` varchar(50) NOT NULL COMMENT 'doc type',
`doc_token` varchar(100) NOT NULL COMMENT 'doc token',
`space` varchar(50) NOT NULL COMMENT 'knowledge space',
`chunk_size` int NOT NULL COMMENT 'chunk size',
`last_sync` TIMESTAMP DEFAULT CURRENT_TIMESTAMP COMMENT 'last sync time',
`status` varchar(50) NOT NULL COMMENT 'status TODO,RUNNING,FAILED,FINISHED',
`content` LONGTEXT NOT NULL COMMENT 'knowledge embedding sync result',
`result` TEXT NULL COMMENT 'knowledge content',
`questions` TEXT NULL COMMENT 'document related questions',
`vector_ids` LONGTEXT NULL COMMENT 'vector_ids',
`summary` LONGTEXT NULL COMMENT 'knowledge summary',
`gmt_created` TIMESTAMP DEFAULT CURRENT_TIMESTAMP COMMENT 'created time',
Expand All @@ -53,6 +55,7 @@ CREATE TABLE IF NOT EXISTS `document_chunk`
`doc_type` varchar(50) NOT NULL COMMENT 'doc type',
`document_id` int NOT NULL COMMENT 'document parent id',
`content` longtext NOT NULL COMMENT 'chunk content',
`questions` text NULL COMMENT 'chunk related questions',
`meta_info` varchar(200) NOT NULL COMMENT 'metadata info',
`gmt_created` timestamp NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'created time',
`gmt_modified` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'update time',
Expand All @@ -61,7 +64,6 @@ CREATE TABLE IF NOT EXISTS `document_chunk`
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COMMENT='knowledge document chunk detail';



CREATE TABLE IF NOT EXISTS `connect_config`
(
`id` int NOT NULL AUTO_INCREMENT COMMENT 'autoincrement id',
Expand All @@ -74,6 +76,9 @@ CREATE TABLE IF NOT EXISTS `connect_config`
`db_pwd` varchar(255) DEFAULT NULL COMMENT 'db password',
`comment` text COMMENT 'db comment',
`sys_code` varchar(128) DEFAULT NULL COMMENT 'System code',
`user_name` varchar(255) DEFAULT NULL COMMENT 'user name',
`user_id` varchar(255) DEFAULT NULL COMMENT 'user id',

PRIMARY KEY (`id`),
UNIQUE KEY `uk_db` (`db_name`),
KEY `idx_q_db_type` (`db_type`)
Expand All @@ -88,11 +93,13 @@ CREATE TABLE IF NOT EXISTS `chat_history`
`user_name` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT 'interlocutor',
`messages` text COLLATE utf8mb4_unicode_ci COMMENT 'Conversation details',
`message_ids` text COLLATE utf8mb4_unicode_ci COMMENT 'Message id list, split by comma',
`app_code` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT 'App unique code',
`sys_code` varchar(128) DEFAULT NULL COMMENT 'System code',
`gmt_created` timestamp NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'created time',
`gmt_modified` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'update time',
UNIQUE KEY `conv_uid` (`conv_uid`),
PRIMARY KEY (`id`)
PRIMARY KEY (`id`),
KEY `idx_chat_his_app_code` (`app_code`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT 'Chat history';

CREATE TABLE IF NOT EXISTS `chat_history_message`
Expand All @@ -108,6 +115,7 @@ CREATE TABLE IF NOT EXISTS `chat_history_message`
PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT 'Chat history message';


CREATE TABLE IF NOT EXISTS `chat_feed_back`
(
`id` bigint(20) NOT NULL AUTO_INCREMENT,
Expand All @@ -118,6 +126,11 @@ CREATE TABLE IF NOT EXISTS `chat_feed_back`
`question` longtext DEFAULT NULL COMMENT 'User question',
`knowledge_space` varchar(128) DEFAULT NULL COMMENT 'Knowledge space name',
`messages` longtext DEFAULT NULL COMMENT 'The details of user feedback',
`message_id` varchar(255) NULL COMMENT 'Message id',
`feedback_type` varchar(50) NULL COMMENT 'Feedback type like or unlike',
`reason_types` varchar(255) NULL COMMENT 'Feedback reason categories',
`remark` text NULL COMMENT 'Feedback remark',
`user_code` varchar(128) NULL COMMENT 'User code',
`user_name` varchar(128) DEFAULT NULL COMMENT 'User name',
`gmt_created` timestamp NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'created time',
`gmt_modified` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'update time',
Expand Down Expand Up @@ -167,17 +180,20 @@ CREATE TABLE IF NOT EXISTS `plugin_hub`
CREATE TABLE IF NOT EXISTS `prompt_manage`
(
`id` int(11) NOT NULL AUTO_INCREMENT,
`chat_scene` varchar(100) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT 'Chat scene',
`sub_chat_scene` varchar(100) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT 'Sub chat scene',
`prompt_type` varchar(100) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT 'Prompt type: common or private',
`prompt_name` varchar(256) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT 'prompt name',
`content` longtext COLLATE utf8mb4_unicode_ci COMMENT 'Prompt content',
`input_variables` varchar(1024) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT 'Prompt input variables(split by comma))',
`model` varchar(128) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT 'Prompt model name(we can use different models for different prompt)',
`prompt_language` varchar(32) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT 'Prompt language(eg:en, zh-cn)',
`prompt_format` varchar(32) COLLATE utf8mb4_unicode_ci DEFAULT 'f-string' COMMENT 'Prompt format(eg: f-string, jinja2)',
`prompt_desc` varchar(512) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT 'Prompt description',
`user_name` varchar(128) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT 'User name',
`chat_scene` varchar(100) DEFAULT NULL COMMENT 'Chat scene',
`sub_chat_scene` varchar(100) DEFAULT NULL COMMENT 'Sub chat scene',
`prompt_type` varchar(100) DEFAULT NULL COMMENT 'Prompt type: common or private',
`prompt_name` varchar(256) DEFAULT NULL COMMENT 'prompt name',
`prompt_code` varchar(256) DEFAULT NULL COMMENT 'prompt code',
`content` longtext COMMENT 'Prompt content',
`input_variables` varchar(1024) DEFAULT NULL COMMENT 'Prompt input variables(split by comma))',
`response_schema` text DEFAULT NULL COMMENT 'Prompt response schema',
`model` varchar(128) DEFAULT NULL COMMENT 'Prompt model name(we can use different models for different prompt)',
`prompt_language` varchar(32) DEFAULT NULL COMMENT 'Prompt language(eg:en, zh-cn)',
`prompt_format` varchar(32) DEFAULT 'f-string' COMMENT 'Prompt format(eg: f-string, jinja2)',
`prompt_desc` varchar(512) DEFAULT NULL COMMENT 'Prompt description',
`user_code` varchar(128) DEFAULT NULL COMMENT 'User code',
`user_name` varchar(128) DEFAULT NULL COMMENT 'User name',
`sys_code` varchar(128) DEFAULT NULL COMMENT 'System code',
`gmt_created` timestamp NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'created time',
`gmt_modified` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'update time',
Expand All @@ -186,6 +202,8 @@ CREATE TABLE IF NOT EXISTS `prompt_manage`
KEY `gmt_created_idx` (`gmt_created`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='Prompt management table';



CREATE TABLE IF NOT EXISTS `gpts_conversations` (
`id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'autoincrement id',
`conv_id` varchar(255) NOT NULL COMMENT 'The unique id of the conversation record',
Expand Down Expand Up @@ -232,11 +250,15 @@ CREATE TABLE `gpts_messages` (
`receiver` varchar(255) NOT NULL COMMENT 'Who receive message in the current conversation turn',
`model_name` varchar(255) DEFAULT NULL COMMENT 'message generate model',
`rounds` int(11) NOT NULL COMMENT 'dialogue turns',
`is_success` int(4) NULL DEFAULT 0 COMMENT 'agent message is success',
`app_code` varchar(255) NOT NULL COMMENT 'Current AI assistant code',
`app_name` varchar(255) NOT NULL COMMENT 'Current AI assistant name',
`content` text COMMENT 'Content of the speech',
`current_goal` text COMMENT 'The target corresponding to the current message',
`context` text COMMENT 'Current conversation context',
`review_info` text COMMENT 'Current conversation review info',
`action_report` text COMMENT 'Current conversation action report',
`resource_info` text DEFAULT NULL COMMENT 'Current conversation resource info',
`role` varchar(255) DEFAULT NULL COMMENT 'The role of the current message content',
`created_at` datetime DEFAULT NULL COMMENT 'create time',
`updated_at` datetime DEFAULT NULL COMMENT 'last update time',
Expand Down Expand Up @@ -286,6 +308,7 @@ CREATE TABLE `dbgpt_serve_flow` (
`define_type` varchar(32) null comment 'Flow define type(json or python)',
`label` varchar(128) DEFAULT NULL COMMENT 'Flow label',
`editable` int DEFAULT NULL COMMENT 'Editable, 0: editable, 1: not editable',
`variables` text DEFAULT NULL COMMENT 'Flow variables, JSON format',
PRIMARY KEY (`id`),
UNIQUE KEY `uk_uid` (`uid`),
KEY `ix_dbgpt_serve_flow_sys_code` (`sys_code`),
Expand All @@ -295,6 +318,51 @@ CREATE TABLE `dbgpt_serve_flow` (
KEY `ix_dbgpt_serve_flow_name` (`name`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;

-- dbgpt.dbgpt_serve_file definition
CREATE TABLE `dbgpt_serve_file` (
`id` int NOT NULL AUTO_INCREMENT COMMENT 'Auto increment id',
`bucket` varchar(255) NOT NULL COMMENT 'Bucket name',
`file_id` varchar(255) NOT NULL COMMENT 'File id',
`file_name` varchar(256) NOT NULL COMMENT 'File name',
`file_size` int DEFAULT NULL COMMENT 'File size',
`storage_type` varchar(32) NOT NULL COMMENT 'Storage type',
`storage_path` varchar(512) NOT NULL COMMENT 'Storage path',
`uri` varchar(512) NOT NULL COMMENT 'File URI',
`custom_metadata` text DEFAULT NULL COMMENT 'Custom metadata, JSON format',
`file_hash` varchar(128) DEFAULT NULL COMMENT 'File hash',
`user_name` varchar(128) DEFAULT NULL COMMENT 'User name',
`sys_code` varchar(128) DEFAULT NULL COMMENT 'System code',
`gmt_created` datetime DEFAULT CURRENT_TIMESTAMP COMMENT 'Record creation time',
`gmt_modified` datetime DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Record update time',
PRIMARY KEY (`id`),
UNIQUE KEY `uk_bucket_file_id` (`bucket`, `file_id`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;

-- dbgpt.dbgpt_serve_variables definition
CREATE TABLE `dbgpt_serve_variables` (
`id` int NOT NULL AUTO_INCREMENT COMMENT 'Auto increment id',
`key` varchar(128) NOT NULL COMMENT 'Variable key',
`name` varchar(128) DEFAULT NULL COMMENT 'Variable name',
`label` varchar(128) DEFAULT NULL COMMENT 'Variable label',
`value` text DEFAULT NULL COMMENT 'Variable value, JSON format',
`value_type` varchar(32) DEFAULT NULL COMMENT 'Variable value type(string, int, float, bool)',
`category` varchar(32) DEFAULT 'common' COMMENT 'Variable category(common or secret)',
`encryption_method` varchar(32) DEFAULT NULL COMMENT 'Variable encryption method(fernet, simple, rsa, aes)',
`salt` varchar(128) DEFAULT NULL COMMENT 'Variable salt',
`scope` varchar(32) DEFAULT 'global' COMMENT 'Variable scope(global,flow,app,agent,datasource,flow_priv,agent_priv, ""etc)',
`scope_key` varchar(256) DEFAULT NULL COMMENT 'Variable scope key, default is empty, for scope is "flow_priv", the scope_key is dag id of flow',
`enabled` int DEFAULT 1 COMMENT 'Variable enabled, 0: disabled, 1: enabled',
`description` text DEFAULT NULL COMMENT 'Variable description',
`user_name` varchar(128) DEFAULT NULL COMMENT 'User name',
`sys_code` varchar(128) DEFAULT NULL COMMENT 'System code',
`gmt_created` datetime DEFAULT CURRENT_TIMESTAMP COMMENT 'Record creation time',
`gmt_modified` datetime DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Record update time',
PRIMARY KEY (`id`),
KEY `ix_your_table_name_key` (`key`),
KEY `ix_your_table_name_name` (`name`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;


-- dbgpt.gpts_app definition
CREATE TABLE `gpts_app` (
`id` int NOT NULL AUTO_INCREMENT COMMENT 'autoincrement id',
Expand All @@ -309,6 +377,9 @@ CREATE TABLE `gpts_app` (
`created_at` datetime DEFAULT NULL COMMENT 'create time',
`updated_at` datetime DEFAULT NULL COMMENT 'last update time',
`icon` varchar(1024) DEFAULT NULL COMMENT 'app icon, url',
`published` varchar(64) DEFAULT 'false' COMMENT 'Has it been published?',
`param_need` text DEFAULT NULL COMMENT 'Parameter information supported by the application',
`admins` text DEFAULT NULL COMMENT 'administrator',
PRIMARY KEY (`id`),
UNIQUE KEY `uk_gpts_app` (`app_name`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
Expand Down Expand Up @@ -363,6 +434,38 @@ CREATE TABLE IF NOT EXISTS `dbgpt_cluster_registry_instance` (
UNIQUE KEY `uk_model_instance` (`model_name`, `host`, `port`, `sys_code`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COMMENT='Cluster model instance table, for registering and managing model instances';

-- dbgpt.recommend_question definition
CREATE TABLE `recommend_question` (
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'autoincrement id',
`gmt_create` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time',
`gmt_modified` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'last update time',
`app_code` varchar(255) DEFAULT NULL COMMENT 'Current AI assistant code',
`question` text DEFAULT NULL COMMENT 'question',
`user_code` int(11) NOT NULL COMMENT 'user code',
`sys_code` varchar(255) NOT NULL COMMENT 'system app code',
`valid` varchar(10) DEFAULT 'true' COMMENT 'is it effective，true/false',
`chat_mode` varchar(255) DEFAULT NULL COMMENT 'Conversation scene mode，chat_knowledge...',
`params` text DEFAULT NULL COMMENT 'question param',
`is_hot_question` varchar(10) DEFAULT 'false' COMMENT 'Is it a popular recommendation question?',
PRIMARY KEY (`id`),
KEY `idx_rec_q_app_code` (`app_code`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT="AI application related recommendation issues";

-- dbgpt.user_recent_apps definition
CREATE TABLE `user_recent_apps` (
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'autoincrement id',
`gmt_create` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time',
`gmt_modified` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'last update time',
`app_code` varchar(255) DEFAULT NULL COMMENT 'AI assistant code',
`last_accessed` timestamp NULL DEFAULT NULL COMMENT 'User recent usage time',
`user_code` varchar(255) DEFAULT NULL COMMENT 'user code',
`sys_code` varchar(255) DEFAULT NULL COMMENT 'system app code',
PRIMARY KEY (`id`),
KEY `idx_user_r_app_code` (`app_code`),
KEY `idx_last_accessed` (`last_accessed`),
KEY `idx_user_code` (`user_code`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='User recently used apps'


CREATE
DATABASE IF NOT EXISTS EXAMPLE_1;
Expand Down
Loading

0 comments on commit 9502251

Please sign in to comment.