Skip to content

Commit

Permalink
Merge branch 'master' into feat/opentelemetry
Browse files Browse the repository at this point in the history
  • Loading branch information
yuki-takei committed Dec 27, 2024
2 parents dc02ae6 + 675b6f1 commit ee1cc49
Show file tree
Hide file tree
Showing 17 changed files with 474 additions and 148 deletions.
19 changes: 18 additions & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,9 +1,26 @@
# Changelog

## [Unreleased](https://github.com/weseek/growi/compare/v7.1.5...HEAD)
## [Unreleased](https://github.com/weseek/growi/compare/v7.1.6...HEAD)

*Please do not manually update this file. We've automated the process.*

## [v7.1.6](https://github.com/weseek/growi/compare/v7.1.5...v7.1.6) - 2024-12-26

### 💎 Features

* feat(ai): Save file to VectorStore in HTML format (#9462) @miya

### 🐛 Bug Fixes

* fix: remark-lsx pagination (#9513) @miya
* fix: Spelling miss of external_link in i18n (#9456) @reiji-h
* fix: Wider copy to clipboard area (#9450) @Ryosei-Fukushima
* fix: Error when creating pages with deep hierarchy (#9487) @reiji-h

### 🧰 Maintenance

* ci(deps): bump next from 14.2.13 to 14.2.15 (#9501) @dependabot

## [v7.1.5](https://github.com/weseek/growi/compare/v7.1.4...v7.1.5) - 2024-12-13

### 🚀 Improvement
Expand Down
1 change: 1 addition & 0 deletions apps/app/bin/swagger-jsdoc/definition-apiv3.js
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@ module.exports = {
'UserGroups',
'Users Management',
'FullTextSearch Management',
'Install',
],
},
{
Expand Down
6 changes: 4 additions & 2 deletions apps/app/package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@growi/app",
"version": "7.1.6-RC.0",
"version": "7.1.7-RC.0",
"license": "MIT",
"private": "true",
"scripts": {
Expand Down Expand Up @@ -165,7 +165,7 @@
"multer": "~1.4.0",
"multer-autoreap": "^1.0.3",
"mustache": "^4.2.0",
"next": "^14.2.13",
"next": "^14.2.15",
"next-dynamic-loading-props": "^0.1.1",
"next-i18next": "^15.3.1",
"next-superjson": "^0.0.4",
Expand Down Expand Up @@ -205,9 +205,11 @@
"reconnecting-websocket": "^4.4.0",
"redis": "^3.0.2",
"rehype-katex": "^7.0.1",
"rehype-meta": "^4.0.1",
"rehype-raw": "^7.0.0",
"rehype-sanitize": "^6.0.0",
"rehype-slug": "^6.0.0",
"rehype-stringify": "^10.0.1",
"rehype-toc": "^3.0.2",
"remark-breaks": "^4.0.0",
"remark-directive": "^3.0.0",
Expand Down
4 changes: 2 additions & 2 deletions apps/app/public/static/locales/en_US/translation.json
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,7 @@
"Create under": "Create page under below:",
"V5 Page Migration": "Convert To V5 Compatibility",
"GROWI.5.0_new_schema": "GROWI.5.0 new schema",
"See_more_detail_on_new_schema": "See more detail on <a href='https://docs.growi.org/en/admin-guide/upgrading/50x.html#about-the-new-v5-compatible-format' target='_blank'>{{title}}</a> <span className='growi-custom-icons'>external_link</span> ",
"See_more_detail_on_new_schema": "See more detail on <a href='https://docs.growi.org/en/admin-guide/upgrading/50x.html#about-the-new-v5-compatible-format' target='_blank'>{{title}}</a> <span class='growi-custom-icons'>external_link</span> ",
"external_account_management": "External Account Management",
"UserGroup": "UserGroup",
"Basic Settings": "Basic Settings",
Expand Down Expand Up @@ -615,7 +615,7 @@
"alert_desc1": "On this page, you can select pages with the checkbox and batch convert to the new v5 compatible format from the \"Bulk operation\" button at the top of the screen.",
"nopages_title": "Congratulations. Ready to use GROWI v5!",
"nopages_desc1": "Now all the pages you can manage seem to be in v5 compatible format.",
"detail_info": "See the detail information from <a href='https://docs.growi.org/en/admin-guide/upgrading/50x.html' target='_blank' class='alert-link'>Upgrading GROWI to v5.0.x <span className='growi-custom-icons'>external_link</span></a>.",
"detail_info": "See the detail information from <a href='https://docs.growi.org/en/admin-guide/upgrading/50x.html' target='_blank' class='alert-link'>Upgrading GROWI to v5.0.x <span class='growi-custom-icons'>external_link</span></a>.",
"modal": {
"title": "Convert to new v5 compatible format",
"converting_pages": "Converting pages",
Expand Down
4 changes: 2 additions & 2 deletions apps/app/public/static/locales/fr_FR/translation.json
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,7 @@
"Create under": "Créer la page sous:",
"V5 Page Migration": "Convertir vers la V5",
"GROWI.5.0_new_schema": "Nouveau schéma GROWI.5.0",
"See_more_detail_on_new_schema": "Plus de détails sur <a href='https://docs.growi.org/en/admin-guide/upgrading/50x.html#about-the-new-v5-compatible-format' target='_blank'>{{title}}</a> <i class='icon-share-alt'></i> ",
"See_more_detail_on_new_schema": "Plus de détails sur <a href='https://docs.growi.org/en/admin-guide/upgrading/50x.html#about-the-new-v5-compatible-format' target='_blank'>{{title}}</a><span class='growi-custom-icons'>external_link</span> ",
"external_account_management": "Gestion des comptes externes",
"UserGroup": "Groupe utilisateur",
"Basic Settings": "Paramètres de base",
Expand Down Expand Up @@ -608,7 +608,7 @@
"alert_desc1": "Sélectionner les pages à convertir vers le format V5 avec le bouton \"Opération de masse\".",
"nopages_title": "GROWI V5 est maintenant utilisable!",
"nopages_desc1": "Toutes les pages ont été converties au format V5.",
"detail_info": "Pour plus de détails, voir <a href='https://docs.growi.org/en/admin-guide/upgrading/50x.html' target='_blank' class='alert-link'>Convertir vers GROWI v5.0.x <span className='growi-custom-icons'>external_link</span></a>.",
"detail_info": "Pour plus de détails, voir <a href='https://docs.growi.org/en/admin-guide/upgrading/50x.html' target='_blank' class='alert-link'>Convertir vers GROWI v5.0.x <span class='growi-custom-icons'>external_link</span></a>.",
"modal": {
"title": "Convertir au format V5",
"converting_pages": "Conversion des pages",
Expand Down
4 changes: 2 additions & 2 deletions apps/app/public/static/locales/ja_JP/translation.json
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@
"Create under": "ページを以下に作成",
"V5 Page Migration": "V5 互換形式 への変換",
"GROWI.5.0_new_schema": "GROWI.5.0における新スキーマについて",
"See_more_detail_on_new_schema": "詳しくは<a href='https://docs.growi.org/ja/admin-guide/upgrading/50x.html#新しい-v5-互換形式について' target='_blank'>{{title}}</a><span className='growi-custom-icons'>external_link</span>を参照ください。",
"See_more_detail_on_new_schema": "詳しくは<a href='https://docs.growi.org/ja/admin-guide/upgrading/50x.html#新しい-v5-互換形式について' target='_blank'>{{title}}</a><span class='growi-custom-icons'>external_link</span>を参照ください。",
"external_account_management": "外部アカウント管理",
"UserGroup": "グループ",
"Basic Settings": "基本設定",
Expand Down Expand Up @@ -647,7 +647,7 @@
"alert_desc1": "このページでは、チェックボックスでページを選択し、画面上部の「一括操作」ボタンから新しい v5 互換形式に一括変換できます。",
"nopages_title": "おめでとうございます。GROWI v5 を使う準備が完了しました!",
"nopages_desc1": "今あなたが管理可能なページはすべて v5 互換形式になっているようです。",
"detail_info": "詳しくは <a href='https://docs.growi.org/ja/admin-guide/upgrading/50x.html' target='_blank' class='alert-link'>GROWI v5.0.x へのアップグレード <span className='growi-custom-icons'>external_link</span></a> を参照ください。",
"detail_info": "詳しくは <a href='https://docs.growi.org/ja/admin-guide/upgrading/50x.html' target='_blank' class='alert-link'>GROWI v5.0.x へのアップグレード <span class='growi-custom-icons'>external_link</span></a> を参照ください。",
"modal": {
"title": "新しい v5 互換形式への変換",
"converting_pages": "以下のページを変換します",
Expand Down
4 changes: 2 additions & 2 deletions apps/app/public/static/locales/zh_CN/translation.json
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@
"Create under": "Create page under below:",
"V5 Page Migration": "转换为V5的兼容性",
"GROWI.5.0_new_schema": "GROWI.5.0 new schema",
"See_more_detail_on_new_schema": "更多详情请见<a href='https://docs.growi.org/en/admin-guide/upgrading/50x.html#about-the-new-v5-compatible-format' target='_blank'> {{title}}</a> <span className='growi-custom-icons'>external_link</span> ",
"See_more_detail_on_new_schema": "更多详情请见<a href='https://docs.growi.org/en/admin-guide/upgrading/50x.html#about-the-new-v5-compatible-format' target='_blank'> {{title}}</a> <span class='growi-custom-icons'>external_link</span> ",
"Markdown Settings": "Markdown设置",
"external_account_management": "外部账户管理",
"UserGroup": "用户组",
Expand Down Expand Up @@ -617,7 +617,7 @@
"alert_desc1": "在这一页,你可以用复选框选择页面,并通过屏幕上方的批量操作按钮批量转换为新的v5兼容格式。",
"nopages_title": "恭喜你。准备使用GROWI v5!",
"nopages_desc1": "现在你能管理的所有页面似乎都是v5兼容的格式。",
"detail_info": "请参见 <a href='https://docs.growi.org/en/admin-guide/upgrading/50x.html' target='_blank' class='alert-link'>升级GROWI到v5.0.x <span className='growi-custom-icons'>external_link</span></a>.的详细内容。",
"detail_info": "请参见 <a href='https://docs.growi.org/en/admin-guide/upgrading/50x.html' target='_blank' class='alert-link'>升级GROWI到v5.0.x <span class='growi-custom-icons'>external_link</span></a>.的详细内容。",
"modal": {
"title": "转换为新的v5兼容格式",
"converting_pages": "转换页面",
Expand Down
15 changes: 8 additions & 7 deletions apps/app/src/features/openai/server/services/openai.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import assert from 'node:assert';
import { Readable, Transform } from 'stream';
import { pipeline } from 'stream/promises';

import type { IPagePopulatedToShowRevision } from '@growi/core';
import { PageGrant, isPopulated } from '@growi/core';
import type { HydratedDocument, Types } from 'mongoose';
import mongoose from 'mongoose';
Expand All @@ -20,7 +21,7 @@ import { createBatchStream } from '~/server/util/batch-stream';
import loggerFactory from '~/utils/logger';

import { OpenaiServiceTypes } from '../../interfaces/ai';
import { sanitizeMarkdown } from '../utils/sanitize-markdown';
import { convertMarkdownToHtml } from '../utils/convert-markdown-to-html';

import { getClient } from './client-delegator';
// import { splitMarkdownIntoChunks } from './markdown-splitter/markdown-token-splitter';
Expand Down Expand Up @@ -157,9 +158,9 @@ class OpenaiService implements IOpenaiService {
// }
// }

private async uploadFile(pageId: Types.ObjectId, body: string): Promise<OpenAI.Files.FileObject> {
const sanitizedMarkdown = await sanitizeMarkdown(body);
const file = await toFile(Readable.from(sanitizedMarkdown), `${pageId}.md`);
private async uploadFile(pageId: Types.ObjectId, pagePath: string, revisionBody: string): Promise<OpenAI.Files.FileObject> {
const convertedHtml = await convertMarkdownToHtml({ pagePath, revisionBody });
const file = await toFile(Readable.from(convertedHtml), `${pageId}.html`);
const uploadedFile = await this.client.uploadFile(file);
return uploadedFile;
}
Expand All @@ -183,17 +184,17 @@ class OpenaiService implements IOpenaiService {
async createVectorStoreFile(pages: Array<HydratedDocument<PageDocument>>): Promise<void> {
const vectorStore = await this.getOrCreateVectorStoreForPublicScope();
const vectorStoreFileRelationsMap: VectorStoreFileRelationsMap = new Map();
const processUploadFile = async(page: PageDocument) => {
const processUploadFile = async(page: HydratedDocument<PageDocument>) => {
if (page._id != null && page.grant === PageGrant.GRANT_PUBLIC && page.revision != null) {
if (isPopulated(page.revision) && page.revision.body.length > 0) {
const uploadedFile = await this.uploadFile(page._id, page.revision.body);
const uploadedFile = await this.uploadFile(page._id, page.path, page.revision.body);
prepareVectorStoreFileRelations(vectorStore._id, page._id, uploadedFile.id, vectorStoreFileRelationsMap);
return;
}

const pagePopulatedToShowRevision = await page.populateDataToShowRevision();
if (pagePopulatedToShowRevision.revision != null && pagePopulatedToShowRevision.revision.body.length > 0) {
const uploadedFile = await this.uploadFile(page._id, pagePopulatedToShowRevision.revision.body);
const uploadedFile = await this.uploadFile(page._id, page.path, pagePopulatedToShowRevision.revision.body);
prepareVectorStoreFileRelations(vectorStore._id, page._id, uploadedFile.id, vectorStoreFileRelationsMap);
}
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
import { dynamicImport } from '@cspell/dynamic-import';
import type { Root, Code } from 'mdast';
import type * as RehypeMeta from 'rehype-meta';
import type * as RehypeStringify from 'rehype-stringify';
import type * as RemarkParse from 'remark-parse';
import type * as RemarkRehype from 'remark-rehype';
import type * as Unified from 'unified';
import type * as UnistUtilVisit from 'unist-util-visit';

interface ModuleCache {
unified?: typeof Unified.unified;
visit?: typeof UnistUtilVisit.visit;
remarkParse?: typeof RemarkParse.default;
remarkRehype?: typeof RemarkRehype.default;
rehypeMeta?: typeof RehypeMeta.default;
rehypeStringify?: typeof RehypeStringify.default;
}

let moduleCache: ModuleCache = {};

const initializeModules = async(): Promise<void> => {
if (moduleCache.unified != null
&& moduleCache.visit != null
&& moduleCache.remarkParse != null
&& moduleCache.remarkRehype != null
&& moduleCache.rehypeMeta != null
&& moduleCache.rehypeStringify != null
) {
return;
}

const [
{ unified },
{ visit },
{ default: remarkParse },
{ default: remarkRehype },
{ default: rehypeMeta },
{ default: rehypeStringify },
] = await Promise.all([
dynamicImport<typeof Unified>('unified', __dirname),
dynamicImport<typeof UnistUtilVisit>('unist-util-visit', __dirname),
dynamicImport<typeof RemarkParse>('remark-parse', __dirname),
dynamicImport<typeof RemarkRehype>('remark-rehype', __dirname),
dynamicImport<typeof RehypeMeta>('rehype-meta', __dirname),
dynamicImport<typeof RehypeStringify>('rehype-stringify', __dirname),
]);

moduleCache = {
unified,
visit,
remarkParse,
remarkRehype,
rehypeMeta,
rehypeStringify,
};
};

export const convertMarkdownToHtml = async({ pagePath, revisionBody }: { pagePath: string, revisionBody: string }): Promise<string> => {
await initializeModules();

const {
unified, visit, remarkParse, remarkRehype, rehypeMeta, rehypeStringify,
} = moduleCache;

if (unified == null || visit == null || remarkParse == null || remarkRehype == null || rehypeMeta == null || rehypeStringify == null) {
throw new Error('Failed to initialize required modules');
}

const sanitizeMarkdown = () => {
return (tree: Root) => {
visit(tree, 'code', (node: Code) => {
if (node.lang === 'drawio') {
node.value = '<!-- drawio content replaced -->';
}
});
};
};

const processor = unified()
.use(remarkParse)
.use(sanitizeMarkdown)
.use(remarkRehype)
.use(rehypeMeta, {
title: pagePath,
})
.use(rehypeStringify);

return processor.processSync(revisionBody).toString();
};
65 changes: 0 additions & 65 deletions apps/app/src/features/openai/server/utils/sanitize-markdown.ts

This file was deleted.

42 changes: 42 additions & 0 deletions apps/app/src/server/routes/apiv3/installer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,48 @@ module.exports = (crowi: Crowi): Router => {

const minPasswordLength = configManager.getConfig('app:minPasswordLength');

/**
* @swagger
*
* /installer:
* post:
* tags: [Install]
* security: []
* operationId: Install
* summary: /installer
* description: Install GROWI
* requestBody:
* required: true
* content:
* application/json:
* schema:
* type: object
* properties:
* registerForm:
* type: object
* properties:
* name:
* type: string
* username:
* type: string
* email:
* type: string
* password:
* type: string
* app:globalLang:
* type: string
* default: en_US
* responses:
* 200:
* description: import settings params
* content:
* application/json:
* schema:
* properties:
* message:
* type: string
* example: Installation completed (Logged in as an admin user)
*/
// eslint-disable-next-line max-len
router.post('/', registerRules(minPasswordLength), registerValidation, addActivity, async(req: FormRequest, res: ApiV3Response) => {
const appService = crowi.appService;
Expand Down
Loading

0 comments on commit ee1cc49

Please sign in to comment.