From e4f1fd206be8d01289c829201c1c95c7bda2245b Mon Sep 17 00:00:00 2001 From: canisminor1990 Date: Tue, 3 Dec 2024 17:56:51 +0800 Subject: [PATCH] =?UTF-8?q?=F0=9F=94=A7=20chore:=20Add=20CDN=20workflow?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/.cdn.cache.json | 22 ++ docs/changelog/2024-07-19-gpt-4o-mini.mdx | 7 +- ...4-08-21-file-upload-and-knowledge-base.mdx | 9 +- docs/changelog/2024-09-20-artifacts.mdx | 7 +- docs/changelog/2024-10-27-pin-assistant.mdx | 4 +- .../2024-10-27-pin-assistant.zh-CN.mdx | 4 +- docs/changelog/2024-11-06-share-text-json.mdx | 4 +- .../2024-11-06-share-text-json.zh-CN.mdx | 4 +- docs/changelog/index.json | 32 +-- package.json | 4 + scripts/cdnWorkflow/index.ts | 219 ++++++++++++++++++ scripts/cdnWorkflow/optimized.ts | 21 ++ scripts/cdnWorkflow/s3/index.ts | 120 ++++++++++ scripts/cdnWorkflow/s3/types.ts | 25 ++ scripts/cdnWorkflow/s3/utils.ts | 106 +++++++++ scripts/cdnWorkflow/uploader.ts | 52 +++++ scripts/cdnWorkflow/utils.ts | 93 ++++++++ 17 files changed, 703 insertions(+), 30 deletions(-) create mode 100644 docs/.cdn.cache.json create mode 100644 scripts/cdnWorkflow/index.ts create mode 100644 scripts/cdnWorkflow/optimized.ts create mode 100644 scripts/cdnWorkflow/s3/index.ts create mode 100644 scripts/cdnWorkflow/s3/types.ts create mode 100644 scripts/cdnWorkflow/s3/utils.ts create mode 100644 scripts/cdnWorkflow/uploader.ts create mode 100644 scripts/cdnWorkflow/utils.ts diff --git a/docs/.cdn.cache.json b/docs/.cdn.cache.json new file mode 100644 index 000000000000..1eb2dc38b073 --- /dev/null +++ b/docs/.cdn.cache.json @@ -0,0 +1,22 @@ +{ + "https://github.com/user-attachments/assets/03433283-08a5-481a-8f6c-069b2fc6bace": "https://hub-apac-1.lobeobjects.space/blog/assets/8d4c2cc0ce8654fa8ac06cc036a7f941.webp", + "https://github.com/user-attachments/assets/0e3a7174-6b66-4432-a319-dff60b033c24": "https://hub-apac-1.lobeobjects.space/blog/assets/39d7890f8cbe21e77db8d3c94f7f22e4.webp", + "https://github.com/user-attachments/assets/2048b4c2-4a56-4029-acf9-71e35ff08652": "https://hub-apac-1.lobeobjects.space/blog/assets/d9cbfcbef130183bc490d515d8a38aa4.webp", + "https://github.com/user-attachments/assets/29508dda-2382-430f-bc81-fb23f02149f8": "https://hub-apac-1.lobeobjects.space/blog/assets/29b13dc042e3b839ad8865354afe2fac.webp", + "https://github.com/user-attachments/assets/2a4116a7-15ad-43e5-b801-cc62d8da2012": "https://hub-apac-1.lobeobjects.space/blog/assets/37d85fdfccff9ed56e9c6827faee01c7.webp", + "https://github.com/user-attachments/assets/385eaca6-daea-484a-9bea-ba7270b4753d": "https://hub-apac-1.lobeobjects.space/blog/assets/d6129350de510a62fe87b2d2f0fb9477.webp", + "https://github.com/user-attachments/assets/484f28f4-017c-4ed7-948b-4a8d51f0b63a": "https://hub-apac-1.lobeobjects.space/blog/assets/5bbb4b421d6df63780b3c7a05f5a102d.webp", + "https://github.com/user-attachments/assets/533f7a5e-8a93-4a57-a62f-8233897d72b5": "https://hub-apac-1.lobeobjects.space/blog/assets/9498087e85f27e692716a63cb3b58d79.webp", + "https://github.com/user-attachments/assets/6069332b-8e15-4d3c-8a77-479e8bc09c23": "https://hub-apac-1.lobeobjects.space/blog/assets/603fefbb944bc6761ebdab5956fc0084.webp", + "https://github.com/user-attachments/assets/635f1c74-6327-48a8-a8d9-68d7376c7749": "https://hub-apac-1.lobeobjects.space/blog/assets/f6d047a345e47a52592cff916c9a64ce.webp", + "https://github.com/user-attachments/assets/6935e155-4a1d-4ab7-a61a-2b813d65bb7b": "https://hub-apac-1.lobeobjects.space/blog/assets/6ee2609d79281b6b915e317461013f31.webp", + "https://github.com/user-attachments/assets/82bfc467-e0c6-4d99-9b1f-18e4aea24285": "https://hub-apac-1.lobeobjects.space/blog/assets/eb477e62217f4d1b644eff975c7ac168.webp", + "https://github.com/user-attachments/assets/aee846d5-b5ee-46cb-9dd0-d952ea708b67": "https://hub-apac-1.lobeobjects.space/blog/assets/8a8d361b4c0cce6da350cc0de65c0ad6.webp", + "https://github.com/user-attachments/assets/bd6d0c82-8f14-4167-ad09-2a841f1e34e4": "https://hub-apac-1.lobeobjects.space/blog/assets/d7e57f8e69f97b76b3c2414f3441b6e4.webp", + "https://github.com/user-attachments/assets/c68e88e4-cf2e-4122-82bc-89ba193b1eb4": "https://hub-apac-1.lobeobjects.space/blog/assets/1f6c4f1c5e6211735ca4924c7807aca1.webp", + "https://github.com/user-attachments/assets/dde2c9c5-cdda-4a65-8f32-b6f4da907df2": "https://hub-apac-1.lobeobjects.space/blog/assets/d47654360d626f80144cdedb979a3526.webp", + "https://github.com/user-attachments/assets/e70c2db6-05c9-43ea-b111-6f6f99e0ae88": "https://hub-apac-1.lobeobjects.space/blog/assets/944c671604833cd2457445b211ebba33.webp", + "https://github.com/user-attachments/assets/eaed3762-136f-4297-b161-ca92a27c4982": "https://hub-apac-1.lobeobjects.space/blog/assets/50b38eac1769ae6f13aef72f3d725eec.webp", + "https://github.com/user-attachments/assets/eb3f3d8a-79ce-40aa-a206-2c846206c0c0": "https://hub-apac-1.lobeobjects.space/blog/assets/f10a4b98782e36797c38071eed785c6f.webp", + "https://github.com/user-attachments/assets/fa8fab19-ace2-4f85-8428-a3a0e28845bb": "https://hub-apac-1.lobeobjects.space/blog/assets/2d678631c55369ba7d753c3ffcb73782.webp" +} diff --git a/docs/changelog/2024-07-19-gpt-4o-mini.mdx b/docs/changelog/2024-07-19-gpt-4o-mini.mdx index 0adc02528f55..b35fcc007dac 100644 --- a/docs/changelog/2024-07-19-gpt-4o-mini.mdx +++ b/docs/changelog/2024-07-19-gpt-4o-mini.mdx @@ -1,6 +1,9 @@ --- -title: LobeChat Fully Enters the GPT-4 Era: GPT-4o Mini Officially Launched -description: LobeChat v1.6 has been released with support for GPT-4o mini, while LobeChat Cloud services have been fully upgraded to provide users with a more powerful AI conversation experience. +title: 'LobeChat Fully Enters the GPT-4 Era: GPT-4o Mini Officially Launched' +description: >- + LobeChat v1.6 has been released with support for GPT-4o mini, while LobeChat + Cloud services have been fully upgraded to provide users with a more powerful + AI conversation experience. --- # GPT-4o Mini Makes a Stunning Debut, Ushering in a New GPT-4 Era 🚀 diff --git a/docs/changelog/2024-08-21-file-upload-and-knowledge-base.mdx b/docs/changelog/2024-08-21-file-upload-and-knowledge-base.mdx index d0ac5bf58183..12c998a05d49 100644 --- a/docs/changelog/2024-08-21-file-upload-and-knowledge-base.mdx +++ b/docs/changelog/2024-08-21-file-upload-and-knowledge-base.mdx @@ -1,6 +1,11 @@ --- -title: LobeChat Launches Knowledge Base Feature: A New Experience in Intelligent File Management and Dialogue -description: LobeChat introduces a brand new knowledge base feature that supports all types of file management, intelligent vectorization, and file dialogue, making knowledge management and information retrieval easier and smarter. +title: >- + LobeChat Launches Knowledge Base Feature: A New Experience in Intelligent File + Management and Dialogue +description: >- + LobeChat introduces a brand new knowledge base feature that supports all types + of file management, intelligent vectorization, and file dialogue, making + knowledge management and information retrieval easier and smarter. --- # Major Release of Knowledge Base Feature: A Revolution in Intelligent File Management and Dialogue diff --git a/docs/changelog/2024-09-20-artifacts.mdx b/docs/changelog/2024-09-20-artifacts.mdx index 534cdc33eaf9..5b8d601cc9ac 100644 --- a/docs/changelog/2024-09-20-artifacts.mdx +++ b/docs/changelog/2024-09-20-artifacts.mdx @@ -1,6 +1,9 @@ --- -title: Major Update: LobeChat Enters the Era of Artifacts -description: LobeChat v1.19 brings significant updates, including full feature support for Claude Artifacts, a brand new discovery page design, and support for GitHub Models providers, greatly enhancing the capabilities of the AI assistant. +title: 'Major Update: LobeChat Enters the Era of Artifacts' +description: >- + LobeChat v1.19 brings significant updates, including full feature support for + Claude Artifacts, a brand new discovery page design, and support for GitHub + Models providers, greatly enhancing the capabilities of the AI assistant. --- # Major Update: LobeChat Enters the Era of Artifacts diff --git a/docs/changelog/2024-10-27-pin-assistant.mdx b/docs/changelog/2024-10-27-pin-assistant.mdx index 9b13bba51427..f21f435545a5 100644 --- a/docs/changelog/2024-10-27-pin-assistant.mdx +++ b/docs/changelog/2024-10-27-pin-assistant.mdx @@ -16,9 +16,9 @@ In version v1.26.0, we are excited to introduce a long-awaited new feature — t - **Space Optimization**: Activating the sidebar automatically hides the conversation list, providing you with a larger conversation area. - **Intelligent Display**: Automatically syncs pinned assistants to the sidebar, ensuring that important assistants are always within view. -![Sidebar Display Effect](https://github.com/user-attachments/assets/6935e155-4a1d-4ab7-a61a-2b813d65bb7b) +![Sidebar Display Effect](https://hub-apac-1.lobeobjects.space/blog/assets/6ee2609d79281b6b915e317461013f31.webp) -![Conversation Interface Effect](https://github.com/user-attachments/assets/c68e88e4-cf2e-4122-82bc-89ba193b1eb4) +![Conversation Interface Effect](https://hub-apac-1.lobeobjects.space/blog/assets/1f6c4f1c5e6211735ca4924c7807aca1.webp) ## How to Use diff --git a/docs/changelog/2024-10-27-pin-assistant.zh-CN.mdx b/docs/changelog/2024-10-27-pin-assistant.zh-CN.mdx index 1b7f84ddd1b4..1cf298c05592 100644 --- a/docs/changelog/2024-10-27-pin-assistant.zh-CN.mdx +++ b/docs/changelog/2024-10-27-pin-assistant.zh-CN.mdx @@ -13,9 +13,9 @@ description: LobeChat v1.26.0 推出助手常驻侧边栏功能，支持快捷 - **空间优化**：激活侧边栏时会自动隐藏会话列表，为您腾出更大的对话空间 - **智能显示**：将置顶助手自动同步到侧边栏，让重要助手始终在视线范围内 -![侧边栏展示效果](https://github.com/user-attachments/assets/6935e155-4a1d-4ab7-a61a-2b813d65bb7b) +![侧边栏展示效果](https://hub-apac-1.lobeobjects.space/blog/assets/6ee2609d79281b6b915e317461013f31.webp) -![对话界面效果](https://github.com/user-attachments/assets/c68e88e4-cf2e-4122-82bc-89ba193b1eb4) +![对话界面效果](https://hub-apac-1.lobeobjects.space/blog/assets/1f6c4f1c5e6211735ca4924c7807aca1.webp) ## 如何使用 diff --git a/docs/changelog/2024-11-06-share-text-json.mdx b/docs/changelog/2024-11-06-share-text-json.mdx index 8fed3949d87b..458421215dad 100644 --- a/docs/changelog/2024-11-06-share-text-json.mdx +++ b/docs/changelog/2024-11-06-share-text-json.mdx @@ -13,11 +13,11 @@ In the latest version v1.28.0, we have launched the text format export feature f The Markdown export feature meets users' needs for directly using conversation content in note-taking and document writing. You can easily save valuable conversation content and manage it across various note-taking applications for reuse. -![Exporting Conversations as Markdown Text](https://github.com/user-attachments/assets/29508dda-2382-430f-bc81-fb23f02149f8) +![Exporting Conversations as Markdown Text](https://hub-apac-1.lobeobjects.space/blog/assets/29b13dc042e3b839ad8865354afe2fac.webp) Additionally, we support exporting conversations in JSON format that complies with OpenAI messages specifications. This format can be used directly for API debugging and serves as high-quality training data for models. -![Exporting Conversations as JSON in OpenAI API Specification](https://github.com/user-attachments/assets/484f28f4-017c-4ed7-948b-4a8d51f0b63a) +![Exporting Conversations as JSON in OpenAI API Specification](https://hub-apac-1.lobeobjects.space/blog/assets/5bbb4b421d6df63780b3c7a05f5a102d.webp) It is particularly noteworthy that we retain the original data of Tools Calling within the conversation, which is crucial for enhancing the model's tool invocation capabilities. diff --git a/docs/changelog/2024-11-06-share-text-json.zh-CN.mdx b/docs/changelog/2024-11-06-share-text-json.zh-CN.mdx index dfc222da5d0f..0c8add008c8a 100644 --- a/docs/changelog/2024-11-06-share-text-json.zh-CN.mdx +++ b/docs/changelog/2024-11-06-share-text-json.zh-CN.mdx @@ -11,11 +11,11 @@ description: >- Markdown 格式导出功能满足了用户将对话内容直接用于笔记和文档撰写的需求。您可以轻松地将有价值的对话内容保存下来，并在各类笔记软件中进行管理和复用。 -![将对话导出为 Markdown 格式文本](https://github.com/user-attachments/assets/29508dda-2382-430f-bc81-fb23f02149f8) +![将对话导出为 Markdown 格式文本](https://hub-apac-1.lobeobjects.space/blog/assets/29b13dc042e3b839ad8865354afe2fac.webp) 同时，我们还支持将对话导出为符合 OpenAI messages 规范的 JSON 格式。这种格式不仅可以直接用于 API 调试，还能作为高质量的模型训练语料。 -![将对话导出为 OpenAI 接口规范的 JSON](https://github.com/user-attachments/assets/484f28f4-017c-4ed7-948b-4a8d51f0b63a) +![将对话导出为 OpenAI 接口规范的 JSON](https://hub-apac-1.lobeobjects.space/blog/assets/5bbb4b421d6df63780b3c7a05f5a102d.webp) 特别值得一提的是，我们会完整保留对话中的 Tools Calling 原始数据，这对提升模型的工具调用能力具有重要价值。 diff --git a/docs/changelog/index.json b/docs/changelog/index.json index ab2662c7fa6a..dfddfc07021e 100644 --- a/docs/changelog/index.json +++ b/docs/changelog/index.json @@ -3,97 +3,97 @@ "cloud": [], "community": [ { - "image": "https://github.com/user-attachments/assets/2048b4c2-4a56-4029-acf9-71e35ff08652", + "image": "https://hub-apac-1.lobeobjects.space/blog/assets/d9cbfcbef130183bc490d515d8a38aa4.webp", "id": "2024-11-27-forkable-chat", "date": "2024-11-27", "versionRange": ["1.34.0", "1.33.1"] }, { - "image": "https://github.com/user-attachments/assets/fa8fab19-ace2-4f85-8428-a3a0e28845bb", + "image": "https://hub-apac-1.lobeobjects.space/blog/assets/2d678631c55369ba7d753c3ffcb73782.webp", "id": "2024-11-25-november-providers", "date": "2024-11-25", "versionRange": ["1.33.0", "1.30.1"] }, { - "image": "https://github.com/user-attachments/assets/eb3f3d8a-79ce-40aa-a206-2c846206c0c0", + "image": "https://hub-apac-1.lobeobjects.space/blog/assets/f10a4b98782e36797c38071eed785c6f.webp", "id": "2024-11-06-share-text-json", "date": "2024-11-06", "versionRange": ["1.28.0", "1.26.1"] }, { - "image": "https://github.com/user-attachments/assets/e70c2db6-05c9-43ea-b111-6f6f99e0ae88", + "image": "https://hub-apac-1.lobeobjects.space/blog/assets/944c671604833cd2457445b211ebba33.webp", "id": "2024-10-27-pin-assistant", "date": "2024-10-27", "versionRange": ["1.26.0", "1.19.1"] }, { - "image": "https://github.com/user-attachments/assets/635f1c74-6327-48a8-a8d9-68d7376c7749", + "image": "https://hub-apac-1.lobeobjects.space/blog/assets/f6d047a345e47a52592cff916c9a64ce.webp", "id": "2024-09-20-artifacts", "date": "2024-09-20", "versionRange": ["1.19.0", "1.17.1"] }, { - "image": "https://github.com/user-attachments/assets/bd6d0c82-8f14-4167-ad09-2a841f1e34e4", + "image": "https://hub-apac-1.lobeobjects.space/blog/assets/d7e57f8e69f97b76b3c2414f3441b6e4.webp", "id": "2024-09-13-openai-o1-models", "date": "2024-09-13", "versionRange": ["1.17.0", "1.12.1"] }, { - "image": "https://github.com/user-attachments/assets/385eaca6-daea-484a-9bea-ba7270b4753d", + "image": "https://hub-apac-1.lobeobjects.space/blog/assets/d6129350de510a62fe87b2d2f0fb9477.webp", "id": "2024-08-21-file-upload-and-knowledge-base", "date": "2024-08-21", "versionRange": ["1.12.0", "1.8.1"] }, { - "image": "https://github.com/user-attachments/assets/2a4116a7-15ad-43e5-b801-cc62d8da2012", + "image": "https://hub-apac-1.lobeobjects.space/blog/assets/37d85fdfccff9ed56e9c6827faee01c7.webp", "id": "2024-08-02-lobe-chat-database-docker", "date": "2024-08-02", "versionRange": ["1.8.0", "1.6.1"] }, { - "image": "https://github.com/user-attachments/assets/0e3a7174-6b66-4432-a319-dff60b033c24", + "image": "https://hub-apac-1.lobeobjects.space/blog/assets/39d7890f8cbe21e77db8d3c94f7f22e4.webp", "id": "2024-07-19-gpt-4o-mini", "date": "2024-07-19", "versionRange": ["1.6.0", "1.0.1"] }, { - "image": "https://github.com/user-attachments/assets/82bfc467-e0c6-4d99-9b1f-18e4aea24285", + "image": "https://hub-apac-1.lobeobjects.space/blog/assets/eb477e62217f4d1b644eff975c7ac168.webp", "id": "2024-06-19-lobe-chat-v1", "date": "2024-06-19", "versionRange": ["1.0.0", "0.147.0"] }, { - "image": "https://github.com/user-attachments/assets/aee846d5-b5ee-46cb-9dd0-d952ea708b67", + "image": "https://hub-apac-1.lobeobjects.space/blog/assets/8a8d361b4c0cce6da350cc0de65c0ad6.webp", "id": "2024-02-14-ollama", "date": "2024-02-14", "versionRange": ["0.127.0", "0.125.1"] }, { - "image": "https://github.com/user-attachments/assets/533f7a5e-8a93-4a57-a62f-8233897d72b5", + "image": "https://hub-apac-1.lobeobjects.space/blog/assets/9498087e85f27e692716a63cb3b58d79.webp", "id": "2024-02-08-sso-oauth", "date": "2024-02-08", "versionRange": ["0.125.0", "0.118.1"] }, { - "image": "https://github.com/user-attachments/assets/6069332b-8e15-4d3c-8a77-479e8bc09c23", + "image": "https://hub-apac-1.lobeobjects.space/blog/assets/603fefbb944bc6761ebdab5956fc0084.webp", "id": "2023-12-22-dalle-3", "date": "2023-12-22", "versionRange": ["0.118.0", "0.102.1"] }, { - "image": "https://github.com/user-attachments/assets/03433283-08a5-481a-8f6c-069b2fc6bace", + "image": "https://hub-apac-1.lobeobjects.space/blog/assets/8d4c2cc0ce8654fa8ac06cc036a7f941.webp", "id": "2023-11-19-tts-stt", "date": "2023-11-19", "versionRange": ["0.102.0", "0.101.1"] }, { - "image": "https://github.com/user-attachments/assets/dde2c9c5-cdda-4a65-8f32-b6f4da907df2", + "image": "https://hub-apac-1.lobeobjects.space/blog/assets/d47654360d626f80144cdedb979a3526.webp", "id": "2023-11-14-gpt4-vision", "date": "2023-11-14", "versionRange": ["0.101.0", "0.90.0"] }, { - "image": "https://github.com/user-attachments/assets/eaed3762-136f-4297-b161-ca92a27c4982", + "image": "https://hub-apac-1.lobeobjects.space/blog/assets/50b38eac1769ae6f13aef72f3d725eec.webp", "id": "2023-09-09-plugin-system", "date": "2023-09-09", "versionRange": ["0.72.0", "0.67.0"] diff --git a/package.json b/package.json index 57ca568ccb3e..4e926fb3e24d 100644 --- a/package.json +++ b/package.json @@ -65,6 +65,7 @@ "test:update": "vitest -u", "type-check": "tsc --noEmit", "webhook:ngrok": "ngrok http http://localhost:3011", + "workflow:cdn": "tsx ./scripts/cdnWorkflow/index.ts", "workflow:changelog": "tsx ./scripts/changelogWorkflow/index.ts", "workflow:countCharters": "tsx scripts/countEnWord.ts", "workflow:docs": "tsx ./scripts/docsWorkflow/index.ts", @@ -252,6 +253,7 @@ "@testing-library/jest-dom": "^6.6.2", "@testing-library/react": "^16.0.1", "@types/chroma-js": "^2.4.4", + "@types/crypto-js": "^4.2.2", "@types/debug": "^4.1.12", "@types/diff": "^6.0.0", "@types/fs-extra": "^11.0.4", @@ -275,6 +277,7 @@ "ajv-keywords": "^5.1.0", "commitlint": "^19.5.0", "consola": "^3.2.3", + "crypto-js": "^4.2.0", "dotenv": "^16.4.5", "dpdm-fast": "^1.0.4", "drizzle-kit": "^0.29.0", @@ -292,6 +295,7 @@ "lodash": "^4.17.21", "markdown-table": "^3.0.3", "markdown-to-txt": "^2.0.1", + "mime": "^4.0.4", "node-fetch": "^3.3.2", "node-gyp": "^10.2.0", "openapi-typescript": "^6.7.6", diff --git a/scripts/cdnWorkflow/index.ts b/scripts/cdnWorkflow/index.ts new file mode 100644 index 000000000000..1ae896b4c78b --- /dev/null +++ b/scripts/cdnWorkflow/index.ts @@ -0,0 +1,219 @@ +import { consola } from 'consola'; +import { writeJSONSync } from 'fs-extra'; +import matter from 'gray-matter'; +import { createHash } from 'node:crypto'; +import { readFileSync, writeFileSync } from 'node:fs'; +import { resolve } from 'node:path'; +import pMap from 'p-map'; + +import { uploader } from './uploader'; +import { + changelogIndex, + changelogIndexPath, + extractHttpsLinks, + fetchImageAsFile, + mergeAndDeduplicateArrays, + posts, + root, +} from './utils'; + +// 定义常量 +const GITHUB_CDN = 'https://github.com/lobehub/lobe-chat/assets/'; +const CHECK_CDN = [ + 'https://cdn.nlark.com/yuque/0/', + 'https://s.imtccdn.com/', + 'https://oss.home.imtc.top/', + 'https://www.anthropic.com/_next/image', + 'https://miro.medium.com/v2/', + 'https://images.unsplash.com/', + 'https://github.com/user-attachments/assets', +]; + +const CACHE_FILE = resolve(root, 'docs', '.cdn.cache.json'); + +class ImageCDNUploader { + private cache: { [link: string]: string } = {}; + + constructor() { + this.loadCache(); + } + + // 从文件加载缓存数据 + private loadCache() { + try { + this.cache = JSON.parse(readFileSync(CACHE_FILE, 'utf8')); + } catch (error) { + consola.error('Failed to load cache', error); + } + } + + // 将缓存数据写入文件 + private writeCache() { + try { + writeFileSync(CACHE_FILE, JSON.stringify(this.cache, null, 2)); + } catch (error) { + consola.error('Failed to write cache', error); + } + } + + // 收集所有的图片链接 + private collectImageLinks(): string[] { + const links: string[][] = posts.map((post) => { + const mdx = readFileSync(post, 'utf8'); + const { content, data } = matter(mdx); + let inlineLinks: string[] = extractHttpsLinks(content); + + // 添加特定字段中的图片链接 + if (data?.image) inlineLinks.push(data.image); + if (data?.seo?.image) inlineLinks.push(data.seo.image); + + // 过滤出有效的 CDN 链接 + return inlineLinks.filter( + (link) => + (link.startsWith(GITHUB_CDN) || CHECK_CDN.some((cdn) => link.startsWith(cdn))) && + !this.cache[link], + ); + }); + + const communityLinks = changelogIndex.community + .map((post) => post.image) + .filter( + (link) => + link && + (link.startsWith(GITHUB_CDN) || CHECK_CDN.some((cdn) => link.startsWith(cdn))) && + !this.cache[link], + ) as string[]; + + const cloudLinks = changelogIndex.cloud + .map((post) => post.image) + .filter( + (link) => + link && + (link.startsWith(GITHUB_CDN) || CHECK_CDN.some((cdn) => link.startsWith(cdn))) && + !this.cache[link], + ) as string[]; + + // 合并和去重链接数组 + return mergeAndDeduplicateArrays(links.flat().concat(communityLinks, cloudLinks)); + } + + // 上传图片到 CDN + private async uploadImagesToCDN(links: string[]) { + const cdnLinks: { [link: string]: string } = {}; + + await pMap(links, async (link) => { + consola.start('Uploading image to CDN', link); + const file = await fetchImageAsFile(link, 1600); + + if (!file) { + consola.error('Failed to fetch image as file', link); + return; + } + + const cdnUrl = await this.uploadFileToCDN(file, link); + if (cdnUrl) { + consola.success(link, '>>>', cdnUrl); + cdnLinks[link] = cdnUrl; + } + }); + + // 更新缓存 + this.cache = { ...this.cache, ...cdnLinks }; + this.writeCache(); + } + + // 根据不同的 CDN 来处理文件上传 + private async uploadFileToCDN(file: File, link: string): Promise { + if (link.startsWith(GITHUB_CDN)) { + const filename = link.replaceAll(GITHUB_CDN, ''); + return uploader(file, filename); + } else if (CHECK_CDN.some((cdn) => link.startsWith(cdn))) { + const buffer = await file.arrayBuffer(); + const hash = createHash('md5').update(Buffer.from(buffer)).digest('hex'); + return uploader(file, hash); + } + + return; + } + + // 替换文章中的图片链接 + private replaceLinksInPosts() { + let count = 0; + + for (const post of posts) { + const mdx = readFileSync(post, 'utf8'); + let { content, data } = matter(mdx); + const inlineLinks = extractHttpsLinks(content); + + for (const link of inlineLinks) { + if (this.cache[link]) { + content = content.replaceAll(link, this.cache[link]); + count++; + } + } + + // 更新特定字段的图片链接 + + if (data['image'] && this.cache[data['image']]) { + data['image'] = this.cache[data['image']]; + count++; + } + + if (data['seo']?.['image'] && this.cache[data['seo']?.['image']]) { + data['seo']['image'] = this.cache[data['seo']['image']]; + count++; + } + + writeFileSync(post, matter.stringify(content, data)); + } + + consola.success(`${count} images have been uploaded to CDN and links have been replaced`); + } + + private replaceLinksInChangelogIndex() { + let count = 0; + changelogIndex.community = changelogIndex.community.map((post) => { + if (!post.image) return post; + count++; + return { + ...post, + image: this.cache[post.image] || post.image, + }; + }); + + changelogIndex.cloud = changelogIndex.cloud.map((post) => { + if (!post.image) return post; + count++; + return { + ...post, + image: this.cache[post.image] || post.image, + }; + }); + + writeJSONSync(changelogIndexPath, changelogIndex, { spaces: 2 }); + + consola.success( + `${count} changelog index images have been uploaded to CDN and links have been replaced`, + ); + } + + // 运行上传过程 + async run() { + const links = this.collectImageLinks(); + + if (links.length > 0) { + consola.info("Found images that haven't been uploaded to CDN:"); + consola.info(links); + await this.uploadImagesToCDN(links); + this.replaceLinksInPosts(); + this.replaceLinksInChangelogIndex(); + } else { + consola.info('No new images to upload.'); + } + } +} + +// 实例化并运行 +const instance = new ImageCDNUploader(); + +instance.run(); diff --git a/scripts/cdnWorkflow/optimized.ts b/scripts/cdnWorkflow/optimized.ts new file mode 100644 index 000000000000..6cfcd3a2ca02 --- /dev/null +++ b/scripts/cdnWorkflow/optimized.ts @@ -0,0 +1,21 @@ +import sharp from 'sharp'; + +const WIDTH = 1600; + +export const opimized = async ( + inputBuffer: ArrayBuffer, + width: number = WIDTH, +): Promise => { + return await sharp(inputBuffer) + .resize({ width: width, withoutEnlargement: true }) + .webp() + .toBuffer(); +}; + +export const opimizedGif = async (inputBuffer: ArrayBuffer): Promise => { + try { + return await sharp(inputBuffer, { animated: true }).webp().toBuffer(); + } catch { + return await sharp(inputBuffer, { animated: true, limitInputPixels: false }).webp().toBuffer(); + } +}; diff --git a/scripts/cdnWorkflow/s3/index.ts b/scripts/cdnWorkflow/s3/index.ts new file mode 100644 index 000000000000..2b3436a580ea --- /dev/null +++ b/scripts/cdnWorkflow/s3/index.ts @@ -0,0 +1,120 @@ +import { + GetObjectCommand, + PutObjectCommand, + PutObjectCommandOutput, + S3Client, + S3ClientConfig, +} from '@aws-sdk/client-s3'; +import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; + +import type { ImgInfo, S3UserConfig, UploadResult } from './types'; +import { extractInfo } from './utils'; + +async function getFileURL( + opts: createUploadTaskOpts, + eTag: string, + versionId: string, +): Promise { + try { + const signedUrl = await getSignedUrl( + opts.client, + new GetObjectCommand({ + Bucket: opts.bucketName, + IfMatch: eTag, + Key: opts.path, + VersionId: versionId, + }), + { expiresIn: 3600 }, + ); + const urlObject = new URL(signedUrl); + urlObject.search = ''; + return urlObject.href; + } catch (error) { + // eslint-disable-next-line unicorn/no-useless-promise-resolve-reject + return Promise.reject(error); + } +} + +function createS3Client(opts: S3UserConfig): S3Client { + const clientOptions: S3ClientConfig = { + credentials: { + accessKeyId: opts.accessKeyId, + secretAccessKey: opts.secretAccessKey, + }, + + endpoint: opts.endpoint || undefined, + forcePathStyle: opts.pathStyleAccess, + region: opts.region || 'auto', + }; + + const client = new S3Client(clientOptions); + return client; +} + +interface createUploadTaskOpts { + acl: string; + bucketName: string; + client: S3Client; + item: ImgInfo; + path: string; + urlPrefix?: string; +} + +async function createUploadTask(opts: createUploadTaskOpts): Promise { + if (!opts.item.buffer) { + throw new Error('undefined image'); + } + + let body: Buffer; + let contentType: string; + let contentEncoding: string; + + try { + ({ body, contentType, contentEncoding } = (await extractInfo(opts.item)) as any); + } catch (error) { + // eslint-disable-next-line unicorn/no-useless-promise-resolve-reject + return Promise.reject(error); + } + + const command = new PutObjectCommand({ + ACL: opts.acl as any, + Body: body, + Bucket: opts.bucketName, + ContentEncoding: contentEncoding, + ContentType: contentType, + Key: opts.path, + }); + + let output: PutObjectCommandOutput; + try { + output = await opts.client.send(command); + } catch (error) { + // eslint-disable-next-line unicorn/no-useless-promise-resolve-reject + return Promise.reject(error); + } + + let url: string; + if (opts.urlPrefix) { + url = `${opts.urlPrefix}/${opts.path}`; + } else { + try { + url = await getFileURL(opts, output.ETag as string, output.VersionId as string); + } catch (error) { + // eslint-disable-next-line unicorn/no-useless-promise-resolve-reject + return Promise.reject(error); + } + } + + return { + eTag: output.ETag, + imgURL: url, + key: opts.path, + url: url, + versionId: output.VersionId, + }; +} + +export default { + createS3Client, + createUploadTask, +}; diff --git a/scripts/cdnWorkflow/s3/types.ts b/scripts/cdnWorkflow/s3/types.ts new file mode 100644 index 000000000000..61d0c41f7cc5 --- /dev/null +++ b/scripts/cdnWorkflow/s3/types.ts @@ -0,0 +1,25 @@ +export interface ImgInfo { + [propName: string]: any; + buffer: Buffer; + extname: string; + fileName: string; +} + +export interface S3UserConfig { + accessKeyId: string; + bucketName: string; + endpoint: string; + pathPrefix: string; + pathStyleAccess?: boolean; + region: string; + secretAccessKey: string; + uploadPath?: string; +} + +export interface UploadResult { + eTag?: string; + imgURL: string; + key: string; + url: string; + versionId?: string; +} diff --git a/scripts/cdnWorkflow/s3/utils.ts b/scripts/cdnWorkflow/s3/utils.ts new file mode 100644 index 000000000000..69c62776b88b --- /dev/null +++ b/scripts/cdnWorkflow/s3/utils.ts @@ -0,0 +1,106 @@ +import CryptoJS from 'crypto-js'; +import mime from 'mime'; + +import { ImgInfo } from './types'; + +class FileNameGenerator { + date: Date; + info: ImgInfo; + + static fields = [ + 'year', + 'month', + 'day', + 'fullName', + 'fileName', + 'extName', + 'timestamp', + 'timestampMS', + 'md5', + ]; + + constructor(info: ImgInfo) { + this.date = new Date(); + this.info = info; + } + + public year(): string { + return `${this.date.getFullYear()}`; + } + + public month(): string { + return this.date.getMonth() < 9 + ? `0${this.date.getMonth() + 1}` + : `${this.date.getMonth() + 1}`; + } + + public day(): string { + return this.date.getDate() < 9 ? `0${this.date.getDate()}` : `${this.date.getDate()}`; + } + + public fullName(): string { + return this.info.fileName; + } + + public fileName(): string { + return this.info.fileName.replace(this.info.extname, ''); + } + + public extName(): string { + return this.info.extname.replace('.', ''); + } + + public timestamp(): string { + return Math.floor(Date.now() / 1000).toString(); + } + + public timestampMS(): string { + return Date.now().toString(); + } + + public md5(): string { + const wordArray = CryptoJS.lib.WordArray.create(this.imgBuffer()); + const md5Hash = CryptoJS.MD5(wordArray); + return md5Hash.toString(CryptoJS.enc.Hex); + } + private imgBuffer(): Buffer { + return this.info.buffer; + } +} + +export function formatPath(info: ImgInfo, format?: string): string { + if (!format) { + return info.fileName; + } + + const fileNameGenerator = new FileNameGenerator(info); + + let formatPath: string = format; + + for (const key of FileNameGenerator.fields) { + const re = new RegExp(`{${key}}`, 'g'); + // @ts-ignore + formatPath = formatPath.replace(re, fileNameGenerator[key]()); + } + + return formatPath; +} + +export async function extractInfo(info: ImgInfo): Promise<{ + body?: Buffer; + contentEncoding?: string; + contentType?: string; +}> { + const result: { + body?: Buffer; + contentEncoding?: string; + contentType?: string; + } = {}; + + if (info.extname) { + result.contentType = mime.getType(info.extname) as string; + } + result.body = info.buffer; + + return result; +} diff --git a/scripts/cdnWorkflow/uploader.ts b/scripts/cdnWorkflow/uploader.ts new file mode 100644 index 000000000000..8543697630a1 --- /dev/null +++ b/scripts/cdnWorkflow/uploader.ts @@ -0,0 +1,52 @@ +import { consola } from 'consola'; +import dotenv from 'dotenv'; + +import s3 from './s3'; +import type { ImgInfo, S3UserConfig, UploadResult } from './s3/types'; +import { formatPath } from './s3/utils'; + +dotenv.config(); +export const uploader = async ( + file: File, + filename: string, + basePath: string = 'blog/assets/', + uploadPath?: string, +) => { + const item: ImgInfo = { + buffer: Buffer.from(await file.arrayBuffer()), + extname: file.name.split('.').pop() as string, + fileName: file.name, + mimeType: file.type, + }; + + const userConfig: S3UserConfig = { + accessKeyId: process.env.S3_ACCESS_KEY_ID || '', + bucketName: 'hub-apac-1', + endpoint: 'https://d35842305b91be4b48e06ff9a9ad83f5.r2.cloudflarestorage.com', + pathPrefix: 'https://hub-apac-1.lobeobjects.space', + pathStyleAccess: true, + region: 'auto', + secretAccessKey: process.env.S3_SECRET_ACCESS_KEY || '', + uploadPath: uploadPath || `${basePath}${filename}.{extName}`, + }; + + const client = s3.createS3Client(userConfig); + + let results: UploadResult; + + try { + results = await s3.createUploadTask({ + acl: 'public-read', + bucketName: userConfig.bucketName, + client, + item: item, + path: formatPath(item, userConfig.uploadPath), + urlPrefix: userConfig.pathPrefix, + }); + + return results.url; + } catch (error) { + consola.error('上传到 S3 存储发生错误，请检查网络连接和配置是否正确'); + consola.error(error); + } +}; diff --git a/scripts/cdnWorkflow/utils.ts b/scripts/cdnWorkflow/utils.ts new file mode 100644 index 000000000000..db3a42654ad5 --- /dev/null +++ b/scripts/cdnWorkflow/utils.ts @@ -0,0 +1,93 @@ +import { readJSONSync } from 'fs-extra'; +import { globSync } from 'glob'; +import { resolve } from 'node:path'; + +import { opimized, opimizedGif } from './optimized'; + +export const fixWinPath = (path: string) => path.replaceAll('\\', '/'); + +export const root = resolve(__dirname, '../..'); + +export const posts = globSync(fixWinPath(resolve(root, 'docs/changelog/*.mdx'))); + +interface ChangelogItem { + date: string; + id: string; + image?: string; + versionRange: string[]; +} + +export const changelogIndexPath = resolve(root, 'docs/changelog/index.json'); + +export const changelogIndex: { + cloud: ChangelogItem[]; + community: ChangelogItem[]; +} = readJSONSync(changelogIndexPath); + +export const extractHttpsLinks = (text: string) => { + const regex = /https:\/\/[^\s"')>]+/g; + const links = text.match(regex); + return links || []; +}; + +export const mergeAndDeduplicateArrays = (...arrays: string[][]) => { + const combinedArray = arrays.flat(); + const uniqueSet = new Set(combinedArray); + return Array.from(uniqueSet); +}; + +const mimeToExtensions = { + 'image/gif': '.gif', + // 图片类型 + 'image/jpeg': '.jpg', + 'image/png': '.png', + 'image/svg+xml': '.svg', + 'image/webp': '.webp', + // 视频类型 + 'video/mp4': '.mp4', + 'video/mpeg': '.mpeg', + 'video/ogg': '.ogv', + 'video/quicktime': '.mov', + 'video/webm': '.webm', + 'video/x-flv': '.flv', + 'video/x-matroska': '.mkv', + 'video/x-ms-wmv': '.wmv', + 'video/x-msvideo': '.avi', +}; + +// @ts-ignore +const getExtension = (type: string) => mimeToExtensions?.[type] || '.png'; + +export const fetchImageAsFile = async (url: string, width: number) => { + try { + // Step 1: Fetch the image + const response = await fetch(url); + if (!response.ok) { + throw new Error(`HTTP error! status: ${response.status}`); + } + + // Step 2: Create a blob from the response data + const blob = await response.blob(); + let buffer: ArrayBuffer | Buffer = await blob.arrayBuffer(); + let type = getExtension(blob.type); + if (type === '.gif') { + buffer = await opimizedGif(buffer); + type = '.webp'; + } else if (type === '.png' || type === '.jpg') { + buffer = await opimized(buffer, width); + type = '.webp'; + } + + const filename = Date.now().toString() + type; + + // Step 3: Create a file from the blob + const file: File = new File([buffer], filename, { + lastModified: Date.now(), + type: type === '.webp' ? 'image/webp' : blob.type, + }); + + return file; + } catch (error) { + console.error('Error fetching image as file:', error); + } +};