Feat: 외부 파일 접근시 재캐싱 기능

packages/backend/src/core/DriveService.ts

@@ -10,6 +10,7 @@ import sharp from 'sharp';
 import { sharpBmp } from '@misskey-dev/sharp-read-bmp';
 import { IsNull } from 'typeorm';
 import { DeleteObjectCommandInput, PutObjectCommandInput, NoSuchKey } from '@aws-sdk/client-s3';
+import * as Redis from 'ioredis';
 import { DI } from '@/di-symbols.js';
 import type { DriveFilesRepository, UsersRepository, DriveFoldersRepository, UserProfilesRepository, MiMeta } from '@/models/_.js';
 import type { Config } from '@/config.js';
@@ -21,7 +22,7 @@ import { isDuplicateKeyValueError } from '@/misc/is-duplicate-key-value-error.js
 import { FILE_TYPE_BROWSERSAFE } from '@/const.js';
 import { IdentifiableError } from '@/misc/identifiable-error.js';
 import { contentDisposition } from '@/misc/content-disposition.js';
-import { GlobalEventService } from '@/core/GlobalEventService.js';
+import { GlobalEvents, GlobalEventService } from '@/core/GlobalEventService.js';
 import { VideoProcessingService } from '@/core/VideoProcessingService.js';
 import { ImageProcessingService } from '@/core/ImageProcessingService.js';
 import type { IImage } from '@/core/ImageProcessingService.js';
@@ -43,6 +44,7 @@ import { correctFilename } from '@/misc/correct-filename.js';
 import { isMimeImage } from '@/misc/is-mime-image.js';
 import { ModerationLogService } from '@/core/ModerationLogService.js';
 import { UtilityService } from '@/core/UtilityService.js';
+import { AppLockService } from '@/core/AppLockService.js';
 
 type AddFileArgs = {
 	/** User who wish to add file */
@@ -113,6 +115,9 @@ export class DriveService {
 		@Inject(DI.driveFoldersRepository)
 		private driveFoldersRepository: DriveFoldersRepository,
 
+		@Inject(DI.redisForSub)
+		private redisForSub: Redis.Redis,
+
 		private fileInfoService: FileInfoService,
 		private userEntityService: UserEntityService,
 		private driveFileEntityService: DriveFileEntityService,
@@ -130,11 +135,13 @@ export class DriveService {
 		private perUserDriveChart: PerUserDriveChart,
 		private instanceChart: InstanceChart,
 		private utilityService: UtilityService,
+		private appLockService: AppLockService,
 	) {
 		const logger = new Logger('drive', 'blue');
 		this.registerLogger = logger.createSubLogger('register', 'yellow');
 		this.downloaderLogger = logger.createSubLogger('downloader');
 		this.deleteLogger = logger.createSubLogger('delete');
+		this.redisForSub.on('message', this.onMessage);
 	}
 
 	/***
@@ -144,7 +151,7 @@ export class DriveService {
 	 * @param type Content-Type for original
 	 * @param hash Hash for original
 	 * @param size Size for original
-	 * @param isRemoteFile is Remote file or Local File
+	 * @param isRemoteFile is Remote file or Local File (Serafuku Custom)
 	 */
 	@bindThis
 	private async save(file: MiDriveFile, path: string, name: string, type: string, hash: string, size: number, isRemoteFile = false): Promise<MiDriveFile> {
@@ -222,7 +229,14 @@ export class DriveService {
 			file.size = size;
 			file.storedInternal = false;
 
-			return await this.driveFilesRepository.insertOne(file);
+			// Re-Cache or create
+			if (await this.driveFilesRepository.exists({ where: { id: file.id } })) {
+				file.isLink = false;
+				await this.driveFilesRepository.update({ id: file.id }, file);
+				return await this.driveFilesRepository.findOneOrFail({ where: { id: file.id } });
+			} else {
+				return await this.driveFilesRepository.insertOne(file);
+			}
 		} else { // use internal storage
 			const accessKey = randomUUID();
 			const thumbnailAccessKey = 'thumbnail-' + randomUUID();
@@ -256,7 +270,14 @@ export class DriveService {
 			file.md5 = hash;
 			file.size = size;
 
-			return await this.driveFilesRepository.insertOne(file);
+			// Re-Cache or create
+			if (await this.driveFilesRepository.exists({ where: { id: file.id } })) {
+				file.isLink = false;
+				await this.driveFilesRepository.update({ id: file.id }, file);
+				return await this.driveFilesRepository.findOneOrFail({ where: { id: file.id } });
+			} else {
+				return await this.driveFilesRepository.insertOne(file);
+			}
 		}
 	}
 
@@ -658,6 +679,51 @@ export class DriveService {
 		return file;
 	}
 
+	@bindThis
+	private async onMessage(_: string, data: string) {
+		const obj = JSON.parse(data);
+
+		if (obj.channel === 'internal') {
+			const { type, body } = obj.message as GlobalEvents['internal']['payload'];
+			switch (type) {
+				case 'remoteFileCacheMiss': {
+					const fileId = body.fileId;
+					await this.reCacheFile(fileId);
+					break;
+				}
+				default:
+					break;
+			}
+		}
+	}
+
+	@bindThis
+	public async reCacheFile(fileId: MiDriveFile['id']) {
+		const unlock = await this.appLockService.getApLock(`DriveFile://${fileId}`, 30000);
+
+		const file = await this.driveFilesRepository.findOne({ where: { id: fileId } });
+		if (!file || !file.uri || !file.isLink || !this.meta.cacheRemoteFiles) return;
+
+		const uri = file.uri;
+		const [path, cleanup] = await createTemp();
+
+		try {
+			this.registerLogger.info(`Re-Caching Remote File... ${file.uri}`);
+			const { filename: name } = await this.downloadService.downloadUrl(uri, path);
+			const info = await this.fileInfoService.getFileInfo(path, {
+				skipSensitiveDetection: true,
+			});
+
+			const newFile = await this.save(file, path, name, info.type.mime, info.md5, info.size, true);
+			this.registerLogger.succ(`drive file has been Re-Cached ${file.url} -> ${newFile.url}`);
+		} catch (err) {
+			this.registerLogger.warn(`Fail to Re-Cache Remote file: ${err}`);
+		} finally {
+			unlock();
+			cleanup();
+		}
+	}
+
 	@bindThis
 	public async updateFile(file: MiDriveFile, values: Partial<MiDriveFile>, updater: MiUser) {
 		const alwaysMarkNsfw = (await this.roleService.getUserPolicies(file.userId)).alwaysMarkNsfw;

packages/backend/src/core/GlobalEventService.ts

@@ -253,6 +253,7 @@ export interface InternalEventTypes {
 	unmute: { muterId: MiUser['id']; muteeId: MiUser['id']; };
 	userListMemberAdded: { userListId: MiUserList['id']; memberId: MiUser['id']; };
 	userListMemberRemoved: { userListId: MiUserList['id']; memberId: MiUser['id']; };
+	remoteFileCacheMiss: { fileId: MiDriveFile['id']; }
 }
 
 type EventTypesToEventPayload<T> = EventUnionFromDictionary<UndefinedAsNullAll<SerializedAll<T>>>;

packages/backend/src/core/entities/DriveFileEntityService.ts

@@ -19,6 +19,7 @@ import { isMimeImage } from '@/misc/is-mime-image.js';
 import { IdService } from '@/core/IdService.js';
 import { UtilityService } from '../UtilityService.js';
 import { VideoProcessingService } from '../VideoProcessingService.js';
+import { GlobalEventService } from '../GlobalEventService.js';
 import { UserEntityService } from './UserEntityService.js';
 import { DriveFolderEntityService } from './DriveFolderEntityService.js';
 
@@ -45,6 +46,7 @@ export class DriveFileEntityService {
 		private driveFolderEntityService: DriveFolderEntityService,
 		private videoProcessingService: VideoProcessingService,
 		private idService: IdService,
+		private globalEventService: GlobalEventService,
 	) {
 	}
 
@@ -96,6 +98,8 @@ export class DriveFileEntityService {
 		}
 
 		if (file.uri != null && file.isLink && this.config.proxyRemoteFiles) {
+			this.globalEventService.publishInternalEvent('remoteFileCacheMiss', { fileId: file.id });
+
 			// リモートかつ期限切れはローカルプロキシを試みる
 			// 従来は/files/${thumbnailAccessKey}にアクセスしていたが、
 			// /filesはメディアプロキシにリダイレクトするようにしたため直接メディアプロキシを指定する
@@ -116,6 +120,8 @@ export class DriveFileEntityService {
 
 		// リモートかつ期限切れはローカルプロキシを試みる
 		if (file.uri != null && file.isLink && this.config.proxyRemoteFiles) {
+			this.globalEventService.publishInternalEvent('remoteFileCacheMiss', { fileId: file.id });
+
 			const key = file.webpublicAccessKey;
 
 			if (key && !key.match('/')) {	// 古いものはここにオブジェクトストレージキーが入ってるので除外