diff --git a/packages/backend/src/core/DriveService.ts b/packages/backend/src/core/DriveService.ts index 7c6d92b2b104..57cf75c30748 100644 --- a/packages/backend/src/core/DriveService.ts +++ b/packages/backend/src/core/DriveService.ts @@ -10,6 +10,7 @@ import sharp from 'sharp'; import { sharpBmp } from '@misskey-dev/sharp-read-bmp'; import { IsNull } from 'typeorm'; import { DeleteObjectCommandInput, PutObjectCommandInput, NoSuchKey } from '@aws-sdk/client-s3'; +import * as Redis from 'ioredis'; import { DI } from '@/di-symbols.js'; import type { DriveFilesRepository, UsersRepository, DriveFoldersRepository, UserProfilesRepository, MiMeta } from '@/models/_.js'; import type { Config } from '@/config.js'; @@ -21,7 +22,7 @@ import { isDuplicateKeyValueError } from '@/misc/is-duplicate-key-value-error.js import { FILE_TYPE_BROWSERSAFE } from '@/const.js'; import { IdentifiableError } from '@/misc/identifiable-error.js'; import { contentDisposition } from '@/misc/content-disposition.js'; -import { GlobalEventService } from '@/core/GlobalEventService.js'; +import { GlobalEvents, GlobalEventService } from '@/core/GlobalEventService.js'; import { VideoProcessingService } from '@/core/VideoProcessingService.js'; import { ImageProcessingService } from '@/core/ImageProcessingService.js'; import type { IImage } from '@/core/ImageProcessingService.js'; @@ -43,6 +44,7 @@ import { correctFilename } from '@/misc/correct-filename.js'; import { isMimeImage } from '@/misc/is-mime-image.js'; import { ModerationLogService } from '@/core/ModerationLogService.js'; import { UtilityService } from '@/core/UtilityService.js'; +import { AppLockService } from '@/core/AppLockService.js'; type AddFileArgs = { /** User who wish to add file */ @@ -113,6 +115,9 @@ export class DriveService { @Inject(DI.driveFoldersRepository) private driveFoldersRepository: DriveFoldersRepository, + @Inject(DI.redisForSub) + private redisForSub: Redis.Redis, + private fileInfoService: FileInfoService, private userEntityService: UserEntityService, private driveFileEntityService: DriveFileEntityService, @@ -130,11 +135,13 @@ export class DriveService { private perUserDriveChart: PerUserDriveChart, private instanceChart: InstanceChart, private utilityService: UtilityService, + private appLockService: AppLockService, ) { const logger = new Logger('drive', 'blue'); this.registerLogger = logger.createSubLogger('register', 'yellow'); this.downloaderLogger = logger.createSubLogger('downloader'); this.deleteLogger = logger.createSubLogger('delete'); + this.redisForSub.on('message', this.onMessage); } /*** @@ -144,7 +151,7 @@ export class DriveService { * @param type Content-Type for original * @param hash Hash for original * @param size Size for original - * @param isRemoteFile is Remote file or Local File + * @param isRemoteFile is Remote file or Local File (Serafuku Custom) */ @bindThis private async save(file: MiDriveFile, path: string, name: string, type: string, hash: string, size: number, isRemoteFile = false): Promise { @@ -222,7 +229,14 @@ export class DriveService { file.size = size; file.storedInternal = false; - return await this.driveFilesRepository.insertOne(file); + // Re-Cache or create + if (await this.driveFilesRepository.exists({ where: { id: file.id } })) { + file.isLink = false; + await this.driveFilesRepository.update({ id: file.id }, file); + return await this.driveFilesRepository.findOneOrFail({ where: { id: file.id } }); + } else { + return await this.driveFilesRepository.insertOne(file); + } } else { // use internal storage const accessKey = randomUUID(); const thumbnailAccessKey = 'thumbnail-' + randomUUID(); @@ -256,7 +270,14 @@ export class DriveService { file.md5 = hash; file.size = size; - return await this.driveFilesRepository.insertOne(file); + // Re-Cache or create + if (await this.driveFilesRepository.exists({ where: { id: file.id } })) { + file.isLink = false; + await this.driveFilesRepository.update({ id: file.id }, file); + return await this.driveFilesRepository.findOneOrFail({ where: { id: file.id } }); + } else { + return await this.driveFilesRepository.insertOne(file); + } } } @@ -658,6 +679,51 @@ export class DriveService { return file; } + @bindThis + private async onMessage(_: string, data: string) { + const obj = JSON.parse(data); + + if (obj.channel === 'internal') { + const { type, body } = obj.message as GlobalEvents['internal']['payload']; + switch (type) { + case 'remoteFileCacheMiss': { + const fileId = body.fileId; + await this.reCacheFile(fileId); + break; + } + default: + break; + } + } + } + + @bindThis + public async reCacheFile(fileId: MiDriveFile['id']) { + const unlock = await this.appLockService.getApLock(`DriveFile://${fileId}`, 30000); + + const file = await this.driveFilesRepository.findOne({ where: { id: fileId } }); + if (!file || !file.uri || !file.isLink || !this.meta.cacheRemoteFiles) return; + + const uri = file.uri; + const [path, cleanup] = await createTemp(); + + try { + this.registerLogger.info(`Re-Caching Remote File... ${file.uri}`); + const { filename: name } = await this.downloadService.downloadUrl(uri, path); + const info = await this.fileInfoService.getFileInfo(path, { + skipSensitiveDetection: true, + }); + + const newFile = await this.save(file, path, name, info.type.mime, info.md5, info.size, true); + this.registerLogger.succ(`drive file has been Re-Cached ${file.url} -> ${newFile.url}`); + } catch (err) { + this.registerLogger.warn(`Fail to Re-Cache Remote file: ${err}`); + } finally { + unlock(); + cleanup(); + } + } + @bindThis public async updateFile(file: MiDriveFile, values: Partial, updater: MiUser) { const alwaysMarkNsfw = (await this.roleService.getUserPolicies(file.userId)).alwaysMarkNsfw; diff --git a/packages/backend/src/core/GlobalEventService.ts b/packages/backend/src/core/GlobalEventService.ts index 0e3d8c3f9924..0e3d8e6e42c1 100644 --- a/packages/backend/src/core/GlobalEventService.ts +++ b/packages/backend/src/core/GlobalEventService.ts @@ -253,6 +253,7 @@ export interface InternalEventTypes { unmute: { muterId: MiUser['id']; muteeId: MiUser['id']; }; userListMemberAdded: { userListId: MiUserList['id']; memberId: MiUser['id']; }; userListMemberRemoved: { userListId: MiUserList['id']; memberId: MiUser['id']; }; + remoteFileCacheMiss: { fileId: MiDriveFile['id']; } } type EventTypesToEventPayload = EventUnionFromDictionary>>; diff --git a/packages/backend/src/core/entities/DriveFileEntityService.ts b/packages/backend/src/core/entities/DriveFileEntityService.ts index c485555f90f9..858e54974d15 100644 --- a/packages/backend/src/core/entities/DriveFileEntityService.ts +++ b/packages/backend/src/core/entities/DriveFileEntityService.ts @@ -19,6 +19,7 @@ import { isMimeImage } from '@/misc/is-mime-image.js'; import { IdService } from '@/core/IdService.js'; import { UtilityService } from '../UtilityService.js'; import { VideoProcessingService } from '../VideoProcessingService.js'; +import { GlobalEventService } from '../GlobalEventService.js'; import { UserEntityService } from './UserEntityService.js'; import { DriveFolderEntityService } from './DriveFolderEntityService.js'; @@ -45,6 +46,7 @@ export class DriveFileEntityService { private driveFolderEntityService: DriveFolderEntityService, private videoProcessingService: VideoProcessingService, private idService: IdService, + private globalEventService: GlobalEventService, ) { } @@ -96,6 +98,8 @@ export class DriveFileEntityService { } if (file.uri != null && file.isLink && this.config.proxyRemoteFiles) { + this.globalEventService.publishInternalEvent('remoteFileCacheMiss', { fileId: file.id }); + // リモートかつ期限切れはローカルプロキシを試みる // 従来は/files/${thumbnailAccessKey}にアクセスしていたが、 // /filesはメディアプロキシにリダイレクトするようにしたため直接メディアプロキシを指定する @@ -116,6 +120,8 @@ export class DriveFileEntityService { // リモートかつ期限切れはローカルプロキシを試みる if (file.uri != null && file.isLink && this.config.proxyRemoteFiles) { + this.globalEventService.publishInternalEvent('remoteFileCacheMiss', { fileId: file.id }); + const key = file.webpublicAccessKey; if (key && !key.match('/')) { // 古いものはここにオブジェクトストレージキーが入ってるので除外