From 03e999875a7678c0e5293b87397ab67040010eeb Mon Sep 17 00:00:00 2001 From: syuilo Date: Mon, 1 Jul 2019 21:12:14 +0900 Subject: [PATCH] =?UTF-8?q?=E3=83=AA=E3=83=A2=E3=83=BC=E3=83=88=E3=83=95?= =?UTF-8?q?=E3=82=A1=E3=82=A4=E3=83=AB=E3=81=AE=E5=89=8A=E9=99=A4=E3=81=8C?= =?UTF-8?q?=E9=87=8D=E3=81=84=E5=95=8F=E9=A1=8C=E3=82=92=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/queue/index.ts | 7 +++ src/queue/processors/db/delete-drive-files.ts | 4 +- .../object-storage/clean-remote-files.ts | 50 ++++++++++++++++++ .../processors/object-storage/delete-file.ts | 16 +----- src/queue/processors/object-storage/index.ts | 2 + .../admin/drive/clean-remote-files.ts | 13 +---- src/services/drive/delete-file.ts | 51 +++++++++++++++++++ 7 files changed, 116 insertions(+), 27 deletions(-) create mode 100644 src/queue/processors/object-storage/clean-remote-files.ts diff --git a/src/queue/index.ts b/src/queue/index.ts index 70233d8318..a7e9b9814f 100644 --- a/src/queue/index.ts +++ b/src/queue/index.ts @@ -194,6 +194,13 @@ export function createDeleteObjectStorageFileJob(key: string) { }); } +export function createCleanRemoteFilesJob() { + return objectStorageQueue.add('cleanRemoteFiles', {}, { + removeOnComplete: true, + removeOnFail: true + }); +} + export default function() { if (!program.onlyServer) { deliverQueue.process(128, processDeliver); diff --git a/src/queue/processors/db/delete-drive-files.ts b/src/queue/processors/db/delete-drive-files.ts index 5ecee9139a..a2fd9050a9 100644 --- a/src/queue/processors/db/delete-drive-files.ts +++ b/src/queue/processors/db/delete-drive-files.ts @@ -1,7 +1,7 @@ import * as Bull from 'bull'; import { queueLogger } from '../../logger'; -import { deleteFile } from '../../../services/drive/delete-file'; +import { deleteFileSync } from '../../../services/drive/delete-file'; import { Users, DriveFiles } from '../../../models'; import { MoreThan } from 'typeorm'; @@ -39,7 +39,7 @@ export async function deleteDriveFiles(job: Bull.Job, done: any): Promise cursor = files[files.length - 1].id; for (const file of files) { - await deleteFile(file); + await deleteFileSync(file); deletedCount++; } diff --git a/src/queue/processors/object-storage/clean-remote-files.ts b/src/queue/processors/object-storage/clean-remote-files.ts new file mode 100644 index 0000000000..7b34892e1f --- /dev/null +++ b/src/queue/processors/object-storage/clean-remote-files.ts @@ -0,0 +1,50 @@ +import * as Bull from 'bull'; + +import { queueLogger } from '../../logger'; +import { deleteFileSync } from '../../../services/drive/delete-file'; +import { DriveFiles } from '../../../models'; +import { MoreThan, Not, IsNull } from 'typeorm'; + +const logger = queueLogger.createSubLogger('clean-remote-files'); + +export default async function cleanRemoteFiles(job: Bull.Job, done: any): Promise { + logger.info(`Deleting cached remote files...`); + + let deletedCount = 0; + let cursor: any = null; + + while (true) { + const files = await DriveFiles.find({ + where: { + userHost: Not(IsNull()), + isLink: false, + ...(cursor ? { id: MoreThan(cursor) } : {}) + }, + take: 8, + order: { + id: 1 + } + }); + + if (files.length === 0) { + job.progress(100); + break; + } + + cursor = files[files.length - 1].id; + + await Promise.all(files.map(file => deleteFileSync(file, true))); + + deletedCount += 8; + + const total = await DriveFiles.count({ + userHost: Not(IsNull()), + isLink: false, + }); + + job.progress(deletedCount / total); + } + + logger.succ(`All cahced remote files has been deleted.`); + done(); +} diff --git a/src/queue/processors/object-storage/delete-file.ts b/src/queue/processors/object-storage/delete-file.ts index 8e6b5f959e..f899df7d2e 100644 --- a/src/queue/processors/object-storage/delete-file.ts +++ b/src/queue/processors/object-storage/delete-file.ts @@ -1,22 +1,10 @@ import * as Bull from 'bull'; -import * as Minio from 'minio'; -import { fetchMeta } from '../../../misc/fetch-meta'; +import { deleteObjectStorageFile } from '../../../services/drive/delete-file'; export default async (job: Bull.Job) => { - const meta = await fetchMeta(); - - const minio = new Minio.Client({ - endPoint: meta.objectStorageEndpoint!, - region: meta.objectStorageRegion ? meta.objectStorageRegion : undefined, - port: meta.objectStoragePort ? meta.objectStoragePort : undefined, - useSSL: meta.objectStorageUseSSL, - accessKey: meta.objectStorageAccessKey!, - secretKey: meta.objectStorageSecretKey!, - }); - const key: string = job.data.key; - await minio.removeObject(meta.objectStorageBucket!, key); + await deleteObjectStorageFile(key); return 'Success'; }; diff --git a/src/queue/processors/object-storage/index.ts b/src/queue/processors/object-storage/index.ts index 60f732ca64..33ef665b38 100644 --- a/src/queue/processors/object-storage/index.ts +++ b/src/queue/processors/object-storage/index.ts @@ -1,8 +1,10 @@ import * as Bull from 'bull'; import deleteFile from './delete-file'; +import cleanRemoteFiles from './clean-remote-files'; const jobs = { deleteFile, + cleanRemoteFiles, } as any; export default function(q: Bull.Queue) { diff --git a/src/server/api/endpoints/admin/drive/clean-remote-files.ts b/src/server/api/endpoints/admin/drive/clean-remote-files.ts index 69cfe0db94..e837ae1bb6 100644 --- a/src/server/api/endpoints/admin/drive/clean-remote-files.ts +++ b/src/server/api/endpoints/admin/drive/clean-remote-files.ts @@ -1,7 +1,5 @@ -import { Not, IsNull } from 'typeorm'; import define from '../../../define'; -import { deleteFile } from '../../../../../services/drive/delete-file'; -import { DriveFiles } from '../../../../../models'; +import { createCleanRemoteFilesJob } from '../../../../../queue'; export const meta = { tags: ['admin'], @@ -11,12 +9,5 @@ export const meta = { }; export default define(meta, async (ps, me) => { - const files = await DriveFiles.find({ - userHost: Not(IsNull()), - isLink: false, - }); - - for (const file of files) { - deleteFile(file, true); - } + createCleanRemoteFilesJob(); }); diff --git a/src/services/drive/delete-file.ts b/src/services/drive/delete-file.ts index 258c0853fb..5b44a0817a 100644 --- a/src/services/drive/delete-file.ts +++ b/src/services/drive/delete-file.ts @@ -1,8 +1,10 @@ +import * as Minio from 'minio'; import { DriveFile } from '../../models/entities/drive-file'; import { InternalStorage } from './internal-storage'; import { DriveFiles, Instances, Notes } from '../../models'; import { driveChart, perUserDriveChart, instanceChart } from '../chart'; import { createDeleteObjectStorageFileJob } from '../../queue'; +import { fetchMeta } from '../../misc/fetch-meta'; export async function deleteFile(file: DriveFile, isExpired = false) { if (file.storedInternal) { @@ -27,6 +29,40 @@ export async function deleteFile(file: DriveFile, isExpired = false) { } } + postProcess(file, isExpired); +} + +export async function deleteFileSync(file: DriveFile, isExpired = false) { + if (file.storedInternal) { + InternalStorage.del(file.accessKey!); + + if (file.thumbnailUrl) { + InternalStorage.del(file.thumbnailAccessKey!); + } + + if (file.webpublicUrl) { + InternalStorage.del(file.webpublicAccessKey!); + } + } else if (!file.isLink) { + const promises = []; + + promises.push(deleteObjectStorageFile(file.accessKey!)); + + if (file.thumbnailUrl) { + promises.push(deleteObjectStorageFile(file.thumbnailAccessKey!)); + } + + if (file.webpublicUrl) { + promises.push(deleteObjectStorageFile(file.webpublicAccessKey!)); + } + + await Promise.all(promises); + } + + postProcess(file, isExpired); +} + +function postProcess(file: DriveFile, isExpired = false) { // リモートファイル期限切れ削除後は直リンクにする if (isExpired && file.userHost !== null && file.uri != null) { DriveFiles.update(file.id, { @@ -53,3 +89,18 @@ export async function deleteFile(file: DriveFile, isExpired = false) { Instances.decrement({ host: file.userHost }, 'driveFiles', 1); } } + +export async function deleteObjectStorageFile(key: string) { + const meta = await fetchMeta(); + + const minio = new Minio.Client({ + endPoint: meta.objectStorageEndpoint!, + region: meta.objectStorageRegion ? meta.objectStorageRegion : undefined, + port: meta.objectStoragePort ? meta.objectStoragePort : undefined, + useSSL: meta.objectStorageUseSSL, + accessKey: meta.objectStorageAccessKey!, + secretKey: meta.objectStorageSecretKey!, + }); + + await minio.removeObject(meta.objectStorageBucket!, key); +}