usbharu
/
misskey
mirror of https://github.com/misskey-dev/misskey.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

wip

This commit is contained in:
syuilo 2018-08-18 23:56:44 +09:00
parent df71c90f9f
commit 335200c31e
9 changed files with 17 additions and 206 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

101
cli/clean-cached-remote-files.js
View File

@ -1,101 +0,0 @@
const chalk = require('chalk');
const log = require('single-line-log').stdout;
const sequential = require('promise-sequential');
const { default: DriveFile, DriveFileChunk } = require('../built/models/drive-file');
const { default: DriveFileThumbnail, DriveFileThumbnailChunk } = require('../built/models/drive-file-thumbnail');
const { default: User } = require('../built/models/user');
const q = {
'metadata._user.host': {
$ne: null
},
'metadata.withoutChunks': false
};
async function main() {
const promiseGens = [];
const count = await DriveFile.count(q);
let prev;
for (let i = 0; i < count; i++) {
promiseGens.push(() => {
const promise = new Promise(async (res, rej) => {
const file = await DriveFile.findOne(prev ? Object.assign({
_id: { $lt: prev._id }
}, q) : q, {
sort: {
_id: -1
}
});
prev = file;
function skip() {
res([i, file, false]);
}
if (file == null) return skip();
log(chalk`{gray ${i}} scanning {bold ${file._id}} ${file.filename} ...`);
const attachingUsersCount = await User.count({
$or: [{
avatarId: file._id
}, {
bannerId: file._id
}]
}, { limit: 1 });
if (attachingUsersCount !== 0) return skip();
Promise.all([
// チャンクをすべて削除
DriveFileChunk.remove({
files_id: file._id
}),
DriveFile.update({ _id: file._id }, {
$set: {
'metadata.withoutChunks': true
}
})
]).then(async () => {
res([i, file, true]);
//#region サムネイルもあれば削除
const thumbnail = await DriveFileThumbnail.findOne({
'metadata.originalId': file._id
});
if (thumbnail) {
DriveFileThumbnailChunk.remove({
files_id: thumbnail._id
});
DriveFileThumbnail.remove({ _id: thumbnail._id });
}
//#endregion
});
});
promise.then(([i, file, deleted]) => {
if (deleted) {
log(chalk`{gray ${i}} {red deleted: {bold ${file._id}} ${file.filename}}`);
} else {
log(chalk`{gray ${i}} {green skipped: {bold ${file._id}} ${file.filename}}`);
}
log.clear();
console.log();
});
return promise;
});
}
return await sequential(promiseGens);
}
main().then(() => {
console.log('ALL DONE');
}).catch(console.error);

80
cli/clean-unused-drive-files.js
View File

@ -1,80 +0,0 @@
const chalk = require('chalk');
const log = require('single-line-log').stdout;
const sequential = require('promise-sequential');
const { default: DriveFile, deleteDriveFile } = require('../built/models/drive-file');
const { default: Note } = require('../built/models/note');
const { default: MessagingMessage } = require('../built/models/messaging-message');
const { default: User } = require('../built/models/user');
async function main() {
const promiseGens = [];
const count = await DriveFile.count({});
let prev;
for (let i = 0; i < count; i++) {
promiseGens.push(() => {
const promise = new Promise(async (res, rej) => {
const file = await DriveFile.findOne(prev ? {
_id: { $lt: prev._id }
} : {}, {
sort: {
_id: -1
}
});
prev = file;
function skip() {
res([i, file, false]);
}
if (file == null) return skip();
log(chalk`{gray ${i}} scanning {bold ${file._id}} ${file.filename} ...`);
const attachingUsersCount = await User.count({
$or: [{
avatarId: file._id
}, {
bannerId: file._id
}]
}, { limit: 1 });
if (attachingUsersCount !== 0) return skip();
const attachingNotesCount = await Note.count({
mediaIds: file._id
}, { limit: 1 });
if (attachingNotesCount !== 0) return skip();
const attachingMessagesCount = await MessagingMessage.count({
fileId: file._id
}, { limit: 1 });
if (attachingMessagesCount !== 0) return skip();
deleteDriveFile(file).then(() => {
res([i, file, true]);
}).catch(rej);
});
promise.then(([i, file, deleted]) => {
if (deleted) {
log(chalk`{gray ${i}} {red deleted: {bold ${file._id}} ${file.filename}}`);
} else {
log(chalk`{gray ${i}} {green skipped: {bold ${file._id}} ${file.filename}}`);
}
log.clear();
console.log();
});
return promise;
});
}
return await sequential(promiseGens);
}
main().then(() => {
console.log('done');
}).catch(console.error);

11
docs/manage.en.md
View File

@ -33,14 +33,3 @@ node cli/suspend @syuilo@misskey.xyz
``` shell
node cli/reset-password (User-ID or Username)
```
## Clean up cached remote files
``` shell
node cli/clean-cached-remote-files
```
## Clean up unused drive files
``` shell
node cli/clean-unused-drive-files
```
> We recommend that you announce a user that unused drive files will be deleted before performing this operation, as it may delete the user's important files.

11
docs/manage.ja.md
View File

@ -33,14 +33,3 @@ node cli/suspend @syuilo@misskey.xyz
``` shell
node cli/reset-password (ユーザーID または ユーザー名)
```
## キャッシュされたリモートファイルをクリーンアップする
``` shell
node cli/clean-cached-remote-files
```
## 使われていないドライブのファイルをクリーンアップする
``` shell
node cli/clean-unused-drive-files
```
> ユーザーの大事なファイルを削除する可能性があるので、この操作を実行する前にユーザーに告知することをお勧めします。

4
src/services/drive/add-file.ts
View File

@ -17,6 +17,7 @@ import { isLocalUser, IUser, IRemoteUser } from '../../models/user';
import delFile from './delete-file';
import config from '../../config';
import { getDriveFileThumbnailBucket } from '../../models/drive-file-thumbnail';
import { updateDriveStats } from '../update-chart';
const log = debug('misskey:drive:add-file');
@ -377,7 +378,8 @@ export default async function(
publishDriveStream(user._id, 'file_created', packedFile);
});
// TODO: サムネイル生成
// 統計を更新
updateDriveStats(driveFile, true);
return driveFile;
}

4
src/services/drive/delete-file.ts
View File

@ -2,6 +2,7 @@ import * as Minio from 'minio';
import DriveFile, { DriveFileChunk, IDriveFile } from '../../models/drive-file';
import DriveFileThumbnail, { DriveFileThumbnailChunk } from '../../models/drive-file-thumbnail';
import config from '../../config';
import { updateDriveStats } from '../update-chart';
export default async function(file: IDriveFile, isExpired = false) {
if (file.metadata.storage == 'minio') {
@ -45,4 +46,7 @@ export default async function(file: IDriveFile, isExpired = false) {
await DriveFileThumbnail.remove({ _id: thumbnail._id });
}
//#endregion
// 統計を更新
updateDriveStats(file, false);
}

4
src/services/note/create.ts
View File

@ -23,6 +23,7 @@ import registerHashtag from '../register-hashtag';
import isQuote from '../../misc/is-quote';
import { TextElementMention } from '../../mfm/parse/elements/mention';
import { TextElementHashtag } from '../../mfm/parse/elements/hashtag';
import { updateNoteStats } from '../update-chart';
type NotificationType = 'reply' | 'renote' | 'quote' | 'mention';
@ -142,6 +143,9 @@ export default async (user: IUser, data: Option, silent = false) => new Promise<
return;
}
// 統計を更新
updateNoteStats(note, true);
// ハッシュタグ登録
tags.map(tag => registerHashtag(user, tag));

4
src/services/note/delete.ts
View File

@ -6,6 +6,7 @@ import pack from '../../remote/activitypub/renderer';
import { deliver } from '../../queue';
import Following from '../../models/following';
import renderNote from '../../remote/activitypub/renderer/note';
import { updateNoteStats } from '../update-chart';
/**
* 投稿を削除します。
@ -43,4 +44,7 @@ export default async function(user: IUser, note: INote) {
});
}
//#endregion
// 統計を更新
updateNoteStats(note, false);
}

4
src/services/update-chart.ts
View File

@ -201,13 +201,13 @@ export async function updateNoteStats(note: INote, isAdditional: boolean) {
await update(inc);
}
export async function updateDriveStats(user: IUser, file: IDriveFile, isAdditional: boolean) {
export async function updateDriveStats(file: IDriveFile, isAdditional: boolean) {
const inc = {} as any;
const amount = isAdditional ? 1 : -1;
const size = isAdditional ? file.length : -file.length;
if (isLocalUser(user)) {
if (isLocalUser(file.metadata._user)) {
inc['drive.local.totalCount'] = amount;
inc['drive.local.diffCount'] = amount;
inc['drive.local.totalSize'] = size;