From e2b5d822292636cdf7cb12413f1ccf6d0ee5802e Mon Sep 17 00:00:00 2001 From: riku6460 <17585784+riku6460@users.noreply.github.com> Date: Sat, 21 Oct 2023 03:38:00 +0900 Subject: [PATCH 1/2] =?UTF-8?q?TL=E3=82=92Push=E5=9E=8B=E3=81=AB=E3=81=99?= =?UTF-8?q?=E3=82=8B=EF=BC=88Write=20only)=20(MisskeyIO#185)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 一時的に Redis に書き込むユーザーをモデレータのみに絞る --------- Co-authored-by: syuilo --- locales/index.d.ts | 1 + locales/ja-JP.yml | 1 + .../1696323464251-user-list-membership.js | 11 + .../1696373953614-meta-cache-settings.js | 22 ++ .../migration/1696807733453-userListUserId.js | 21 ++ .../1696808725134-userListUserId-2.js | 16 ++ packages/backend/src/GlobalModule.ts | 13 +- packages/backend/src/config.ts | 13 ++ .../backend/src/core/AccountMoveService.ts | 3 +- packages/backend/src/core/CoreModule.ts | 12 ++ packages/backend/src/core/FeaturedService.ts | 116 +++++++++++ .../backend/src/core/NoteCreateService.ts | 194 ++++++++++++++++-- packages/backend/src/core/ReactionService.ts | 20 ++ .../backend/src/core/RedisTimelineService.ts | 80 ++++++++ packages/backend/src/core/UserListService.ts | 1 + packages/backend/src/di-symbols.ts | 1 + packages/backend/src/models/entities/Meta.ts | 20 ++ .../src/models/entities/UserListJoining.ts | 9 +- .../src/server/api/endpoints/admin/meta.ts | 20 ++ .../server/api/endpoints/admin/update-meta.ts | 20 ++ .../src/pages/admin/external-services.vue | 81 ++++++++ packages/frontend/src/pages/admin/index.vue | 5 + .../frontend/src/pages/admin/settings.vue | 40 ++-- packages/frontend/src/router.ts | 4 + 24 files changed, 688 insertions(+), 36 deletions(-) create mode 100644 packages/backend/migration/1696323464251-user-list-membership.js create mode 100644 packages/backend/migration/1696373953614-meta-cache-settings.js create mode 100644 packages/backend/migration/1696807733453-userListUserId.js create mode 100644 packages/backend/migration/1696808725134-userListUserId-2.js create mode 100644 packages/backend/src/core/FeaturedService.ts create mode 100644 packages/backend/src/core/RedisTimelineService.ts create mode 100644 packages/frontend/src/pages/admin/external-services.vue diff --git a/locales/index.d.ts b/locales/index.d.ts index 85d959697d..c41f40bbcf 100644 --- a/locales/index.d.ts +++ b/locales/index.d.ts @@ -1107,6 +1107,7 @@ export interface Locale { "currentAnnouncements": string; "pastAnnouncements": string; "youHaveUnreadAnnouncements": string; + "externalServices": string; "_announcement": { "forExistingUsers": string; "forExistingUsersDescription": string; diff --git a/locales/ja-JP.yml b/locales/ja-JP.yml index 5a194d5c81..3464b1be40 100644 --- a/locales/ja-JP.yml +++ b/locales/ja-JP.yml @@ -1104,6 +1104,7 @@ forYou: "あなたへ" currentAnnouncements: "現在のお知らせ" pastAnnouncements: "過去のお知らせ" youHaveUnreadAnnouncements: "未読のお知らせがあります。" +externalServices: "外部サービス" _announcement: forExistingUsers: "既存ユーザーのみ" diff --git a/packages/backend/migration/1696323464251-user-list-membership.js b/packages/backend/migration/1696323464251-user-list-membership.js new file mode 100644 index 0000000000..7534040c4c --- /dev/null +++ b/packages/backend/migration/1696323464251-user-list-membership.js @@ -0,0 +1,11 @@ +export class UserListMembership1696323464251 { + name = 'UserListMembership1696323464251' + + async up(queryRunner) { + await queryRunner.query(`ALTER TABLE "user_list_joining" RENAME TO "user_list_membership"`); + } + + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "user_list_membership" RENAME TO "user_list_joining"`); + } +} diff --git a/packages/backend/migration/1696373953614-meta-cache-settings.js b/packages/backend/migration/1696373953614-meta-cache-settings.js new file mode 100644 index 0000000000..f994b76ef2 --- /dev/null +++ b/packages/backend/migration/1696373953614-meta-cache-settings.js @@ -0,0 +1,22 @@ +/* + * SPDX-FileCopyrightText: syuilo and other misskey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +export class MetaCacheSettings1696373953614 { + name = 'MetaCacheSettings1696373953614' + + async up(queryRunner) { + await queryRunner.query(`ALTER TABLE "meta" ADD "perLocalUserUserTimelineCacheMax" integer NOT NULL DEFAULT '300'`); + await queryRunner.query(`ALTER TABLE "meta" ADD "perRemoteUserUserTimelineCacheMax" integer NOT NULL DEFAULT '100'`); + await queryRunner.query(`ALTER TABLE "meta" ADD "perUserHomeTimelineCacheMax" integer NOT NULL DEFAULT '300'`); + await queryRunner.query(`ALTER TABLE "meta" ADD "perUserListTimelineCacheMax" integer NOT NULL DEFAULT '300'`); + } + + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "perUserListTimelineCacheMax"`); + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "perUserHomeTimelineCacheMax"`); + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "perRemoteUserUserTimelineCacheMax"`); + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "perLocalUserUserTimelineCacheMax"`); + } +} diff --git a/packages/backend/migration/1696807733453-userListUserId.js b/packages/backend/migration/1696807733453-userListUserId.js new file mode 100644 index 0000000000..ab2ba07fb5 --- /dev/null +++ b/packages/backend/migration/1696807733453-userListUserId.js @@ -0,0 +1,21 @@ +/* + * SPDX-FileCopyrightText: syuilo and other misskey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +export class UserListUserId1696807733453 { + name = 'UserListUserId1696807733453' + + async up(queryRunner) { + await queryRunner.query(`ALTER TABLE "user_list_membership" ADD "userListUserId" character varying(32) NOT NULL DEFAULT ''`); + const memberships = await queryRunner.query(`SELECT "id", "userListId" FROM "user_list_membership"`); + for(let i = 0; i < memberships.length; i++) { + const userList = await queryRunner.query(`SELECT "userId" FROM "user_list" WHERE "id" = $1`, [memberships[i].userListId]); + await queryRunner.query(`UPDATE "user_list_membership" SET "userListUserId" = $1 WHERE "id" = $2`, [userList[0].userId, memberships[i].id]); + } + } + + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "user_list_membership" DROP COLUMN "userListUserId"`); + } +} diff --git a/packages/backend/migration/1696808725134-userListUserId-2.js b/packages/backend/migration/1696808725134-userListUserId-2.js new file mode 100644 index 0000000000..5bcb5aedc2 --- /dev/null +++ b/packages/backend/migration/1696808725134-userListUserId-2.js @@ -0,0 +1,16 @@ +/* + * SPDX-FileCopyrightText: syuilo and other misskey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +export class UserListUserId21696808725134 { + name = 'UserListUserId21696808725134' + + async up(queryRunner) { + await queryRunner.query(`ALTER TABLE "user_list_membership" ALTER COLUMN "userListUserId" DROP DEFAULT`); + } + + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "user_list_membership" ALTER COLUMN "userListUserId" SET DEFAULT ''`); + } +} diff --git a/packages/backend/src/GlobalModule.ts b/packages/backend/src/GlobalModule.ts index 666461c1bb..f6dfe53942 100644 --- a/packages/backend/src/GlobalModule.ts +++ b/packages/backend/src/GlobalModule.ts @@ -71,11 +71,18 @@ const $redisForSub: Provider = { inject: [DI.config], }; +const $redisForTimelines: Provider = { + provide: DI.redisForTimelines, + useFactory: (config: Config) => { + return new Redis.Redis(config.redisForTimelines); + }, + inject: [DI.config], +}; @Global() @Module({ imports: [RepositoryModule], - providers: [$config, $db, $meilisearch, $redis, $redisForPub, $redisForSub], - exports: [$config, $db, $meilisearch, $redis, $redisForPub, $redisForSub, RepositoryModule], + providers: [$config, $db, $meilisearch, $redis, $redisForPub, $redisForSub, $redisForTimelines], + exports: [$config, $db, $meilisearch, $redis, $redisForPub, $redisForSub, $redisForTimelines, RepositoryModule], }) export class GlobalModule implements OnApplicationShutdown { constructor( @@ -83,6 +90,7 @@ export class GlobalModule implements OnApplicationShutdown { @Inject(DI.redis) private redisClient: Redis.Redis, @Inject(DI.redisForPub) private redisForPub: Redis.Redis, @Inject(DI.redisForSub) private redisForSub: Redis.Redis, + @Inject(DI.redisForTimelines) private redisForTimelines: Redis.Redis, ) {} public async dispose(): Promise { @@ -99,6 +107,7 @@ export class GlobalModule implements OnApplicationShutdown { this.redisClient.disconnect(), this.redisForPub.disconnect(), this.redisForSub.disconnect(), + this.redisForTimelines.disconnect(), ]); } diff --git a/packages/backend/src/config.ts b/packages/backend/src/config.ts index 23ee3a91f5..5ef8209405 100644 --- a/packages/backend/src/config.ts +++ b/packages/backend/src/config.ts @@ -53,6 +53,7 @@ export type Source = { redis: RedisOptionsSource; redisForPubsub?: RedisOptionsSource; redisForJobQueue?: RedisOptionsSource; + redisForTimelines?: RedisOptionsSource; meilisearch?: { host: string; port: string; @@ -94,6 +95,10 @@ export type Source = { videoThumbnailGenerator?: string; signToActivityPubGet?: boolean; + + perChannelMaxNoteCacheCount?: number; + perUserNotificationsMaxCount?: number; + deactivateAntennaThreshold?: number; }; /** @@ -118,6 +123,10 @@ export type Mixin = { redis: RedisOptions & RedisOptionsSource; redisForPubsub: RedisOptions & RedisOptionsSource; redisForJobQueue: RedisOptions & RedisOptionsSource; + redisForTimelines: RedisOptions & RedisOptionsSource; + perChannelMaxNoteCacheCount: number; + perUserNotificationsMaxCount: number; + deactivateAntennaThreshold: number; }; export type Config = Source & Mixin; @@ -182,6 +191,10 @@ export function loadConfig() { mixin.redis = convertRedisOptions(config.redis, mixin.host); mixin.redisForPubsub = config.redisForPubsub ? convertRedisOptions(config.redisForPubsub, mixin.host) : mixin.redis; mixin.redisForJobQueue = config.redisForJobQueue ? convertRedisOptions(config.redisForJobQueue, mixin.host) : mixin.redis; + mixin.redisForTimelines = config.redisForTimelines ? convertRedisOptions(config.redisForTimelines, mixin.host) : mixin.redis; + mixin.perChannelMaxNoteCacheCount = config.perChannelMaxNoteCacheCount ?? 1000; + mixin.perUserNotificationsMaxCount = config.perUserNotificationsMaxCount ?? 300; + mixin.deactivateAntennaThreshold = config.deactivateAntennaThreshold ?? (1000 * 60 * 60 * 24 * 7); return Object.assign(config, mixin); } diff --git a/packages/backend/src/core/AccountMoveService.ts b/packages/backend/src/core/AccountMoveService.ts index 07e7e98eb4..f0a3b78946 100644 --- a/packages/backend/src/core/AccountMoveService.ts +++ b/packages/backend/src/core/AccountMoveService.ts @@ -232,7 +232,7 @@ export class AccountMoveService { }, }).then(joinings => joinings.map(joining => joining.userListId)); - const newJoinings: Map = new Map(); + const newJoinings: Map = new Map(); // 重複しないようにIDを生成 const genId = (): string => { @@ -248,6 +248,7 @@ export class AccountMoveService { createdAt: new Date(), userId: dst.id, userListId: joining.userListId, + userListUserId: joining.userListUserId, }); } diff --git a/packages/backend/src/core/CoreModule.ts b/packages/backend/src/core/CoreModule.ts index baf31ddb35..f816428b1e 100644 --- a/packages/backend/src/core/CoreModule.ts +++ b/packages/backend/src/core/CoreModule.ts @@ -57,6 +57,8 @@ import { ProxyAccountService } from './ProxyAccountService.js'; import { UtilityService } from './UtilityService.js'; import { FileInfoService } from './FileInfoService.js'; import { SearchService } from './SearchService.js'; +import { FeaturedService } from './FeaturedService.js'; +import { RedisTimelineService } from './RedisTimelineService.js'; import { ChartLoggerService } from './chart/ChartLoggerService.js'; import FederationChart from './chart/charts/federation.js'; import NotesChart from './chart/charts/notes.js'; @@ -182,6 +184,8 @@ const $WebhookService: Provider = { provide: 'WebhookService', useExisting: Webh const $UtilityService: Provider = { provide: 'UtilityService', useExisting: UtilityService }; const $FileInfoService: Provider = { provide: 'FileInfoService', useExisting: FileInfoService }; const $SearchService: Provider = { provide: 'SearchService', useExisting: SearchService }; +const $FeaturedService: Provider = { provide: 'FeaturedService', useExisting: FeaturedService }; +const $RedisTimelineService: Provider = { provide: 'RedisTimelineService', useExisting: RedisTimelineService }; const $ChartLoggerService: Provider = { provide: 'ChartLoggerService', useExisting: ChartLoggerService }; const $FederationChart: Provider = { provide: 'FederationChart', useExisting: FederationChart }; @@ -311,6 +315,8 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting UtilityService, FileInfoService, SearchService, + FeaturedService, + RedisTimelineService, ChartLoggerService, FederationChart, NotesChart, @@ -433,6 +439,8 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting $UtilityService, $FileInfoService, $SearchService, + $FeaturedService, + $RedisTimelineService, $ChartLoggerService, $FederationChart, $NotesChart, @@ -556,6 +564,8 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting UtilityService, FileInfoService, SearchService, + FeaturedService, + RedisTimelineService, FederationChart, NotesChart, UsersChart, @@ -677,6 +687,8 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting $UtilityService, $FileInfoService, $SearchService, + $FeaturedService, + $RedisTimelineService, $FederationChart, $NotesChart, $UsersChart, diff --git a/packages/backend/src/core/FeaturedService.ts b/packages/backend/src/core/FeaturedService.ts new file mode 100644 index 0000000000..555a646cdc --- /dev/null +++ b/packages/backend/src/core/FeaturedService.ts @@ -0,0 +1,116 @@ +/* + * SPDX-FileCopyrightText: syuilo and other misskey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { Inject, Injectable } from '@nestjs/common'; +import * as Redis from 'ioredis'; +import type { MiNote, MiUser } from '@/models/index.js'; +import { DI } from '@/di-symbols.js'; +import { bindThis } from '@/decorators.js'; + +const GLOBAL_NOTES_RANKING_WINDOW = 1000 * 60 * 60 * 24 * 3; // 3日ごと +const PER_USER_NOTES_RANKING_WINDOW = 1000 * 60 * 60 * 24 * 7; // 1週間ごと +const HASHTAG_RANKING_WINDOW = 1000 * 60 * 60; // 1時間ごと + +@Injectable() +export class FeaturedService { + constructor( + @Inject(DI.redis) + private redisClient: Redis.Redis, // TODO: 専用のRedisサーバーを設定できるようにする + ) { + } + + @bindThis + private getCurrentWindow(windowRange: number): number { + const passed = new Date().getTime() - new Date(new Date().getFullYear(), 0, 1).getTime(); + return Math.floor(passed / windowRange); + } + + @bindThis + private async updateRankingOf(name: string, windowRange: number, element: string, score = 1): Promise { + const currentWindow = this.getCurrentWindow(windowRange); + const redisTransaction = this.redisClient.multi(); + redisTransaction.zincrby( + `${name}:${currentWindow}`, + score, + element); + redisTransaction.expire( + `${name}:${currentWindow}`, + (windowRange * 3) / 1000, + 'NX'); // "NX -- Set expiry only when the key has no expiry" = 有効期限がないときだけ設定 + await redisTransaction.exec(); + } + + @bindThis + private async getRankingOf(name: string, windowRange: number, threshold: number): Promise { + const currentWindow = this.getCurrentWindow(windowRange); + const previousWindow = currentWindow - 1; + + const redisPipeline = this.redisClient.pipeline(); + redisPipeline.zrange( + `${name}:${currentWindow}`, 0, threshold, 'REV', 'WITHSCORES'); + redisPipeline.zrange( + `${name}:${previousWindow}`, 0, threshold, 'REV', 'WITHSCORES'); + const [currentRankingResult, previousRankingResult] = await redisPipeline.exec().then(result => result ? result.map(r => r[1] as string[]) : [[], []]); + + const ranking = new Map(); + for (let i = 0; i < currentRankingResult.length; i += 2) { + const noteId = currentRankingResult[i]; + const score = parseInt(currentRankingResult[i + 1], 10); + ranking.set(noteId, score); + } + for (let i = 0; i < previousRankingResult.length; i += 2) { + const noteId = previousRankingResult[i]; + const score = parseInt(previousRankingResult[i + 1], 10); + const exist = ranking.get(noteId); + if (exist != null) { + ranking.set(noteId, (exist + score) / 2); + } else { + ranking.set(noteId, score); + } + } + + return Array.from(ranking.keys()); + } + + @bindThis + public updateGlobalNotesRanking(noteId: MiNote['id'], score = 1): Promise { + return this.updateRankingOf('featuredGlobalNotesRanking', GLOBAL_NOTES_RANKING_WINDOW, noteId, score); + } + + @bindThis + public updateInChannelNotesRanking(channelId: MiNote['channelId'], noteId: MiNote['id'], score = 1): Promise { + return this.updateRankingOf(`featuredInChannelNotesRanking:${channelId}`, GLOBAL_NOTES_RANKING_WINDOW, noteId, score); + } + + @bindThis + public updatePerUserNotesRanking(userId: MiUser['id'], noteId: MiNote['id'], score = 1): Promise { + return this.updateRankingOf(`featuredPerUserNotesRanking:${userId}`, PER_USER_NOTES_RANKING_WINDOW, noteId, score); + } + + @bindThis + public updateHashtagsRanking(hashtag: string, score = 1): Promise { + return this.updateRankingOf('featuredHashtagsRanking', HASHTAG_RANKING_WINDOW, hashtag, score); + } + + @bindThis + public getGlobalNotesRanking(threshold: number): Promise { + return this.getRankingOf('featuredGlobalNotesRanking', GLOBAL_NOTES_RANKING_WINDOW, threshold); + } + + @bindThis + public getInChannelNotesRanking(channelId: MiNote['channelId'], threshold: number): Promise { + return this.getRankingOf(`featuredInChannelNotesRanking:${channelId}`, GLOBAL_NOTES_RANKING_WINDOW, threshold); + } + + @bindThis + public getPerUserNotesRanking(userId: MiUser['id'], threshold: number): Promise { + return this.getRankingOf(`featuredPerUserNotesRanking:${userId}`, PER_USER_NOTES_RANKING_WINDOW, threshold); + } + + @bindThis + public getHashtagsRanking(threshold: number): Promise { + return this.getRankingOf('featuredHashtagsRanking', HASHTAG_RANKING_WINDOW, threshold); + } +} diff --git a/packages/backend/src/core/NoteCreateService.ts b/packages/backend/src/core/NoteCreateService.ts index bea536b327..006baf8f47 100644 --- a/packages/backend/src/core/NoteCreateService.ts +++ b/packages/backend/src/core/NoteCreateService.ts @@ -5,7 +5,7 @@ import { setImmediate } from 'node:timers/promises'; import * as mfm from 'mfm-js'; -import { In, DataSource } from 'typeorm'; +import { In, DataSource, IsNull, LessThan } from 'typeorm'; import * as Redis from 'ioredis'; import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common'; import RE2 from 're2'; @@ -14,7 +14,7 @@ import { extractCustomEmojisFromMfm } from '@/misc/extract-custom-emojis-from-mf import { extractHashtags } from '@/misc/extract-hashtags.js'; import type { IMentionedRemoteUsers } from '@/models/entities/Note.js'; import { MiNote } from '@/models/entities/Note.js'; -import type { ChannelFollowingsRepository, ChannelsRepository, InstancesRepository, MutedNotesRepository, MutingsRepository, NotesRepository, NoteThreadMutingsRepository, UserProfilesRepository, UsersRepository } from '@/models/index.js'; +import type { ChannelFollowingsRepository, ChannelsRepository, FollowingsRepository, InstancesRepository, MutedNotesRepository, MutingsRepository, NotesRepository, NoteThreadMutingsRepository, UserListJoiningsRepository, UserProfilesRepository, UsersRepository } from '@/models/index.js'; import type { MiDriveFile } from '@/models/entities/DriveFile.js'; import type { MiApp } from '@/models/entities/App.js'; import { concat } from '@/misc/prelude/array.js'; @@ -53,6 +53,8 @@ import { DB_MAX_NOTE_TEXT_LENGTH } from '@/const.js'; import { RoleService } from '@/core/RoleService.js'; import { MetaService } from '@/core/MetaService.js'; import { SearchService } from '@/core/SearchService.js'; +import { FeaturedService } from '@/core/FeaturedService.js'; +import { RedisTimelineService } from '@/core/RedisTimelineService.js'; const mutedWordsCache = new MemorySingleCache<{ userId: MiUserProfile['userId']; mutedWords: MiUserProfile['mutedWords']; }[]>(1000 * 60 * 5); @@ -161,6 +163,9 @@ export class NoteCreateService implements OnApplicationShutdown { @Inject(DI.redis) private redisClient: Redis.Redis, + @Inject(DI.redisForTimelines) + private redisForTimelines: Redis.Redis, + @Inject(DI.usersRepository) private usersRepository: UsersRepository, @@ -179,20 +184,27 @@ export class NoteCreateService implements OnApplicationShutdown { @Inject(DI.mutedNotesRepository) private mutedNotesRepository: MutedNotesRepository, + @Inject(DI.userListJoiningsRepository) + private userListJoiningsRepository: UserListJoiningsRepository, + @Inject(DI.channelsRepository) private channelsRepository: ChannelsRepository, - @Inject(DI.channelFollowingsRepository) - private channelFollowingsRepository: ChannelFollowingsRepository, - @Inject(DI.noteThreadMutingsRepository) private noteThreadMutingsRepository: NoteThreadMutingsRepository, + @Inject(DI.followingsRepository) + private followingsRepository: FollowingsRepository, + + @Inject(DI.channelFollowingsRepository) + private channelFollowingsRepository: ChannelFollowingsRepository, + private userEntityService: UserEntityService, private noteEntityService: NoteEntityService, private idService: IdService, private globalEventService: GlobalEventService, private queueService: QueueService, + private redisTimelineService: RedisTimelineService, private noteReadService: NoteReadService, private notificationService: NotificationService, private relayService: RelayService, @@ -200,6 +212,7 @@ export class NoteCreateService implements OnApplicationShutdown { private hashtagService: HashtagService, private antennaService: AntennaService, private webhookService: WebhookService, + private featuredService: FeaturedService, private remoteUserResolveService: RemoteUserResolveService, private apDeliverManagerService: ApDeliverManagerService, private apRendererService: ApRendererService, @@ -252,19 +265,30 @@ export class NoteCreateService implements OnApplicationShutdown { } } - // Renote対象が「ホームまたは全体」以外の公開範囲ならreject - if (data.renote && data.renote.visibility !== 'public' && data.renote.visibility !== 'home' && data.renote.userId !== user.id) { - throw new Error('Renote target is not public or home'); - } + if (data.renote) { + switch (data.renote.visibility) { + case 'public': + // public noteは無条件にrenote可能 + break; + case 'home': + // home noteはhome以下にrenote可能 + if (data.visibility === 'public') { + data.visibility = 'home'; + } + break; + case 'followers': + // 他人のfollowers noteはreject + if (data.renote.userId !== user.id) { + throw new Error('Renote target is not public or home'); + } - // Renote対象がpublicではないならhomeにする - if (data.renote && data.renote.visibility !== 'public' && data.visibility === 'public') { - data.visibility = 'home'; - } - - // Renote対象がfollowersならfollowersにする - if (data.renote && data.renote.visibility === 'followers') { - data.visibility = 'followers'; + // Renote対象がfollowersならfollowersにする + data.visibility = 'followers'; + break; + case 'specified': + // specified / direct noteはreject + throw new Error('Renote target is not public or home'); + } } // 返信対象がpublicではないならhomeにする @@ -367,7 +391,7 @@ export class NoteCreateService implements OnApplicationShutdown { name: data.name, text: data.text, hasPoll: data.poll != null, - cw: data.cw == null ? null : data.cw, + cw: data.cw ?? null, tags: tags.map(tag => normalizeForSearch(tag)), emojis, userId: user.id, @@ -402,7 +426,7 @@ export class NoteCreateService implements OnApplicationShutdown { const url = profile != null ? profile.url : null; return { uri: u.uri, - url: url == null ? undefined : url, + url: url ?? undefined, username: u.username, host: u.host, } as IMentionedRemoteUsers[0]; @@ -502,6 +526,10 @@ export class NoteCreateService implements OnApplicationShutdown { } }); + if (await this.roleService.isModerator({ id: user.id, isRoot: false })) { + this.pushToTl(note, user); + } + this.antennaService.addNoteToAntennas(note, user); if (data.reply) { @@ -714,6 +742,20 @@ export class NoteCreateService implements OnApplicationShutdown { }) .where('id = :id', { id: renote.id }) .execute(); + + // 30%の確率、3日以内に投稿されたノートの場合ハイライト用ランキング更新 + if (Math.random() < 0.3 && (Date.now() - this.idService.parse(renote.id).date.getTime()) < 1000 * 60 * 60 * 24 * 3) { + if (renote.channelId != null) { + if (renote.replyId == null) { + this.featuredService.updateInChannelNotesRanking(renote.channelId, renote.id, 5); + } + } else { + if (renote.visibility === 'public' && renote.userHost == null && renote.replyId == null) { + this.featuredService.updateGlobalNotesRanking(renote.id, 5); + this.featuredService.updatePerUserNotesRanking(renote.userId, renote.id, 5); + } + } + } } @bindThis @@ -799,6 +841,120 @@ export class NoteCreateService implements OnApplicationShutdown { return mentionedUsers; } + @bindThis + private async pushToTl(note: MiNote, user: { id: MiUser['id']; host: MiUser['host']; }) { + const meta = await this.metaService.fetch(); + + const r = this.redisForTimelines.pipeline(); + + if (note.channelId) { + this.redisTimelineService.push(`channelTimeline:${note.channelId}`, note.id, this.config.perChannelMaxNoteCacheCount, r); + + this.redisTimelineService.push(`userTimelineWithChannel:${user.id}`, note.id, note.userHost == null ? meta.perLocalUserUserTimelineCacheMax : meta.perRemoteUserUserTimelineCacheMax, r); + + const channelFollowings = await this.channelFollowingsRepository.find({ + where: { + followeeId: note.channelId, + }, + select: ['followerId'], + }); + + for (const channelFollowing of channelFollowings) { + this.redisTimelineService.push(`homeTimeline:${channelFollowing.followerId}`, note.id, meta.perUserHomeTimelineCacheMax, r); + if (note.fileIds.length > 0) { + this.redisTimelineService.push(`homeTimelineWithFiles:${channelFollowing.followerId}`, note.id, meta.perUserHomeTimelineCacheMax / 2, r); + } + } + } else { + // TODO: キャッシュ？ + // eslint-disable-next-line prefer-const + let [followings, userListMemberships] = await Promise.all([ + this.followingsRepository.find({ + where: { + followeeId: user.id, + followerHost: IsNull(), + }, + select: ['followerId'], + }), + this.userListJoiningsRepository.find({ + where: { + userId: user.id, + }, + select: ['userListId', 'userListUserId'], + }), + ]); + + if (note.visibility === 'followers') { + // TODO: 重そうだから何とかしたい Set 使う？ + userListMemberships = (userListMemberships).filter(x => followings.some(f => f.followerId === x.userListUserId)); + } + + // TODO: あまりにも数が多いと redisPipeline.exec に失敗する(理由は不明)ため、3万件程度を目安に分割して実行するようにする + for (const following of followings) { + // 基本的にvisibleUserIdsには自身のidが含まれている前提であること + if (note.visibility === 'specified' && !note.visibleUserIds.some(v => v === following.followerId)) continue; + + // 「自分自身への返信 or そのフォロワーへの返信」のどちらでもない場合 + if (note.replyId && !(note.replyUserId === note.userId || note.replyUserId === following.followerId)) { + continue; + } + + this.redisTimelineService.push(`homeTimeline:${following.followerId}`, note.id, meta.perUserHomeTimelineCacheMax, r); + if (note.fileIds.length > 0) { + this.redisTimelineService.push(`homeTimelineWithFiles:${following.followerId}`, note.id, meta.perUserHomeTimelineCacheMax / 2, r); + } + } + + for (const userListMembership of userListMemberships) { + // ダイレクトのとき、そのリストが対象外のユーザーの場合 + if ( + note.visibility === 'specified' && + !note.visibleUserIds.some(v => v === userListMembership.userListUserId) + ) continue; + + // 「自分自身への返信 or そのリストの作成者への返信」のどちらでもない場合 + if (note.replyId && !(note.replyUserId === note.userId || note.replyUserId === userListMembership.userListUserId)) { + continue; + } + + this.redisTimelineService.push(`userListTimeline:${userListMembership.userListId}`, note.id, meta.perUserListTimelineCacheMax, r); + if (note.fileIds.length > 0) { + this.redisTimelineService.push(`userListTimelineWithFiles:${userListMembership.userListId}`, note.id, meta.perUserListTimelineCacheMax / 2, r); + } + } + + if (note.visibility !== 'specified' || !note.visibleUserIds.some(v => v === user.id)) { // 自分自身のHTL + this.redisTimelineService.push(`homeTimeline:${user.id}`, note.id, meta.perUserHomeTimelineCacheMax, r); + if (note.fileIds.length > 0) { + this.redisTimelineService.push(`homeTimelineWithFiles:${user.id}`, note.id, meta.perUserHomeTimelineCacheMax / 2, r); + } + } + + // 自分自身以外への返信 + if (note.replyId && note.replyUserId !== note.userId) { + this.redisTimelineService.push(`userTimelineWithReplies:${user.id}`, note.id, note.userHost == null ? meta.perLocalUserUserTimelineCacheMax : meta.perRemoteUserUserTimelineCacheMax, r); + + if (note.visibility === 'public' && note.userHost == null) { + this.redisTimelineService.push('localTimelineWithReplies', note.id, 300, r); + } + } else { + this.redisTimelineService.push(`userTimeline:${user.id}`, note.id, note.userHost == null ? meta.perLocalUserUserTimelineCacheMax : meta.perRemoteUserUserTimelineCacheMax, r); + if (note.fileIds.length > 0) { + this.redisTimelineService.push(`userTimelineWithFiles:${user.id}`, note.id, note.userHost == null ? meta.perLocalUserUserTimelineCacheMax / 2 : meta.perRemoteUserUserTimelineCacheMax / 2, r); + } + + if (note.visibility === 'public' && note.userHost == null) { + this.redisTimelineService.push('localTimeline', note.id, 1000, r); + if (note.fileIds.length > 0) { + this.redisTimelineService.push('localTimelineWithFiles', note.id, 500, r); + } + } + } + } + + r.exec(); + } + @bindThis public dispose(): void { this.#shutdownController.abort(); diff --git a/packages/backend/src/core/ReactionService.ts b/packages/backend/src/core/ReactionService.ts index ad87f4b421..18a1cf8b47 100644 --- a/packages/backend/src/core/ReactionService.ts +++ b/packages/backend/src/core/ReactionService.ts @@ -26,6 +26,7 @@ import { UtilityService } from '@/core/UtilityService.js'; import { UserBlockingService } from '@/core/UserBlockingService.js'; import { CustomEmojiService } from '@/core/CustomEmojiService.js'; import { RoleService } from '@/core/RoleService.js'; +import { FeaturedService } from '@/core/FeaturedService.js'; const FALLBACK = '❤'; @@ -86,6 +87,7 @@ export class ReactionService { private noteEntityService: NoteEntityService, private userBlockingService: UserBlockingService, private idService: IdService, + private featuredService: FeaturedService, private globalEventService: GlobalEventService, private apRendererService: ApRendererService, private apDeliverManagerService: ApDeliverManagerService, @@ -190,6 +192,24 @@ export class ReactionService { .where('id = :id', { id: note.id }) .execute(); + // 30%の確率、セルフではない、3日以内に投稿されたノートの場合ハイライト用ランキング更新 + if ( + Math.random() < 0.3 && + note.userId !== user.id && + (Date.now() - this.idService.parse(note.id).date.getTime()) < 1000 * 60 * 60 * 24 * 3 + ) { + if (note.channelId != null) { + if (note.replyId == null) { + this.featuredService.updateInChannelNotesRanking(note.channelId, note.id, 1); + } + } else { + if (note.visibility === 'public' && note.userHost == null && note.replyId == null) { + this.featuredService.updateGlobalNotesRanking(note.id, 1); + this.featuredService.updatePerUserNotesRanking(note.userId, note.id, 1); + } + } + } + const meta = await this.metaService.fetch(); if (meta.enableChartsForRemoteUser || (user.host == null)) { diff --git a/packages/backend/src/core/RedisTimelineService.ts b/packages/backend/src/core/RedisTimelineService.ts new file mode 100644 index 0000000000..94541759cc --- /dev/null +++ b/packages/backend/src/core/RedisTimelineService.ts @@ -0,0 +1,80 @@ +/* + * SPDX-FileCopyrightText: syuilo and other misskey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { Inject, Injectable } from '@nestjs/common'; +import * as Redis from 'ioredis'; +import { DI } from '@/di-symbols.js'; +import { bindThis } from '@/decorators.js'; +import { IdService } from '@/core/IdService.js'; + +@Injectable() +export class RedisTimelineService { + constructor( + @Inject(DI.redisForTimelines) + private redisForTimelines: Redis.Redis, + + private idService: IdService, + ) { + } + + @bindThis + public push(tl: string, id: string, maxlen: number, pipeline: Redis.ChainableCommander) { + // リモートから遅れて届いた(もしくは後から追加された)投稿日時が古い投稿が追加されるとページネーション時に問題を引き起こすため、 + // 3分以内に投稿されたものでない場合、Redisにある最古のIDより新しい場合のみ追加する + if (this.idService.parse(id).date.getTime() > Date.now() - 1000 * 60 * 3) { + pipeline.lpush('list:' + tl, id); + if (Math.random() < 0.1) { // 10%の確率でトリム + pipeline.ltrim('list:' + tl, 0, maxlen - 1); + } + } else { + // 末尾のIDを取得 + this.redisForTimelines.lindex('list:' + tl, -1).then(lastId => { + if (lastId == null || (this.idService.parse(id).date.getTime() > this.idService.parse(lastId).date.getTime())) { + this.redisForTimelines.lpush('list:' + tl, id); + } else { + Promise.resolve(); + } + }); + } + } + + @bindThis + public get(name: string, untilId?: string | null, sinceId?: string | null) { + if (untilId && sinceId) { + return this.redisForTimelines.lrange('list:' + name, 0, -1) + .then(ids => ids.filter(id => id < untilId && id > sinceId).sort((a, b) => a > b ? -1 : 1)); + } else if (untilId) { + return this.redisForTimelines.lrange('list:' + name, 0, -1) + .then(ids => ids.filter(id => id < untilId).sort((a, b) => a > b ? -1 : 1)); + } else if (sinceId) { + return this.redisForTimelines.lrange('list:' + name, 0, -1) + .then(ids => ids.filter(id => id > sinceId).sort((a, b) => a < b ? -1 : 1)); + } else { + return this.redisForTimelines.lrange('list:' + name, 0, -1) + .then(ids => ids.sort((a, b) => a > b ? -1 : 1)); + } + } + + @bindThis + public getMulti(name: string[], untilId?: string | null, sinceId?: string | null): Promise { + const pipeline = this.redisForTimelines.pipeline(); + for (const n of name) { + pipeline.lrange('list:' + n, 0, -1); + } + return pipeline.exec().then(res => { + if (res == null) return []; + const tls = res.map(r => r[1] as string[]); + return tls.map(ids => + (untilId && sinceId) + ? ids.filter(id => id < untilId && id > sinceId).sort((a, b) => a > b ? -1 : 1) + : untilId + ? ids.filter(id => id < untilId).sort((a, b) => a > b ? -1 : 1) + : sinceId + ? ids.filter(id => id > sinceId).sort((a, b) => a < b ? -1 : 1) + : ids.sort((a, b) => a > b ? -1 : 1), + ); + }); + } +} diff --git a/packages/backend/src/core/UserListService.ts b/packages/backend/src/core/UserListService.ts index 7e9ec4657b..e10608d803 100644 --- a/packages/backend/src/core/UserListService.ts +++ b/packages/backend/src/core/UserListService.ts @@ -53,6 +53,7 @@ export class UserListService { createdAt: new Date(), userId: target.id, userListId: list.id, + userListUserId: list.userId, } as MiUserListJoining); this.globalEventService.publishUserListStream(list.id, 'userAdded', await this.userEntityService.pack(target, me)); diff --git a/packages/backend/src/di-symbols.ts b/packages/backend/src/di-symbols.ts index ab1ac250b6..e409cedecd 100644 --- a/packages/backend/src/di-symbols.ts +++ b/packages/backend/src/di-symbols.ts @@ -10,6 +10,7 @@ export const DI = { redis: Symbol('redis'), redisForPub: Symbol('redisForPub'), redisForSub: Symbol('redisForSub'), + redisForTimelines: Symbol('redisForTimelines'), //#region Repositories usersRepository: Symbol('usersRepository'), diff --git a/packages/backend/src/models/entities/Meta.ts b/packages/backend/src/models/entities/Meta.ts index 5f850a81e6..6015296a20 100644 --- a/packages/backend/src/models/entities/Meta.ts +++ b/packages/backend/src/models/entities/Meta.ts @@ -448,4 +448,24 @@ export class MiMeta { length: 1024, array: true, default: '{ "admin", "administrator", "root", "system", "maintainer", "host", "mod", "moderator", "owner", "superuser", "staff", "auth", "i", "me", "everyone", "all", "mention", "mentions", "example", "user", "users", "account", "accounts", "official", "help", "helps", "support", "supports", "info", "information", "informations", "announce", "announces", "announcement", "announcements", "notice", "notification", "notifications", "dev", "developer", "developers", "tech", "misskey" }', }) public preservedUsernames: string[]; + + @Column('integer', { + default: 300, + }) + public perLocalUserUserTimelineCacheMax: number; + + @Column('integer', { + default: 100, + }) + public perRemoteUserUserTimelineCacheMax: number; + + @Column('integer', { + default: 300, + }) + public perUserHomeTimelineCacheMax: number; + + @Column('integer', { + default: 300, + }) + public perUserListTimelineCacheMax: number; } diff --git a/packages/backend/src/models/entities/UserListJoining.ts b/packages/backend/src/models/entities/UserListJoining.ts index e4912024c1..314601f83c 100644 --- a/packages/backend/src/models/entities/UserListJoining.ts +++ b/packages/backend/src/models/entities/UserListJoining.ts @@ -8,7 +8,7 @@ import { id } from '../id.js'; import { MiUser } from './User.js'; import { MiUserList } from './UserList.js'; -@Entity('user_list_joining') +@Entity('user_list_membership') @Index(['userId', 'userListId'], { unique: true }) export class MiUserListJoining { @PrimaryColumn(id()) @@ -44,4 +44,11 @@ export class MiUserListJoining { }) @JoinColumn() public userList: MiUserList | null; + + //#region Denormalized fields + @Column({ + ...id(), + }) + public userListUserId: MiUser['id']; + //#endregion } diff --git a/packages/backend/src/server/api/endpoints/admin/meta.ts b/packages/backend/src/server/api/endpoints/admin/meta.ts index 93423a743d..d4da22e1a0 100644 --- a/packages/backend/src/server/api/endpoints/admin/meta.ts +++ b/packages/backend/src/server/api/endpoints/admin/meta.ts @@ -282,6 +282,22 @@ export const meta = { type: 'object', optional: false, nullable: false, }, + perLocalUserUserTimelineCacheMax: { + type: 'number', + optional: false, nullable: false, + }, + perRemoteUserUserTimelineCacheMax: { + type: 'number', + optional: false, nullable: false, + }, + perUserHomeTimelineCacheMax: { + type: 'number', + optional: false, nullable: false, + }, + perUserListTimelineCacheMax: { + type: 'number', + optional: false, nullable: false, + }, }, }, } as const; @@ -384,6 +400,10 @@ export default class extends Endpoint { enableServerMachineStats: instance.enableServerMachineStats, enableIdenticonGeneration: instance.enableIdenticonGeneration, policies: { ...DEFAULT_POLICIES, ...instance.policies }, + perLocalUserUserTimelineCacheMax: instance.perLocalUserUserTimelineCacheMax, + perRemoteUserUserTimelineCacheMax: instance.perRemoteUserUserTimelineCacheMax, + perUserHomeTimelineCacheMax: instance.perUserHomeTimelineCacheMax, + perUserListTimelineCacheMax: instance.perUserListTimelineCacheMax, }; }); } diff --git a/packages/backend/src/server/api/endpoints/admin/update-meta.ts b/packages/backend/src/server/api/endpoints/admin/update-meta.ts index b2a66895db..5d5812ff38 100644 --- a/packages/backend/src/server/api/endpoints/admin/update-meta.ts +++ b/packages/backend/src/server/api/endpoints/admin/update-meta.ts @@ -106,6 +106,10 @@ export const paramDef = { enableIdenticonGeneration: { type: 'boolean' }, serverRules: { type: 'array', items: { type: 'string' } }, preservedUsernames: { type: 'array', items: { type: 'string' } }, + perLocalUserUserTimelineCacheMax: { type: 'integer' }, + perRemoteUserUserTimelineCacheMax: { type: 'integer' }, + perUserHomeTimelineCacheMax: { type: 'integer' }, + perUserListTimelineCacheMax: { type: 'integer' }, }, required: [], } as const; @@ -427,6 +431,22 @@ export default class extends Endpoint { set.preservedUsernames = ps.preservedUsernames; } + if (ps.perLocalUserUserTimelineCacheMax !== undefined) { + set.perLocalUserUserTimelineCacheMax = ps.perLocalUserUserTimelineCacheMax; + } + + if (ps.perRemoteUserUserTimelineCacheMax !== undefined) { + set.perRemoteUserUserTimelineCacheMax = ps.perRemoteUserUserTimelineCacheMax; + } + + if (ps.perUserHomeTimelineCacheMax !== undefined) { + set.perUserHomeTimelineCacheMax = ps.perUserHomeTimelineCacheMax; + } + + if (ps.perUserListTimelineCacheMax !== undefined) { + set.perUserListTimelineCacheMax = ps.perUserListTimelineCacheMax; + } + await this.metaService.update(set); this.moderationLogService.insertModerationLog(me, 'updateMeta'); }); diff --git a/packages/frontend/src/pages/admin/external-services.vue b/packages/frontend/src/pages/admin/external-services.vue new file mode 100644 index 0000000000..e88860166c --- /dev/null +++ b/packages/frontend/src/pages/admin/external-services.vue @@ -0,0 +1,81 @@ + + + + + + + diff --git a/packages/frontend/src/pages/admin/index.vue b/packages/frontend/src/pages/admin/index.vue index 5bd6fce814..864fe53800 100644 --- a/packages/frontend/src/pages/admin/index.vue +++ b/packages/frontend/src/pages/admin/index.vue @@ -193,6 +193,11 @@ const menuDef = $computed(() => [{ text: i18n.ts.proxyAccount, to: '/admin/proxy-account', active: currentPage?.route.name === 'proxy-account', + }, { + icon: 'ti ti-link', + text: i18n.ts.externalServices, + to: '/admin/external-services', + active: currentPage?.route.name === 'external-services', }, { icon: 'ti ti-adjustments', text: i18n.ts.other, diff --git a/packages/frontend/src/pages/admin/settings.vue b/packages/frontend/src/pages/admin/settings.vue index 81ef32ac06..9b208e5138 100644 --- a/packages/frontend/src/pages/admin/settings.vue +++ b/packages/frontend/src/pages/admin/settings.vue @@ -76,16 +76,24 @@ SPDX-License-Identifier: AGPL-3.0-only - +
- - - + + + + + + + + + + + + + + - - -
@@ -127,8 +135,10 @@ let cacheRemoteSensitiveFiles: boolean = $ref(false); let enableServiceWorker: boolean = $ref(false); let swPublicKey: any = $ref(null); let swPrivateKey: any = $ref(null); -let deeplAuthKey: string = $ref(''); -let deeplIsPro: boolean = $ref(false); +let perLocalUserUserTimelineCacheMax: number = $ref(0); +let perRemoteUserUserTimelineCacheMax: number = $ref(0); +let perUserHomeTimelineCacheMax: number = $ref(0); +let perUserListTimelineCacheMax: number = $ref(0); async function init(): Promise { const meta = await os.api('admin/meta'); @@ -142,8 +152,10 @@ async function init(): Promise { enableServiceWorker = meta.enableServiceWorker; swPublicKey = meta.swPublickey; swPrivateKey = meta.swPrivateKey; - deeplAuthKey = meta.deeplAuthKey; - deeplIsPro = meta.deeplIsPro; + perLocalUserUserTimelineCacheMax = meta.perLocalUserUserTimelineCacheMax; + perRemoteUserUserTimelineCacheMax = meta.perRemoteUserUserTimelineCacheMax; + perUserHomeTimelineCacheMax = meta.perUserHomeTimelineCacheMax; + perUserListTimelineCacheMax = meta.perUserListTimelineCacheMax; } function save(): void { @@ -158,8 +170,10 @@ function save(): void { enableServiceWorker, swPublicKey, swPrivateKey, - deeplAuthKey, - deeplIsPro, + perLocalUserUserTimelineCacheMax, + perRemoteUserUserTimelineCacheMax, + perUserHomeTimelineCacheMax, + perUserListTimelineCacheMax, }).then(() => { fetchInstance(); }); diff --git a/packages/frontend/src/router.ts b/packages/frontend/src/router.ts index 52ca09be8b..3ee4562470 100644 --- a/packages/frontend/src/router.ts +++ b/packages/frontend/src/router.ts @@ -429,6 +429,10 @@ export const routes = [{ path: '/proxy-account', name: 'proxy-account', component: page(() => import('./pages/admin/proxy-account.vue')), + }, { + path: '/external-services', + name: 'external-services', + component: page(() => import('./pages/admin/external-services.vue')), }, { path: '/other-settings', name: 'other-settings', From 2053f6496e00562e59048e7c2de0e7cadfedc9ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=81=BE=E3=81=A3=E3=81=A1=E3=82=83=E3=81=A8=E3=83=BC?= =?UTF-8?q?=E3=81=AB=E3=82=85?= <17376330+u1-liquid@users.noreply.github.com> Date: Sat, 21 Oct 2023 03:38:25 +0900 Subject: [PATCH 2/2] Bump up version to 13.14.2-io.12 (MisskeyIO#189) --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index edee573a2d..efef451588 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "misskey", - "version": "13.14.2-io.11", + "version": "13.14.2-io.12", "codename": "nasubi", "repository": { "type": "git",