From a631b976c99a4b3977079a4bafc8a7b7de0bf269 Mon Sep 17 00:00:00 2001 From: anatawa12 Date: Sat, 2 Dec 2023 18:25:07 +0900 Subject: [PATCH] Refine fanout timeline (#12507) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(endpoints/hybrid-timeline): don't pack inside getFromDb * chore(endpoints/hybrid-timeline): Redisから取得する部分のうちSTLに依存しなそうなところを別のServiceに切り出し * chore(endpoints/local-timeline): FanoutTimelineEndpointServiceで再実装 * chore(endpoints/channels/timeline): FanoutTimelineEndpointServiceで再実装 * chore(endpoints/timeline): FanoutTimelineEndpointServiceで再実装 * chore(endpoints/user-list-timeline): FanoutTimelineEndpointServiceで再実装 * chore(endpoints/users/notes): FanoutTimelineEndpointServiceで再実装 * chore: add useDbFallback to FanoutTimelineEndpointService.timeline and always true for channel / user note list * style: fix lint error * chore: split logic to multiple functions * chore: implement redis fallback * chore: 成功率を上げる * fix: db fallback not working * feat: allowPartial * chore(frontend): set allowPartial * chore(backend): remove fallbackIfEmpty HTL will never be purged so it's no longer required * fix: missing allowPartial in channel timeline * fix: type of timelineConfig in hybrid-timeline --------- Co-authored-by: syuilo --- packages/backend/src/core/CoreModule.ts | 6 + .../src/core/FanoutTimelineEndpointService.ts | 123 +++++++++++ .../server/api/endpoints/channels/timeline.ts | 112 +++++----- .../api/endpoints/notes/hybrid-timeline.ts | 110 ++++----- .../api/endpoints/notes/local-timeline.ts | 103 ++++----- .../server/api/endpoints/notes/timeline.ts | 91 ++++---- .../api/endpoints/notes/user-list-timeline.ts | 78 +++---- .../src/server/api/endpoints/users/notes.ts | 208 +++++++++--------- .../frontend/src/components/MkPagination.vue | 1 + 9 files changed, 438 insertions(+), 394 deletions(-) create mode 100644 packages/backend/src/core/FanoutTimelineEndpointService.ts diff --git a/packages/backend/src/core/CoreModule.ts b/packages/backend/src/core/CoreModule.ts index bf6f0ef879..bc6d24b951 100644 --- a/packages/backend/src/core/CoreModule.ts +++ b/packages/backend/src/core/CoreModule.ts @@ -4,6 +4,7 @@ */ import { Module } from '@nestjs/common'; +import { FanoutTimelineEndpointService } from '@/core/FanoutTimelineEndpointService.js'; import { AccountMoveService } from './AccountMoveService.js'; import { AccountUpdateService } from './AccountUpdateService.js'; import { AiService } from './AiService.js'; @@ -195,6 +196,7 @@ const $SearchService: Provider = { provide: 'SearchService', useExisting: Search const $ClipService: Provider = { provide: 'ClipService', useExisting: ClipService }; const $FeaturedService: Provider = { provide: 'FeaturedService', useExisting: FeaturedService }; const $FanoutTimelineService: Provider = { provide: 'FanoutTimelineService', useExisting: FanoutTimelineService }; +const $FanoutTimelineEndpointService: Provider = { provide: 'FanoutTimelineEndpointService', useExisting: FanoutTimelineEndpointService }; const $ChannelFollowingService: Provider = { provide: 'ChannelFollowingService', useExisting: ChannelFollowingService }; const $RegistryApiService: Provider = { provide: 'RegistryApiService', useExisting: RegistryApiService }; @@ -331,6 +333,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting ClipService, FeaturedService, FanoutTimelineService, + FanoutTimelineEndpointService, ChannelFollowingService, RegistryApiService, ChartLoggerService, @@ -460,6 +463,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting $ClipService, $FeaturedService, $FanoutTimelineService, + $FanoutTimelineEndpointService, $ChannelFollowingService, $RegistryApiService, $ChartLoggerService, @@ -590,6 +594,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting ClipService, FeaturedService, FanoutTimelineService, + FanoutTimelineEndpointService, ChannelFollowingService, RegistryApiService, FederationChart, @@ -718,6 +723,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting $ClipService, $FeaturedService, $FanoutTimelineService, + $FanoutTimelineEndpointService, $ChannelFollowingService, $RegistryApiService, $FederationChart, diff --git a/packages/backend/src/core/FanoutTimelineEndpointService.ts b/packages/backend/src/core/FanoutTimelineEndpointService.ts new file mode 100644 index 0000000000..157fcbe877 --- /dev/null +++ b/packages/backend/src/core/FanoutTimelineEndpointService.ts @@ -0,0 +1,123 @@ +/* + * SPDX-FileCopyrightText: syuilo and other misskey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { Inject, Injectable } from '@nestjs/common'; +import { DI } from '@/di-symbols.js'; +import { bindThis } from '@/decorators.js'; +import type { MiUser } from '@/models/User.js'; +import type { MiNote } from '@/models/Note.js'; +import { Packed } from '@/misc/json-schema.js'; +import type { NotesRepository } from '@/models/_.js'; +import { NoteEntityService } from '@/core/entities/NoteEntityService.js'; +import { FanoutTimelineService } from '@/core/FanoutTimelineService.js'; + +@Injectable() +export class FanoutTimelineEndpointService { + constructor( + @Inject(DI.notesRepository) + private notesRepository: NotesRepository, + + private noteEntityService: NoteEntityService, + private fanoutTimelineService: FanoutTimelineService, + ) { + } + + @bindThis + async timeline(ps: { + untilId: string | null, + sinceId: string | null, + limit: number, + allowPartial: boolean, + me?: { id: MiUser['id'] } | undefined | null, + useDbFallback: boolean, + redisTimelines: string[], + noteFilter: (note: MiNote) => boolean, + dbFallback: (untilId: string | null, sinceId: string | null, limit: number) => Promise, + }): Promise[]> { + return await this.noteEntityService.packMany(await this.getMiNotes(ps), ps.me); + } + + @bindThis + private async getMiNotes(ps: { + untilId: string | null, + sinceId: string | null, + limit: number, + allowPartial: boolean, + me?: { id: MiUser['id'] } | undefined | null, + useDbFallback: boolean, + redisTimelines: string[], + noteFilter: (note: MiNote) => boolean, + dbFallback: (untilId: string | null, sinceId: string | null, limit: number) => Promise, + }): Promise { + let noteIds: string[]; + let shouldFallbackToDb = false; + + // 呼び出し元と以下の処理をシンプルにするためにdbFallbackを置き換える + if (!ps.useDbFallback) ps.dbFallback = () => Promise.resolve([]); + + const redisResult = await this.fanoutTimelineService.getMulti(ps.redisTimelines, ps.untilId, ps.sinceId); + + const redisResultIds = Array.from(new Set(redisResult.flat(1))); + + redisResultIds.sort((a, b) => a > b ? -1 : 1); + noteIds = redisResultIds.slice(0, ps.limit); + + shouldFallbackToDb = shouldFallbackToDb || (noteIds.length === 0); + + if (!shouldFallbackToDb) { + const redisTimeline: MiNote[] = []; + let readFromRedis = 0; + let lastSuccessfulRate = 1; // rateをキャッシュする？ + let trialCount = 1; + + while ((redisResultIds.length - readFromRedis) !== 0) { + const remainingToRead = ps.limit - redisTimeline.length; + + // DBからの取り直しを減らす初回と同じ割合以上で成功すると仮定するが、クエリの長さを考えて三倍まで + const countToGet = remainingToRead * Math.ceil(Math.min(1.1 / lastSuccessfulRate, 3)); + noteIds = redisResultIds.slice(readFromRedis, readFromRedis + countToGet); + + readFromRedis += noteIds.length; + + const gotFromDb = await this.getAndFilterFromDb(noteIds, ps.noteFilter); + redisTimeline.push(...gotFromDb); + lastSuccessfulRate = gotFromDb.length / noteIds.length; + + console.log(`fanoutTimelineTrial#${trialCount++}: req: ${ps.limit}, tried: ${noteIds.length}, got: ${gotFromDb.length}, rate: ${lastSuccessfulRate}, total: ${redisTimeline.length}, fromRedis: ${redisResultIds.length}`); + + if (ps.allowPartial ? redisTimeline.length !== 0 : redisTimeline.length >= ps.limit) { + // 十分Redisからとれた + return redisTimeline.slice(0, ps.limit); + } + } + + // まだ足りない分はDBにフォールバック + const remainingToRead = ps.limit - redisTimeline.length; + const gotFromDb = await ps.dbFallback(noteIds[noteIds.length - 1], ps.sinceId, remainingToRead); + redisTimeline.push(...gotFromDb); + console.log(`fanoutTimelineTrial#db: req: ${ps.limit}, tried: ${remainingToRead}, got: ${gotFromDb.length}, since: ${noteIds[noteIds.length - 1]}, until: ${ps.untilId}, total: ${redisTimeline.length}`); + return redisTimeline; + } + + return await ps.dbFallback(ps.untilId, ps.sinceId, ps.limit); + } + + private async getAndFilterFromDb(noteIds: string[], noteFilter: (note: MiNote) => boolean): Promise { + const query = this.notesRepository.createQueryBuilder('note') + .where('note.id IN (:...noteIds)', { noteIds: noteIds }) + .innerJoinAndSelect('note.user', 'user') + .leftJoinAndSelect('note.reply', 'reply') + .leftJoinAndSelect('note.renote', 'renote') + .leftJoinAndSelect('reply.user', 'replyUser') + .leftJoinAndSelect('renote.user', 'renoteUser') + .leftJoinAndSelect('note.channel', 'channel'); + + const notes = (await query.getMany()).filter(noteFilter); + + notes.sort((a, b) => a.id > b.id ? -1 : 1); + + return notes; + } +} diff --git a/packages/backend/src/server/api/endpoints/channels/timeline.ts b/packages/backend/src/server/api/endpoints/channels/timeline.ts index f9207199d6..9ef494d6d8 100644 --- a/packages/backend/src/server/api/endpoints/channels/timeline.ts +++ b/packages/backend/src/server/api/endpoints/channels/timeline.ts @@ -12,10 +12,11 @@ import { NoteEntityService } from '@/core/entities/NoteEntityService.js'; import ActiveUsersChart from '@/core/chart/charts/active-users.js'; import { DI } from '@/di-symbols.js'; import { IdService } from '@/core/IdService.js'; -import { FanoutTimelineService } from '@/core/FanoutTimelineService.js'; import { isUserRelated } from '@/misc/is-user-related.js'; import { CacheService } from '@/core/CacheService.js'; import { MetaService } from '@/core/MetaService.js'; +import { FanoutTimelineEndpointService } from '@/core/FanoutTimelineEndpointService.js'; +import { MiLocalUser } from '@/models/User.js'; import { ApiError } from '../../error.js'; export const meta = { @@ -51,6 +52,7 @@ export const paramDef = { untilId: { type: 'string', format: 'misskey:id' }, sinceDate: { type: 'integer' }, untilDate: { type: 'integer' }, + allowPartial: { type: 'boolean', default: false }, // true is recommended but for compatibility false by default }, required: ['channelId'], } as const; @@ -58,9 +60,6 @@ export const paramDef = { @Injectable() export default class extends Endpoint { // eslint-disable-line import/no-default-export constructor( - @Inject(DI.redisForTimelines) - private redisForTimelines: Redis.Redis, - @Inject(DI.notesRepository) private notesRepository: NotesRepository, @@ -70,7 +69,7 @@ export default class extends Endpoint { // eslint- private idService: IdService, private noteEntityService: NoteEntityService, private queryService: QueryService, - private fanoutTimelineService: FanoutTimelineService, + private fanoutTimelineEndpointService: FanoutTimelineEndpointService, private cacheService: CacheService, private activeUsersChart: ActiveUsersChart, private metaService: MetaService, @@ -78,7 +77,6 @@ export default class extends Endpoint { // eslint- super(meta, paramDef, async (ps, me) => { const untilId = ps.untilId ?? (ps.untilDate ? this.idService.gen(ps.untilDate!) : null); const sinceId = ps.sinceId ?? (ps.sinceDate ? this.idService.gen(ps.sinceDate!) : null); - const isRangeSpecified = untilId != null && sinceId != null; const serverSettings = await this.metaService.fetch(); @@ -92,64 +90,58 @@ export default class extends Endpoint { // eslint- if (me) this.activeUsersChart.read(me); - if (serverSettings.enableFanoutTimeline && (isRangeSpecified || sinceId == null)) { - const [ - userIdsWhoMeMuting, - ] = me ? await Promise.all([ - this.cacheService.userMutingsCache.fetch(me.id), - ]) : [new Set()]; - - let noteIds = await this.fanoutTimelineService.get(`channelTimeline:${channel.id}`, untilId, sinceId); - noteIds = noteIds.slice(0, ps.limit); - - if (noteIds.length > 0) { - const query = this.notesRepository.createQueryBuilder('note') - .where('note.id IN (:...noteIds)', { noteIds: noteIds }) - .innerJoinAndSelect('note.user', 'user') - .leftJoinAndSelect('note.reply', 'reply') - .leftJoinAndSelect('note.renote', 'renote') - .leftJoinAndSelect('reply.user', 'replyUser') - .leftJoinAndSelect('renote.user', 'renoteUser') - .leftJoinAndSelect('note.channel', 'channel'); - - let timeline = await query.getMany(); - - timeline = timeline.filter(note => { - if (me && isUserRelated(note, userIdsWhoMeMuting)) return false; - - return true; - }); - - // TODO: フィルタで件数が減った場合の埋め合わせ処理 - - timeline.sort((a, b) => a.id > b.id ? -1 : 1); - - if (timeline.length > 0) { - return await this.noteEntityService.packMany(timeline, me); - } - } + if (!serverSettings.enableFanoutTimeline) { + return await this.noteEntityService.packMany(await this.getFromDb({ untilId, sinceId, limit: ps.limit, channelId: channel.id }, me), me); } - //#region fallback to database - const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), ps.sinceId, ps.untilId, ps.sinceDate, ps.untilDate) - .andWhere('note.channelId = :channelId', { channelId: channel.id }) - .innerJoinAndSelect('note.user', 'user') - .leftJoinAndSelect('note.reply', 'reply') - .leftJoinAndSelect('note.renote', 'renote') - .leftJoinAndSelect('reply.user', 'replyUser') - .leftJoinAndSelect('renote.user', 'renoteUser') - .leftJoinAndSelect('note.channel', 'channel'); + const [ + userIdsWhoMeMuting, + ] = me ? await Promise.all([ + this.cacheService.userMutingsCache.fetch(me.id), + ]) : [new Set()]; - if (me) { - this.queryService.generateMutedUserQuery(query, me); - this.queryService.generateBlockedUserQuery(query, me); - } - //#endregion + return await this.fanoutTimelineEndpointService.timeline({ + untilId, + sinceId, + limit: ps.limit, + allowPartial: ps.allowPartial, + me, + useDbFallback: true, + redisTimelines: [`channelTimeline:${channel.id}`], + noteFilter: note => { + if (me && isUserRelated(note, userIdsWhoMeMuting)) return false; - const timeline = await query.limit(ps.limit).getMany(); - - return await this.noteEntityService.packMany(timeline, me); - //#endregion + return true; + }, + dbFallback: async (untilId, sinceId, limit) => { + return await this.getFromDb({ untilId, sinceId, limit, channelId: channel.id }, me); + }, + }); }); } + + private async getFromDb(ps: { + untilId: string | null, + sinceId: string | null, + limit: number, + channelId: string + }, me: MiLocalUser | null) { + //#region fallback to database + const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), ps.sinceId, ps.untilId) + .andWhere('note.channelId = :channelId', { channelId: ps.channelId }) + .innerJoinAndSelect('note.user', 'user') + .leftJoinAndSelect('note.reply', 'reply') + .leftJoinAndSelect('note.renote', 'renote') + .leftJoinAndSelect('reply.user', 'replyUser') + .leftJoinAndSelect('renote.user', 'renoteUser') + .leftJoinAndSelect('note.channel', 'channel'); + + if (me) { + this.queryService.generateMutedUserQuery(query, me); + this.queryService.generateBlockedUserQuery(query, me); + } + //#endregion + + return await query.limit(ps.limit).getMany(); + } } diff --git a/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts b/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts index 372199844d..820692626c 100644 --- a/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts @@ -5,7 +5,7 @@ import { Brackets } from 'typeorm'; import { Inject, Injectable } from '@nestjs/common'; -import type { NotesRepository, FollowingsRepository, MiNote, ChannelFollowingsRepository } from '@/models/_.js'; +import type { NotesRepository, ChannelFollowingsRepository } from '@/models/_.js'; import { Endpoint } from '@/server/api/endpoint-base.js'; import ActiveUsersChart from '@/core/chart/charts/active-users.js'; import { NoteEntityService } from '@/core/entities/NoteEntityService.js'; @@ -19,6 +19,7 @@ import { QueryService } from '@/core/QueryService.js'; import { UserFollowingService } from '@/core/UserFollowingService.js'; import { MetaService } from '@/core/MetaService.js'; import { MiLocalUser } from '@/models/User.js'; +import { FanoutTimelineEndpointService } from '@/core/FanoutTimelineEndpointService.js'; import { ApiError } from '../../error.js'; export const meta = { @@ -53,6 +54,7 @@ export const paramDef = { untilId: { type: 'string', format: 'misskey:id' }, sinceDate: { type: 'integer' }, untilDate: { type: 'integer' }, + allowPartial: { type: 'boolean', default: false }, // true is recommended but for compatibility false by default includeMyRenotes: { type: 'boolean', default: true }, includeRenotedMyNotes: { type: 'boolean', default: true }, includeLocalRenotes: { type: 'boolean', default: true }, @@ -77,10 +79,10 @@ export default class extends Endpoint { // eslint- private activeUsersChart: ActiveUsersChart, private idService: IdService, private cacheService: CacheService, - private fanoutTimelineService: FanoutTimelineService, private queryService: QueryService, private userFollowingService: UserFollowingService, private metaService: MetaService, + private fanoutTimelineEndpointService: FanoutTimelineEndpointService, ) { super(meta, paramDef, async (ps, me) => { const untilId = ps.untilId ?? (ps.untilDate ? this.idService.gen(ps.untilDate!) : null); @@ -94,7 +96,7 @@ export default class extends Endpoint { // eslint- const serverSettings = await this.metaService.fetch(); if (!serverSettings.enableFanoutTimeline) { - return await this.getFromDb({ + const timeline = await this.getFromDb({ untilId, sinceId, limit: ps.limit, @@ -104,6 +106,12 @@ export default class extends Endpoint { // eslint- withFiles: ps.withFiles, withReplies: ps.withReplies, }, me); + + process.nextTick(() => { + this.activeUsersChart.read(me); + }); + + return await this.noteEntityService.packMany(timeline, me); } const [ @@ -116,51 +124,34 @@ export default class extends Endpoint { // eslint- this.cacheService.userBlockedCache.fetch(me.id), ]); - let noteIds: string[]; - let shouldFallbackToDb = false; + let timelineConfig: string[]; if (ps.withFiles) { - const [htlNoteIds, ltlNoteIds] = await this.fanoutTimelineService.getMulti([ + timelineConfig = [ `homeTimelineWithFiles:${me.id}`, 'localTimelineWithFiles', - ], untilId, sinceId); - noteIds = Array.from(new Set([...htlNoteIds, ...ltlNoteIds])); + ]; } else if (ps.withReplies) { - const [htlNoteIds, ltlNoteIds, ltlReplyNoteIds] = await this.fanoutTimelineService.getMulti([ + timelineConfig = [ `homeTimeline:${me.id}`, 'localTimeline', 'localTimelineWithReplies', - ], untilId, sinceId); - noteIds = Array.from(new Set([...htlNoteIds, ...ltlNoteIds, ...ltlReplyNoteIds])); + ]; } else { - const [htlNoteIds, ltlNoteIds] = await this.fanoutTimelineService.getMulti([ + timelineConfig = [ `homeTimeline:${me.id}`, 'localTimeline', - ], untilId, sinceId); - noteIds = Array.from(new Set([...htlNoteIds, ...ltlNoteIds])); - shouldFallbackToDb = htlNoteIds.length === 0; + ]; } - noteIds.sort((a, b) => a > b ? -1 : 1); - noteIds = noteIds.slice(0, ps.limit); - - shouldFallbackToDb = shouldFallbackToDb || (noteIds.length === 0); - - let redisTimeline: MiNote[] = []; - - if (!shouldFallbackToDb) { - const query = this.notesRepository.createQueryBuilder('note') - .where('note.id IN (:...noteIds)', { noteIds: noteIds }) - .innerJoinAndSelect('note.user', 'user') - .leftJoinAndSelect('note.reply', 'reply') - .leftJoinAndSelect('note.renote', 'renote') - .leftJoinAndSelect('reply.user', 'replyUser') - .leftJoinAndSelect('renote.user', 'renoteUser') - .leftJoinAndSelect('note.channel', 'channel'); - - redisTimeline = await query.getMany(); - - redisTimeline = redisTimeline.filter(note => { + const redisTimeline = await this.fanoutTimelineEndpointService.timeline({ + untilId, + sinceId, + limit: ps.limit, + allowPartial: ps.allowPartial, + redisTimelines: timelineConfig, + useDbFallback: serverSettings.enableFanoutTimelineDbFallback, + noteFilter: (note) => { if (note.userId === me.id) { return true; } @@ -174,33 +165,24 @@ export default class extends Endpoint { // eslint- } return true; - }); + }, + dbFallback: async (untilId, sinceId, limit) => await this.getFromDb({ + untilId, + sinceId, + limit, + includeMyRenotes: ps.includeMyRenotes, + includeRenotedMyNotes: ps.includeRenotedMyNotes, + includeLocalRenotes: ps.includeLocalRenotes, + withFiles: ps.withFiles, + withReplies: ps.withReplies, + }, me), + }); - redisTimeline.sort((a, b) => a.id > b.id ? -1 : 1); - } + process.nextTick(() => { + this.activeUsersChart.read(me); + }); - if (redisTimeline.length > 0) { - process.nextTick(() => { - this.activeUsersChart.read(me); - }); - - return await this.noteEntityService.packMany(redisTimeline, me); - } else { - if (serverSettings.enableFanoutTimelineDbFallback) { // fallback to db - return await this.getFromDb({ - untilId, - sinceId, - limit: ps.limit, - includeMyRenotes: ps.includeMyRenotes, - includeRenotedMyNotes: ps.includeRenotedMyNotes, - includeLocalRenotes: ps.includeLocalRenotes, - withFiles: ps.withFiles, - withReplies: ps.withReplies, - }, me); - } else { - return []; - } - } + return redisTimeline; }); } @@ -301,12 +283,6 @@ export default class extends Endpoint { // eslint- } //#endregion - const timeline = await query.limit(ps.limit).getMany(); - - process.nextTick(() => { - this.activeUsersChart.read(me); - }); - - return await this.noteEntityService.packMany(timeline, me); + return await query.limit(ps.limit).getMany(); } } diff --git a/packages/backend/src/server/api/endpoints/notes/local-timeline.ts b/packages/backend/src/server/api/endpoints/notes/local-timeline.ts index 8867070055..97b05016ec 100644 --- a/packages/backend/src/server/api/endpoints/notes/local-timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/local-timeline.ts @@ -14,10 +14,10 @@ import { RoleService } from '@/core/RoleService.js'; import { IdService } from '@/core/IdService.js'; import { CacheService } from '@/core/CacheService.js'; import { isUserRelated } from '@/misc/is-user-related.js'; -import { FanoutTimelineService } from '@/core/FanoutTimelineService.js'; import { QueryService } from '@/core/QueryService.js'; import { MetaService } from '@/core/MetaService.js'; import { MiLocalUser } from '@/models/User.js'; +import { FanoutTimelineEndpointService } from '@/core/FanoutTimelineEndpointService.js'; import { ApiError } from '../../error.js'; export const meta = { @@ -51,6 +51,7 @@ export const paramDef = { limit: { type: 'integer', minimum: 1, maximum: 100, default: 10 }, sinceId: { type: 'string', format: 'misskey:id' }, untilId: { type: 'string', format: 'misskey:id' }, + allowPartial: { type: 'boolean', default: false }, // true is recommended but for compatibility false by default sinceDate: { type: 'integer' }, untilDate: { type: 'integer' }, }, @@ -68,7 +69,7 @@ export default class extends Endpoint { // eslint- private activeUsersChart: ActiveUsersChart, private idService: IdService, private cacheService: CacheService, - private fanoutTimelineService: FanoutTimelineService, + private fanoutTimelineEndpointService: FanoutTimelineEndpointService, private queryService: QueryService, private metaService: MetaService, ) { @@ -84,13 +85,21 @@ export default class extends Endpoint { // eslint- const serverSettings = await this.metaService.fetch(); if (!serverSettings.enableFanoutTimeline) { - return await this.getFromDb({ + const timeline = await this.getFromDb({ untilId, sinceId, limit: ps.limit, withFiles: ps.withFiles, withReplies: ps.withReplies, }, me); + + process.nextTick(() => { + if (me) { + this.activeUsersChart.read(me); + } + }); + + return await this.noteEntityService.packMany(timeline, me); } const [ @@ -103,36 +112,15 @@ export default class extends Endpoint { // eslint- this.cacheService.userBlockedCache.fetch(me.id), ]) : [new Set(), new Set(), new Set()]; - let noteIds: string[]; - - if (ps.withFiles) { - noteIds = await this.fanoutTimelineService.get('localTimelineWithFiles', untilId, sinceId); - } else { - const [nonReplyNoteIds, replyNoteIds] = await this.fanoutTimelineService.getMulti([ - 'localTimeline', - 'localTimelineWithReplies', - ], untilId, sinceId); - noteIds = Array.from(new Set([...nonReplyNoteIds, ...replyNoteIds])); - noteIds.sort((a, b) => a > b ? -1 : 1); - } - - noteIds = noteIds.slice(0, ps.limit); - - let redisTimeline: MiNote[] = []; - - if (noteIds.length > 0) { - const query = this.notesRepository.createQueryBuilder('note') - .where('note.id IN (:...noteIds)', { noteIds: noteIds }) - .innerJoinAndSelect('note.user', 'user') - .leftJoinAndSelect('note.reply', 'reply') - .leftJoinAndSelect('note.renote', 'renote') - .leftJoinAndSelect('reply.user', 'replyUser') - .leftJoinAndSelect('renote.user', 'renoteUser') - .leftJoinAndSelect('note.channel', 'channel'); - - redisTimeline = await query.getMany(); - - redisTimeline = redisTimeline.filter(note => { + const timeline = await this.fanoutTimelineEndpointService.timeline({ + untilId, + sinceId, + limit: ps.limit, + allowPartial: ps.allowPartial, + me, + useDbFallback: serverSettings.enableFanoutTimelineDbFallback, + redisTimelines: ps.withFiles ? ['localTimelineWithFiles'] : ['localTimeline', 'localTimelineWithReplies'], + noteFilter: note => { if (me && (note.userId === me.id)) { return true; } @@ -147,32 +135,23 @@ export default class extends Endpoint { // eslint- } return true; - }); + }, + dbFallback: async (untilId, sinceId, limit) => await this.getFromDb({ + untilId, + sinceId, + limit, + withFiles: ps.withFiles, + withReplies: ps.withReplies, + }, me), + }); - redisTimeline.sort((a, b) => a.id > b.id ? -1 : 1); - } - - if (redisTimeline.length > 0) { - process.nextTick(() => { - if (me) { - this.activeUsersChart.read(me); - } - }); - - return await this.noteEntityService.packMany(redisTimeline, me); - } else { - if (serverSettings.enableFanoutTimelineDbFallback) { // fallback to db - return await this.getFromDb({ - untilId, - sinceId, - limit: ps.limit, - withFiles: ps.withFiles, - withReplies: ps.withReplies, - }, me); - } else { - return []; + process.nextTick(() => { + if (me) { + this.activeUsersChart.read(me); } - } + }); + + return timeline; }); } @@ -213,14 +192,6 @@ export default class extends Endpoint { // eslint- })); } - const timeline = await query.limit(ps.limit).getMany(); - - process.nextTick(() => { - if (me) { - this.activeUsersChart.read(me); - } - }); - - return await this.noteEntityService.packMany(timeline, me); + return await query.limit(ps.limit).getMany(); } } diff --git a/packages/backend/src/server/api/endpoints/notes/timeline.ts b/packages/backend/src/server/api/endpoints/notes/timeline.ts index 470abe0b14..74d0a6e0c0 100644 --- a/packages/backend/src/server/api/endpoints/notes/timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/timeline.ts @@ -5,7 +5,7 @@ import { Brackets } from 'typeorm'; import { Inject, Injectable } from '@nestjs/common'; -import type { MiNote, NotesRepository, ChannelFollowingsRepository } from '@/models/_.js'; +import type { NotesRepository, ChannelFollowingsRepository } from '@/models/_.js'; import { Endpoint } from '@/server/api/endpoint-base.js'; import { QueryService } from '@/core/QueryService.js'; import ActiveUsersChart from '@/core/chart/charts/active-users.js'; @@ -14,10 +14,10 @@ import { DI } from '@/di-symbols.js'; import { IdService } from '@/core/IdService.js'; import { CacheService } from '@/core/CacheService.js'; import { isUserRelated } from '@/misc/is-user-related.js'; -import { FanoutTimelineService } from '@/core/FanoutTimelineService.js'; import { UserFollowingService } from '@/core/UserFollowingService.js'; import { MiLocalUser } from '@/models/User.js'; import { MetaService } from '@/core/MetaService.js'; +import { FanoutTimelineEndpointService } from '@/core/FanoutTimelineEndpointService.js'; export const meta = { tags: ['notes'], @@ -43,6 +43,7 @@ export const paramDef = { untilId: { type: 'string', format: 'misskey:id' }, sinceDate: { type: 'integer' }, untilDate: { type: 'integer' }, + allowPartial: { type: 'boolean', default: false }, // true is recommended but for compatibility false by default includeMyRenotes: { type: 'boolean', default: true }, includeRenotedMyNotes: { type: 'boolean', default: true }, includeLocalRenotes: { type: 'boolean', default: true }, @@ -65,7 +66,7 @@ export default class extends Endpoint { // eslint- private activeUsersChart: ActiveUsersChart, private idService: IdService, private cacheService: CacheService, - private fanoutTimelineService: FanoutTimelineService, + private fanoutTimelineEndpointService: FanoutTimelineEndpointService, private userFollowingService: UserFollowingService, private queryService: QueryService, private metaService: MetaService, @@ -77,7 +78,7 @@ export default class extends Endpoint { // eslint- const serverSettings = await this.metaService.fetch(); if (!serverSettings.enableFanoutTimeline) { - return await this.getFromDb({ + const timeline = await this.getFromDb({ untilId, sinceId, limit: ps.limit, @@ -87,6 +88,12 @@ export default class extends Endpoint { // eslint- withFiles: ps.withFiles, withRenotes: ps.withRenotes, }, me); + + process.nextTick(() => { + this.activeUsersChart.read(me); + }); + + return await this.noteEntityService.packMany(timeline, me); } const [ @@ -101,24 +108,15 @@ export default class extends Endpoint { // eslint- this.cacheService.userBlockedCache.fetch(me.id), ]); - let noteIds = await this.fanoutTimelineService.get(ps.withFiles ? `homeTimelineWithFiles:${me.id}` : `homeTimeline:${me.id}`, untilId, sinceId); - noteIds = noteIds.slice(0, ps.limit); - - let redisTimeline: MiNote[] = []; - - if (noteIds.length > 0) { - const query = this.notesRepository.createQueryBuilder('note') - .where('note.id IN (:...noteIds)', { noteIds: noteIds }) - .innerJoinAndSelect('note.user', 'user') - .leftJoinAndSelect('note.reply', 'reply') - .leftJoinAndSelect('note.renote', 'renote') - .leftJoinAndSelect('reply.user', 'replyUser') - .leftJoinAndSelect('renote.user', 'renoteUser') - .leftJoinAndSelect('note.channel', 'channel'); - - redisTimeline = await query.getMany(); - - redisTimeline = redisTimeline.filter(note => { + const timeline = this.fanoutTimelineEndpointService.timeline({ + untilId, + sinceId, + limit: ps.limit, + allowPartial: ps.allowPartial, + me, + useDbFallback: serverSettings.enableFanoutTimelineDbFallback, + redisTimelines: ps.withFiles ? [`homeTimelineWithFiles:${me.id}`] : [`homeTimeline:${me.id}`], + noteFilter: note => { if (note.userId === me.id) { return true; } @@ -135,33 +133,24 @@ export default class extends Endpoint { // eslint- } return true; - }); + }, + dbFallback: async (untilId, sinceId, limit) => await this.getFromDb({ + untilId, + sinceId, + limit, + includeMyRenotes: ps.includeMyRenotes, + includeRenotedMyNotes: ps.includeRenotedMyNotes, + includeLocalRenotes: ps.includeLocalRenotes, + withFiles: ps.withFiles, + withRenotes: ps.withRenotes, + }, me), + }); - redisTimeline.sort((a, b) => a.id > b.id ? -1 : 1); - } + process.nextTick(() => { + this.activeUsersChart.read(me); + }); - if (redisTimeline.length > 0) { - process.nextTick(() => { - this.activeUsersChart.read(me); - }); - - return await this.noteEntityService.packMany(redisTimeline, me); - } else { - if (serverSettings.enableFanoutTimelineDbFallback) { // fallback to db - return await this.getFromDb({ - untilId, - sinceId, - limit: ps.limit, - includeMyRenotes: ps.includeMyRenotes, - includeRenotedMyNotes: ps.includeRenotedMyNotes, - includeLocalRenotes: ps.includeLocalRenotes, - withFiles: ps.withFiles, - withRenotes: ps.withRenotes, - }, me); - } else { - return []; - } - } + return timeline; }); } @@ -269,12 +258,6 @@ export default class extends Endpoint { // eslint- } //#endregion - const timeline = await query.limit(ps.limit).getMany(); - - process.nextTick(() => { - this.activeUsersChart.read(me); - }); - - return await this.noteEntityService.packMany(timeline, me); + return await query.limit(ps.limit).getMany(); } } diff --git a/packages/backend/src/server/api/endpoints/notes/user-list-timeline.ts b/packages/backend/src/server/api/endpoints/notes/user-list-timeline.ts index 1ac1d37f48..f39cac5c3e 100644 --- a/packages/backend/src/server/api/endpoints/notes/user-list-timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/user-list-timeline.ts @@ -17,6 +17,7 @@ import { FanoutTimelineService } from '@/core/FanoutTimelineService.js'; import { QueryService } from '@/core/QueryService.js'; import { MiLocalUser } from '@/models/User.js'; import { MetaService } from '@/core/MetaService.js'; +import { FanoutTimelineEndpointService } from '@/core/FanoutTimelineEndpointService.js'; import { ApiError } from '../../error.js'; export const meta = { @@ -52,6 +53,7 @@ export const paramDef = { untilId: { type: 'string', format: 'misskey:id' }, sinceDate: { type: 'integer' }, untilDate: { type: 'integer' }, + allowPartial: { type: 'boolean', default: false }, // true is recommended but for compatibility false by default includeMyRenotes: { type: 'boolean', default: true }, includeRenotedMyNotes: { type: 'boolean', default: true }, includeLocalRenotes: { type: 'boolean', default: true }, @@ -82,6 +84,7 @@ export default class extends Endpoint { // eslint- private cacheService: CacheService, private idService: IdService, private fanoutTimelineService: FanoutTimelineService, + private fanoutTimelineEndpointService: FanoutTimelineEndpointService, private queryService: QueryService, private metaService: MetaService, ) { @@ -101,7 +104,7 @@ export default class extends Endpoint { // eslint- const serverSettings = await this.metaService.fetch(); if (!serverSettings.enableFanoutTimeline) { - return await this.getFromDb(list, { + const timeline = await this.getFromDb(list, { untilId, sinceId, limit: ps.limit, @@ -111,6 +114,10 @@ export default class extends Endpoint { // eslint- withFiles: ps.withFiles, withRenotes: ps.withRenotes, }, me); + + this.activeUsersChart.read(me); + + await this.noteEntityService.packMany(timeline, me); } const [ @@ -123,24 +130,15 @@ export default class extends Endpoint { // eslint- this.cacheService.userBlockedCache.fetch(me.id), ]); - let noteIds = await this.fanoutTimelineService.get(ps.withFiles ? `userListTimelineWithFiles:${list.id}` : `userListTimeline:${list.id}`, untilId, sinceId); - noteIds = noteIds.slice(0, ps.limit); - - let redisTimeline: MiNote[] = []; - - if (noteIds.length > 0) { - const query = this.notesRepository.createQueryBuilder('note') - .where('note.id IN (:...noteIds)', { noteIds: noteIds }) - .innerJoinAndSelect('note.user', 'user') - .leftJoinAndSelect('note.reply', 'reply') - .leftJoinAndSelect('note.renote', 'renote') - .leftJoinAndSelect('reply.user', 'replyUser') - .leftJoinAndSelect('renote.user', 'renoteUser') - .leftJoinAndSelect('note.channel', 'channel'); - - redisTimeline = await query.getMany(); - - redisTimeline = redisTimeline.filter(note => { + const timeline = await this.fanoutTimelineEndpointService.timeline({ + untilId, + sinceId, + limit: ps.limit, + allowPartial: ps.allowPartial, + me, + useDbFallback: serverSettings.enableFanoutTimelineDbFallback, + redisTimelines: ps.withFiles ? [`userListTimelineWithFiles:${list.id}`] : [`userListTimeline:${list.id}`], + noteFilter: note => { if (note.userId === me.id) { return true; } @@ -154,30 +152,22 @@ export default class extends Endpoint { // eslint- } return true; - }); + }, + dbFallback: async (untilId, sinceId, limit) => await this.getFromDb(list, { + untilId, + sinceId, + limit, + includeMyRenotes: ps.includeMyRenotes, + includeRenotedMyNotes: ps.includeRenotedMyNotes, + includeLocalRenotes: ps.includeLocalRenotes, + withFiles: ps.withFiles, + withRenotes: ps.withRenotes, + }, me), + }); - redisTimeline.sort((a, b) => a.id > b.id ? -1 : 1); - } + this.activeUsersChart.read(me); - if (redisTimeline.length > 0) { - this.activeUsersChart.read(me); - return await this.noteEntityService.packMany(redisTimeline, me); - } else { - if (serverSettings.enableFanoutTimelineDbFallback) { // fallback to db - return await this.getFromDb(list, { - untilId, - sinceId, - limit: ps.limit, - includeMyRenotes: ps.includeMyRenotes, - includeRenotedMyNotes: ps.includeRenotedMyNotes, - includeLocalRenotes: ps.includeLocalRenotes, - withFiles: ps.withFiles, - withRenotes: ps.withRenotes, - }, me); - } else { - return []; - } - } + return timeline; }); } @@ -271,10 +261,6 @@ export default class extends Endpoint { // eslint- } //#endregion - const timeline = await query.limit(ps.limit).getMany(); - - this.activeUsersChart.read(me); - - return await this.noteEntityService.packMany(timeline, me); + return await query.limit(ps.limit).getMany(); } } diff --git a/packages/backend/src/server/api/endpoints/users/notes.ts b/packages/backend/src/server/api/endpoints/users/notes.ts index 76033ddb06..56983f7bc4 100644 --- a/packages/backend/src/server/api/endpoints/users/notes.ts +++ b/packages/backend/src/server/api/endpoints/users/notes.ts @@ -5,8 +5,7 @@ import { Brackets } from 'typeorm'; import { Inject, Injectable } from '@nestjs/common'; -import * as Redis from 'ioredis'; -import type { MiNote, NotesRepository } from '@/models/_.js'; +import type { NotesRepository } from '@/models/_.js'; import { Endpoint } from '@/server/api/endpoint-base.js'; import { NoteEntityService } from '@/core/entities/NoteEntityService.js'; import { DI } from '@/di-symbols.js'; @@ -14,9 +13,9 @@ import { CacheService } from '@/core/CacheService.js'; import { IdService } from '@/core/IdService.js'; import { isUserRelated } from '@/misc/is-user-related.js'; import { QueryService } from '@/core/QueryService.js'; -import { FanoutTimelineService } from '@/core/FanoutTimelineService.js'; import { MetaService } from '@/core/MetaService.js'; -import { ApiError } from '../../error.js'; +import { MiLocalUser } from '@/models/User.js'; +import { FanoutTimelineEndpointService } from '@/core/FanoutTimelineEndpointService.js'; export const meta = { tags: ['users', 'notes'], @@ -52,6 +51,7 @@ export const paramDef = { untilId: { type: 'string', format: 'misskey:id' }, sinceDate: { type: 'integer' }, untilDate: { type: 'integer' }, + allowPartial: { type: 'boolean', default: false }, // true is recommended but for compatibility false by default withFiles: { type: 'boolean', default: false }, }, required: ['userId'], @@ -60,9 +60,6 @@ export const paramDef = { @Injectable() export default class extends Endpoint { // eslint-disable-line import/no-default-export constructor( - @Inject(DI.redisForTimelines) - private redisForTimelines: Redis.Redis, - @Inject(DI.notesRepository) private notesRepository: NotesRepository, @@ -70,121 +67,130 @@ export default class extends Endpoint { // eslint- private queryService: QueryService, private cacheService: CacheService, private idService: IdService, - private fanoutTimelineService: FanoutTimelineService, + private fanoutTimelineEndpointService: FanoutTimelineEndpointService, private metaService: MetaService, ) { super(meta, paramDef, async (ps, me) => { const untilId = ps.untilId ?? (ps.untilDate ? this.idService.gen(ps.untilDate!) : null); const sinceId = ps.sinceId ?? (ps.sinceDate ? this.idService.gen(ps.sinceDate!) : null); - const isRangeSpecified = untilId != null && sinceId != null; const isSelf = me && (me.id === ps.userId); const serverSettings = await this.metaService.fetch(); - if (serverSettings.enableFanoutTimeline && (isRangeSpecified || sinceId == null)) { - const [ - userIdsWhoMeMuting, - ] = me ? await Promise.all([ - this.cacheService.userMutingsCache.fetch(me.id), - ]) : [new Set()]; + if (!serverSettings.enableFanoutTimeline) { + const timeline = await this.getFromDb({ + untilId, + sinceId, + limit: ps.limit, + userId: ps.userId, + withChannelNotes: ps.withChannelNotes, + withFiles: ps.withFiles, + withRenotes: ps.withRenotes, + }, me); - const [noteIdsRes, repliesNoteIdsRes, channelNoteIdsRes] = await Promise.all([ - this.fanoutTimelineService.get(ps.withFiles ? `userTimelineWithFiles:${ps.userId}` : `userTimeline:${ps.userId}`, untilId, sinceId), - ps.withReplies ? this.fanoutTimelineService.get(`userTimelineWithReplies:${ps.userId}`, untilId, sinceId) : Promise.resolve([]), - ps.withChannelNotes ? this.fanoutTimelineService.get(`userTimelineWithChannel:${ps.userId}`, untilId, sinceId) : Promise.resolve([]), - ]); + return await this.noteEntityService.packMany(timeline, me); + } - let noteIds = Array.from(new Set([ - ...noteIdsRes, - ...repliesNoteIdsRes, - ...channelNoteIdsRes, - ])); - noteIds.sort((a, b) => a > b ? -1 : 1); - noteIds = noteIds.slice(0, ps.limit); + const [ + userIdsWhoMeMuting, + ] = me ? await Promise.all([ + this.cacheService.userMutingsCache.fetch(me.id), + ]) : [new Set()]; - if (noteIds.length > 0) { - const isFollowing = me && Object.hasOwn(await this.cacheService.userFollowingsCache.fetch(me.id), ps.userId); + const redisTimelines = [ps.withFiles ? `userTimelineWithFiles:${ps.userId}` : `userTimeline:${ps.userId}`]; - const query = this.notesRepository.createQueryBuilder('note') - .where('note.id IN (:...noteIds)', { noteIds: noteIds }) - .innerJoinAndSelect('note.user', 'user') - .leftJoinAndSelect('note.reply', 'reply') - .leftJoinAndSelect('note.renote', 'renote') - .leftJoinAndSelect('reply.user', 'replyUser') - .leftJoinAndSelect('renote.user', 'renoteUser') - .leftJoinAndSelect('note.channel', 'channel'); + if (ps.withReplies) redisTimelines.push(`userTimelineWithReplies:${ps.userId}`); + if (ps.withChannelNotes) redisTimelines.push(`userTimelineWithChannel:${ps.userId}`); - let timeline = await query.getMany(); + const isFollowing = me && Object.hasOwn(await this.cacheService.userFollowingsCache.fetch(me.id), ps.userId); - timeline = timeline.filter(note => { - if (me && isUserRelated(note, userIdsWhoMeMuting, true)) return false; + const timeline = await this.fanoutTimelineEndpointService.timeline({ + untilId, + sinceId, + limit: ps.limit, + allowPartial: ps.allowPartial, + me, + redisTimelines, + useDbFallback: true, + noteFilter: note => { + if (me && isUserRelated(note, userIdsWhoMeMuting, true)) return false; - if (note.renoteId) { - if (note.text == null && note.fileIds.length === 0 && !note.hasPoll) { - if (ps.withRenotes === false) return false; - } + if (note.renoteId) { + if (note.text == null && note.fileIds.length === 0 && !note.hasPoll) { + if (ps.withRenotes === false) return false; } - - if (note.channel?.isSensitive && !isSelf) return false; - if (note.visibility === 'specified' && (!me || (me.id !== note.userId && !note.visibleUserIds.some(v => v === me.id)))) return false; - if (note.visibility === 'followers' && !isFollowing && !isSelf) return false; - - return true; - }); - - // TODO: フィルタで件数が減った場合の埋め合わせ処理 - - timeline.sort((a, b) => a.id > b.id ? -1 : 1); - - if (timeline.length > 0) { - return await this.noteEntityService.packMany(timeline, me); } - } - } - //#region fallback to database - const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), ps.sinceId, ps.untilId, ps.sinceDate, ps.untilDate) - .andWhere('note.userId = :userId', { userId: ps.userId }) - .innerJoinAndSelect('note.user', 'user') - .leftJoinAndSelect('note.reply', 'reply') - .leftJoinAndSelect('note.renote', 'renote') - .leftJoinAndSelect('note.channel', 'channel') - .leftJoinAndSelect('reply.user', 'replyUser') - .leftJoinAndSelect('renote.user', 'renoteUser'); + if (note.channel?.isSensitive && !isSelf) return false; + if (note.visibility === 'specified' && (!me || (me.id !== note.userId && !note.visibleUserIds.some(v => v === me.id)))) return false; + if (note.visibility === 'followers' && !isFollowing && !isSelf) return false; - if (ps.withChannelNotes) { - if (!isSelf) query.andWhere(new Brackets(qb => { - qb.orWhere('note.channelId IS NULL'); - qb.orWhere('channel.isSensitive = false'); - })); - } else { - query.andWhere('note.channelId IS NULL'); - } + return true; + }, + dbFallback: async (untilId, sinceId, limit) => await this.getFromDb({ + untilId, + sinceId, + limit, + userId: ps.userId, + withChannelNotes: ps.withChannelNotes, + withFiles: ps.withFiles, + withRenotes: ps.withRenotes, + }, me), + }); - this.queryService.generateVisibilityQuery(query, me); - if (me) { - this.queryService.generateMutedUserQuery(query, me, { id: ps.userId }); - this.queryService.generateBlockedUserQuery(query, me); - } - - if (ps.withFiles) { - query.andWhere('note.fileIds != \'{}\''); - } - - if (ps.withRenotes === false) { - query.andWhere(new Brackets(qb => { - qb.orWhere('note.userId != :userId', { userId: ps.userId }); - qb.orWhere('note.renoteId IS NULL'); - qb.orWhere('note.text IS NOT NULL'); - qb.orWhere('note.fileIds != \'{}\''); - qb.orWhere('0 < (SELECT COUNT(*) FROM poll WHERE poll."noteId" = note.id)'); - })); - } - - const timeline = await query.limit(ps.limit).getMany(); - - return await this.noteEntityService.packMany(timeline, me); - //#endregion + return timeline; }); } + + private async getFromDb(ps: { + untilId: string | null, + sinceId: string | null, + limit: number, + userId: string, + withChannelNotes: boolean, + withFiles: boolean, + withRenotes: boolean, + }, me: MiLocalUser | null) { + const isSelf = me && (me.id === ps.userId); + + const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), ps.sinceId, ps.untilId) + .andWhere('note.userId = :userId', { userId: ps.userId }) + .innerJoinAndSelect('note.user', 'user') + .leftJoinAndSelect('note.reply', 'reply') + .leftJoinAndSelect('note.renote', 'renote') + .leftJoinAndSelect('note.channel', 'channel') + .leftJoinAndSelect('reply.user', 'replyUser') + .leftJoinAndSelect('renote.user', 'renoteUser'); + + if (ps.withChannelNotes) { + if (!isSelf) query.andWhere(new Brackets(qb => { + qb.orWhere('note.channelId IS NULL'); + qb.orWhere('channel.isSensitive = false'); + })); + } else { + query.andWhere('note.channelId IS NULL'); + } + + this.queryService.generateVisibilityQuery(query, me); + if (me) { + this.queryService.generateMutedUserQuery(query, me, { id: ps.userId }); + this.queryService.generateBlockedUserQuery(query, me); + } + + if (ps.withFiles) { + query.andWhere('note.fileIds != \'{}\''); + } + + if (ps.withRenotes === false) { + query.andWhere(new Brackets(qb => { + qb.orWhere('note.userId != :userId', { userId: ps.userId }); + qb.orWhere('note.renoteId IS NULL'); + qb.orWhere('note.text IS NOT NULL'); + qb.orWhere('note.fileIds != \'{}\''); + qb.orWhere('0 < (SELECT COUNT(*) FROM poll WHERE poll."noteId" = note.id)'); + })); + } + + return await query.limit(ps.limit).getMany(); + } } diff --git a/packages/frontend/src/components/MkPagination.vue b/packages/frontend/src/components/MkPagination.vue index 2c59e6d4e8..57348cde53 100644 --- a/packages/frontend/src/components/MkPagination.vue +++ b/packages/frontend/src/components/MkPagination.vue @@ -206,6 +206,7 @@ async function init(): Promise { await os.api(props.pagination.endpoint, { ...params, limit: props.pagination.limit ?? 10, + allowPartial: true, }).then(res => { for (let i = 0; i < res.length; i++) { const item = res[i];