diff --git a/packages/backend/src/queue/processors/CleanRemoteNotesProcessorService.ts b/packages/backend/src/queue/processors/CleanRemoteNotesProcessorService.ts index 6c64d6aa39..da3bb804c2 100644 --- a/packages/backend/src/queue/processors/CleanRemoteNotesProcessorService.ts +++ b/packages/backend/src/queue/processors/CleanRemoteNotesProcessorService.ts @@ -5,9 +5,8 @@ import { setTimeout } from 'node:timers/promises'; import { Inject, Injectable } from '@nestjs/common'; -import { And, Brackets, In, IsNull, LessThan, MoreThan, Not } from 'typeorm'; import { DI } from '@/di-symbols.js'; -import type { MiMeta, MiNote, NoteFavoritesRepository, NotesRepository, UserNotePiningsRepository } from '@/models/_.js'; +import type { MiMeta, MiNote, NotesRepository } from '@/models/_.js'; import type Logger from '@/logger.js'; import { bindThis } from '@/decorators.js'; import { IdService } from '@/core/IdService.js'; @@ -25,12 +24,6 @@ export class CleanRemoteNotesProcessorService { @Inject(DI.notesRepository) private notesRepository: NotesRepository, - @Inject(DI.noteFavoritesRepository) - private noteFavoritesRepository: NoteFavoritesRepository, - - @Inject(DI.userNotePiningsRepository) - private userNotePiningsRepository: UserNotePiningsRepository, - private idService: IdService, private queueLoggerService: QueueLoggerService, ) { @@ -61,6 +54,69 @@ export class CleanRemoteNotesProcessorService { const MAX_NOTE_COUNT_PER_QUERY = 50; + //#retion queries + // We use string literals instead of query builder for several reasons: + // - for removeCondition, we need to use it in having clause, which is not supported by Brackets. + // - for recursive part, we need to preserve the order of columns, but typeorm query builder does not guarantee the order of columns in the result query + + // The condition for removing the notes. + // The note must be: + // - old enough (older than the newestLimit) + // - a remote note (userHost is not null). + // - not have clipped + // - not have pinned on the user profile + // - not has been favorite by any user + const removeCondition = 'note.id < :newestLimit' + + ' AND note."clippedCount" = 0' + + ' AND note."userHost" IS NOT NULL' + // using both userId and noteId instead of just noteId to use index on user_note_pining table. + // This is safe because notes are only pinned by the user who created them. + + ' AND NOT EXISTS(SELECT 1 FROM "user_note_pining" WHERE "noteId" = note."id" AND "userId" = note."userId")' + // We cannot use userId trick because users can favorite notes from other users. + + ' AND NOT EXISTS(SELECT 1 FROM "note_favorite" WHERE "noteId" = note."id")' + ; + + // The initiator query contains the oldest ${MAX_NOTE_COUNT_PER_QUERY} remote non-clipped notes + const initiatorQuery = this.notesRepository.createQueryBuilder('note') + .select('note.id', 'id') + .where(removeCondition) + .andWhere('note.id > :cursor') + .orderBy('note.id', 'ASC') + .limit(MAX_NOTE_COUNT_PER_QUERY); + + // The union query queries the related notes and replies related to the initiator query + const unionQuery = ` + SELECT "note"."id", "note"."replyId", "note"."renoteId", rn."initiatorId" + FROM "note" "note" + INNER JOIN "related_notes" "rn" + ON "note"."replyId" = rn.id + OR "note"."renoteId" = rn.id + OR "note"."id" = rn."replyId" + OR "note"."id" = rn."renoteId" + `; + + const selectRelatedNotesFromInitiatorIdsQuery = ` + SELECT "note"."id" AS "id", "note"."replyId" AS "replyId", "note"."renoteId" AS "renoteId", "note"."id" AS "initiatorId" + FROM "note" "note" WHERE "note"."id" IN (:...initiatorIds) + `; + + const recursiveQuery = `(${selectRelatedNotesFromInitiatorIdsQuery}) UNION (${unionQuery})`; + + const removableInitiatorNotesQuery = this.notesRepository.createQueryBuilder('note') + .select('rn."initiatorId"') + .innerJoin('related_notes', 'rn', 'note.id = rn.id') + .groupBy('rn."initiatorId"') + .having(`bool_and(${removeCondition})`); + + const notesQuery = this.notesRepository.createQueryBuilder('note') + .addCommonTableExpression(recursiveQuery, 'related_notes', { recursive: true }) + .select('note.id', 'id') + .addSelect('rn."initiatorId"') + .innerJoin('related_notes', 'rn', 'note.id = rn.id') + .where(`rn."initiatorId" IN (${removableInitiatorNotesQuery.getQuery()})`) + .distinctOn(['note.id']); + //#endregion + const stats = { deletedCount: 0, oldest: null as number | null, @@ -74,77 +130,45 @@ export class CleanRemoteNotesProcessorService { let cursor = '0'; // oldest note ID to start from while (true) { + //#region check time const batchBeginAt = Date.now(); - // We use string literals instead of query builder for several reasons: - // - for removeCondition, we need to use it in having clause, which is not supported by Brackets. - // - for recursive part, we need to preserve the order of columns, but typeorm query builder does not guarantee the order of columns in the result query + const elapsed = batchBeginAt - startAt; - // The condition for removing the notes. - // The note must be: - // - old enough (older than the newestLimit) - // - a remote note (userHost is not null). - // - not have clipped - // - not have pinned on the user profile - // - not has been favorite by any user - const removeCondition = 'note.id < :newestLimit' - + ' AND note."clippedCount" = 0' - + ' AND note."userHost" IS NOT NULL' - // using both userId and noteId instead of just noteId to use index on user_note_pining table. - // This is safe because notes are only pinned by the user who created them. - + ' AND NOT EXISTS(SELECT 1 FROM "user_note_pining" WHERE "noteId" = note."id" AND "userId" = note."userId")' - // We cannot use userId trick because users can favorite notes from other users. - + ' AND NOT EXISTS(SELECT 1 FROM "note_favorite" WHERE "noteId" = note."id")' - ; + if (elapsed >= maxDuration) { + this.logger.info(`Reached maximum duration of ${maxDuration}ms, stopping...`); + job.log('Reached maximum duration, stopping cleaning.'); + job.updateProgress(100); + break; + } - // The initiator query contains the oldest ${MAX_NOTE_COUNT_PER_QUERY} remote non-clipped notes - const initiatorQuery = ` - SELECT "note"."id" AS "id", "note"."replyId" AS "replyId", "note"."renoteId" AS "renoteId", "note"."id" AS "initiatorId" - FROM "note" "note" WHERE ${removeCondition} AND "note"."id" > :cursor ORDER BY "note"."id" ASC LIMIT ${MAX_NOTE_COUNT_PER_QUERY}`; + job.updateProgress((elapsed / maxDuration) * 100); + //#endregion - // The union query queries the related notes and replies related to the initiator query - const unionQuery = ` - SELECT "note"."id", "note"."replyId", "note"."renoteId", rn."initiatorId" - FROM "note" "note" - INNER JOIN "related_notes" "rn" - ON "note"."replyId" = rn.id - OR "note"."renoteId" = rn.id - OR "note"."id" = rn."replyId" - OR "note"."id" = rn."renoteId" - `; - const recursiveQuery = `(${initiatorQuery}) UNION (${unionQuery})`; - - const removableInitiatorNotesQuery = this.notesRepository.createQueryBuilder('note') - .select('rn."initiatorId"') - .innerJoin('related_notes', 'rn', 'note.id = rn.id') - .groupBy('rn."initiatorId"') - .having(`bool_and(${removeCondition})`); - - const notesQuery = this.notesRepository.createQueryBuilder('note') - .addCommonTableExpression(recursiveQuery, 'related_notes', { recursive: true }) - .select('note.id', 'id') - .addSelect('rn."initiatorId"') - .innerJoin('related_notes', 'rn', 'note.id = rn.id') - .where(`rn."initiatorId" IN (${ removableInitiatorNotesQuery.getQuery() })`) - .setParameters({ cursor, newestLimit }); - - const notes: { id: MiNote['id'], initiatorId: MiNote['id'] }[] = await notesQuery.getRawMany(); - - const fetchedCount = notes.length; + // First, we fetch the initiator notes that are older than the newestLimit. + const initiatorNotes: { id: MiNote['id'] }[] = await initiatorQuery.setParameters({ cursor, newestLimit }).getRawMany(); // update the cursor to the newest initiatorId found in the fetched notes. - // We don't use 'id' since the note can be newer than the initiator note. - for (const note of notes) { - if (cursor < note.initiatorId) { - cursor = note.initiatorId; - } + const newCursor = initiatorNotes.reduce((max, note) => note.id > max ? note.id : max, cursor); + + if (initiatorNotes.length === 0 || cursor === newCursor || newCursor >= newestLimit) { + // If no notes were found or the cursor did not change, we can stop. + job.log('No more notes to clean. (no initiator notes found or cursor did not change.)'); + break; } + const notes: { id: MiNote['id'], initiatorId: MiNote['id'] }[] = await notesQuery.setParameters({ + initiatorIds: initiatorNotes.map(note => note.id), + newestLimit, + }).getRawMany(); + + cursor = newCursor; + if (notes.length > 0) { await this.notesRepository.delete(notes.map(note => note.id)); - for (const note of notes) { - const t = this.idService.parse(note.id).date.getTime(); + for (const { id } of notes) { + const t = this.idService.parse(id).date.getTime(); if (stats.oldest === null || t < stats.oldest) { stats.oldest = t; } @@ -156,19 +180,14 @@ export class CleanRemoteNotesProcessorService { stats.deletedCount += notes.length; } - job.log(`Deleted ${notes.length} of ${fetchedCount}; ${Date.now() - batchBeginAt}ms`); + job.log(`Deleted ${notes.length} from ${initiatorNotes.length} initiators; ${Date.now() - batchBeginAt}ms`); - const elapsed = Date.now() - startAt; - - if (elapsed >= maxDuration) { - this.logger.info(`Reached maximum duration of ${maxDuration}ms, stopping...`); - job.log('Reached maximum duration, stopping cleaning.'); - job.updateProgress(100); + if (initiatorNotes.length < MAX_NOTE_COUNT_PER_QUERY) { + // If we fetched less than the maximum, it means there are no more notes to process. + job.log(`No more notes to clean. (fewer than MAX_NOTE_COUNT_PER_QUERY =${MAX_NOTE_COUNT_PER_QUERY}.)`); break; } - job.updateProgress((elapsed / maxDuration) * 100); - await setTimeout(1000 * 5); // Wait a moment to avoid overwhelming the db } diff --git a/packages/backend/test/unit/queue/processors/CleanRemoteNotesProcessorService.ts b/packages/backend/test/unit/queue/processors/CleanRemoteNotesProcessorService.ts new file mode 100644 index 0000000000..15f8eda865 --- /dev/null +++ b/packages/backend/test/unit/queue/processors/CleanRemoteNotesProcessorService.ts @@ -0,0 +1,631 @@ +/* + * SPDX-FileCopyrightText: syuilo and misskey-project + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { jest } from '@jest/globals'; +import { Test, TestingModule } from '@nestjs/testing'; +import ms from 'ms'; +import { + type MiNote, + type MiUser, + type NotesRepository, + type NoteFavoritesRepository, + type UserNotePiningsRepository, + type UsersRepository, + type UserProfilesRepository, + MiMeta, +} from '@/models/_.js'; +import { CleanRemoteNotesProcessorService } from '@/queue/processors/CleanRemoteNotesProcessorService.js'; +import { DI } from '@/di-symbols.js'; +import { IdService } from '@/core/IdService.js'; +import { QueueLoggerService } from '@/queue/QueueLoggerService.js'; +import { GlobalModule } from '@/GlobalModule.js'; +import { secureRndstr } from '@/misc/secure-rndstr.js'; + +describe('CleanRemoteNotesProcessorService', () => { + let app: TestingModule; + let service: CleanRemoteNotesProcessorService; + let idService: IdService; + let notesRepository: NotesRepository; + let noteFavoritesRepository: NoteFavoritesRepository; + let userNotePiningsRepository: UserNotePiningsRepository; + let usersRepository: UsersRepository; + let userProfilesRepository: UserProfilesRepository; + + // Local user + let alice: MiUser; + // Remote user 1 + let bob: MiUser; + // Remote user 2 + let carol: MiUser; + + const meta = new MiMeta(); + + // Mock job object + const createMockJob = () => ({ + log: jest.fn(), + updateProgress: jest.fn(), + }); + + async function createUser(data: Partial = {}) { + const id = idService.gen(); + const un = data.username || secureRndstr(16); + const user = await usersRepository + .insert({ + id, + username: un, + usernameLower: un.toLowerCase(), + ...data, + }) + .then(x => usersRepository.findOneByOrFail(x.identifiers[0])); + + await userProfilesRepository.save({ + userId: id, + }); + + return user; + } + + async function createNote(data: Partial, user: MiUser, time?: number): Promise { + const id = idService.gen(time); + const note = await notesRepository + .insert({ + id: id, + text: `note_${id}`, + userId: user.id, + userHost: user.host, + visibility: 'public', + ...data, + }) + .then(x => notesRepository.findOneByOrFail(x.identifiers[0])); + return note; + } + + beforeAll(async () => { + app = await Test + .createTestingModule({ + imports: [ + GlobalModule, + ], + providers: [ + CleanRemoteNotesProcessorService, + IdService, + { + provide: QueueLoggerService, + useFactory: () => ({ + logger: { + createSubLogger: () => ({ + info: jest.fn(), + warn: jest.fn(), + succ: jest.fn(), + }), + }, + }), + }, + ], + }) + .overrideProvider(DI.meta).useFactory({ factory: () => meta }) + .compile(); + + service = app.get(CleanRemoteNotesProcessorService); + idService = app.get(IdService); + notesRepository = app.get(DI.notesRepository); + noteFavoritesRepository = app.get(DI.noteFavoritesRepository); + userNotePiningsRepository = app.get(DI.userNotePiningsRepository); + usersRepository = app.get(DI.usersRepository); + userProfilesRepository = app.get(DI.userProfilesRepository); + + alice = await createUser({ username: 'alice', host: null }); + bob = await createUser({ username: 'bob', host: 'remote1.example.com' }); + carol = await createUser({ username: 'carol', host: 'remote2.example.com' }); + + app.enableShutdownHooks(); + }); + + beforeEach(() => { + // Reset mocks + jest.clearAllMocks(); + + // Set default meta values + meta.enableRemoteNotesCleaning = true; + meta.remoteNotesCleaningMaxProcessingDurationInMinutes = 0.3; + meta.remoteNotesCleaningExpiryDaysForEachNotes = 30; + }, 60 * 1000); + + afterEach(async () => { + // Clean up test data + await Promise.all([ + notesRepository.createQueryBuilder().delete().execute(), + userNotePiningsRepository.createQueryBuilder().delete().execute(), + noteFavoritesRepository.createQueryBuilder().delete().execute(), + ]); + }, 60 * 1000); + + afterAll(async () => { + await app.close(); + }); + + describe('basic', () => { + test('should skip cleaning when enableRemoteNotesCleaning is false', async () => { + meta.enableRemoteNotesCleaning = false; + const job = createMockJob(); + + const result = await service.process(job as any); + + expect(result).toEqual({ + deletedCount: 0, + oldest: null, + newest: null, + skipped: true, + }); + }); + + test('should return success result when enableRemoteNotesCleaning is true and no notes to clean', async () => { + const job = createMockJob(); + + await createNote({}, alice); + const result = await service.process(job as any); + + expect(result).toEqual({ + deletedCount: 0, + oldest: null, + newest: null, + skipped: false, + }); + }, 3000); + + test('should clean remote notes and return stats', async () => { + // Remote notes + const remoteNotes = await Promise.all([ + createNote({}, bob), + createNote({}, carol), + createNote({}, bob, Date.now() - ms(`${meta.remoteNotesCleaningExpiryDaysForEachNotes} days`) - 1000), + createNote({}, carol, Date.now() - ms(`${meta.remoteNotesCleaningExpiryDaysForEachNotes} days`) - 2000), // Note older than expiry + ]); + + // Local notes + const localNotes = await Promise.all([ + createNote({}, alice), + createNote({}, alice, Date.now() - ms(`${meta.remoteNotesCleaningExpiryDaysForEachNotes} days`) - 1000), + ]); + + const job = createMockJob(); + + const result = await service.process(job as any); + + expect(result).toEqual({ + deletedCount: 2, + oldest: expect.any(Number), + newest: expect.any(Number), + skipped: false, + }); + + // Check side-by-side from all notes + const remainingNotes = await notesRepository.find(); + expect(remainingNotes.length).toBe(4); + expect(remainingNotes.some(n => n.id === remoteNotes[0].id)).toBe(true); + expect(remainingNotes.some(n => n.id === remoteNotes[1].id)).toBe(true); + expect(remainingNotes.some(n => n.id === remoteNotes[2].id)).toBe(false); + expect(remainingNotes.some(n => n.id === remoteNotes[3].id)).toBe(false); + expect(remainingNotes.some(n => n.id === localNotes[0].id)).toBe(true); + expect(remainingNotes.some(n => n.id === localNotes[1].id)).toBe(true); + }); + }); + + describe('advanced', () => { + // お気に入り + test('should not delete note that is favorited by any user', async () => { + const job = createMockJob(); + + // Create old remote note that should be deleted + const olderRemoteNote = await createNote({}, bob, Date.now() - ms(`${meta.remoteNotesCleaningExpiryDaysForEachNotes} days`) - 1000); + + // Favorite the note + await noteFavoritesRepository.save({ + id: idService.gen(), + userId: alice.id, + noteId: olderRemoteNote.id, + }); + + const result = await service.process(job as any); + + expect(result.deletedCount).toBe(0); + expect(result.skipped).toBe(false); + + const remainingNote = await notesRepository.findOneBy({ id: olderRemoteNote.id }); + expect(remainingNote).not.toBeNull(); + }); + + // ピン留め + test('should not delete note that is pinned by the user', async () => { + const job = createMockJob(); + + // Create old remote note that should be deleted + const olderRemoteNote = await createNote({}, bob, Date.now() - ms(`${meta.remoteNotesCleaningExpiryDaysForEachNotes} days`) - 1000); + + // Pin the note by the user who created it + await userNotePiningsRepository.save({ + id: idService.gen(), + userId: bob.id, // Same user as the note creator + noteId: olderRemoteNote.id, + }); + + const result = await service.process(job as any); + + expect(result.deletedCount).toBe(0); + expect(result.skipped).toBe(false); + + const remainingNote = await notesRepository.findOneBy({ id: olderRemoteNote.id }); + expect(remainingNote).not.toBeNull(); + }); + + // クリップ + test('should not delete note that is clipped', async () => { + const job = createMockJob(); + + // Create old remote note that is clipped + const clippedNote = await createNote({ + clippedCount: 1, // Clipped + }, bob, Date.now() - ms(`${meta.remoteNotesCleaningExpiryDaysForEachNotes} days`) - 1000); + + const result = await service.process(job as any); + + expect(result.deletedCount).toBe(0); + expect(result.skipped).toBe(false); + + const remainingNote = await notesRepository.findOneBy({ id: clippedNote.id }); + expect(remainingNote).not.toBeNull(); + }); + + // 古いreply, renoteが含まれている時の挙動 + test('should handle reply/renote relationships correctly', async () => { + const job = createMockJob(); + + // Create old remote notes with reply/renote relationships + const originalNote = await createNote({}, bob, Date.now() - ms(`${meta.remoteNotesCleaningExpiryDaysForEachNotes} days`) - 1000); + const replyNote = await createNote({ + replyId: originalNote.id, + }, carol, Date.now() - ms(`${meta.remoteNotesCleaningExpiryDaysForEachNotes} days`) - 2000); + const renoteNote = await createNote({ + renoteId: originalNote.id, + }, bob, Date.now() - ms(`${meta.remoteNotesCleaningExpiryDaysForEachNotes} days`) - 3000); + + const result = await service.process(job as any); + + // Should delete all three notes as they are all old and remote + expect(result.deletedCount).toBe(3); + expect(result.skipped).toBe(false); + + const remainingNotes = await notesRepository.find(); + expect(remainingNotes.some(n => n.id === originalNote.id)).toBe(false); + expect(remainingNotes.some(n => n.id === replyNote.id)).toBe(false); + expect(remainingNotes.some(n => n.id === renoteNote.id)).toBe(false); + }); + + // 古いリモートノートに新しいリプライがある時、どちらも削除されない + test('should not delete both old remote note with new reply', async () => { + const job = createMockJob(); + + // Create old remote note that should be deleted + const oldNote = await createNote({}, bob, Date.now() - ms(`${meta.remoteNotesCleaningExpiryDaysForEachNotes} days`) - 1000); + + // Create a reply note that is newer than the expiry period + const recentReplyNote = await createNote({ + replyId: oldNote.id, + }, carol, Date.now() - ms(`${meta.remoteNotesCleaningExpiryDaysForEachNotes} days`) + 1000); + + const result = await service.process(job as any); + + expect(result.deletedCount).toBe(0); // Only the old note should be deleted + expect(result.skipped).toBe(false); + + const remainingNotes = await notesRepository.find(); + expect(remainingNotes.some(n => n.id === oldNote.id)).toBe(true); + expect(remainingNotes.some(n => n.id === recentReplyNote.id)).toBe(true); // Recent reply note should remain + }); + + // 古いリモートノートに新しいリプライと古いリプライがある時、全て残る + test('should not delete old remote note with new reply and old reply', async () => { + const job = createMockJob(); + + // Create old remote note that should be deleted + const oldNote = await createNote({}, bob, Date.now() - ms(`${meta.remoteNotesCleaningExpiryDaysForEachNotes} days`) - 1000); + + // Create a reply note that is newer than the expiry period + const recentReplyNote = await createNote({ + replyId: oldNote.id, + }, carol, Date.now() - ms(`${meta.remoteNotesCleaningExpiryDaysForEachNotes} days`) + 1000); + + // Create an old reply note that should be deleted + const oldReplyNote = await createNote({ + replyId: oldNote.id, + }, carol, Date.now() - ms(`${meta.remoteNotesCleaningExpiryDaysForEachNotes} days`) - 2000); + + const result = await service.process(job as any); + + expect(result.deletedCount).toBe(0); + expect(result.skipped).toBe(false); + + const remainingNotes = await notesRepository.find(); + expect(remainingNotes.some(n => n.id === oldNote.id)).toBe(true); + expect(remainingNotes.some(n => n.id === recentReplyNote.id)).toBe(true); // Recent reply note should remain + expect(remainingNotes.some(n => n.id === oldReplyNote.id)).toBe(true); // Old reply note should be deleted + }); + + // リプライがお気に入りされているとき、どちらも削除されない + test('should not delete reply note that is favorited', async () => { + const job = createMockJob(); + + // Create old remote note that should be deleted + const olderRemoteNote = await createNote({}, bob, Date.now() - ms(`${meta.remoteNotesCleaningExpiryDaysForEachNotes} days`) - 1000); + + // Create a reply note that is newer than the expiry period + const replyNote = await createNote({ + replyId: olderRemoteNote.id, + }, carol, Date.now() - ms(`${meta.remoteNotesCleaningExpiryDaysForEachNotes} days`) - 2000); + + // Favorite the reply note + await noteFavoritesRepository.save({ + id: idService.gen(), + userId: alice.id, + noteId: replyNote.id, + }); + + const result = await service.process(job as any); + + expect(result.deletedCount).toBe(0); // Only the old note should be deleted + expect(result.skipped).toBe(false); + + const remainingNotes = await notesRepository.find(); + expect(remainingNotes.some(n => n.id === olderRemoteNote.id)).toBe(true); + expect(remainingNotes.some(n => n.id === replyNote.id)).toBe(true); // Recent reply note should remain + }); + + // リプライがピン留めされているとき、どちらも削除されない + test('should not delete reply note that is pinned', async () => { + const job = createMockJob(); + + // Create old remote note that should be deleted + const olderRemoteNote = await createNote({}, bob, Date.now() - ms(`${meta.remoteNotesCleaningExpiryDaysForEachNotes} days`) - 1000); + + // Create a reply note that is newer than the expiry period + const replyNote = await createNote({ + replyId: olderRemoteNote.id, + }, carol, Date.now() - ms(`${meta.remoteNotesCleaningExpiryDaysForEachNotes} days`) - 2000); + + // Pin the reply note + await userNotePiningsRepository.save({ + id: idService.gen(), + userId: carol.id, + noteId: replyNote.id, + }); + + const result = await service.process(job as any); + + expect(result.deletedCount).toBe(0); // Only the old note should be deleted + expect(result.skipped).toBe(false); + + const remainingNotes = await notesRepository.find(); + expect(remainingNotes.some(n => n.id === olderRemoteNote.id)).toBe(true); + expect(remainingNotes.some(n => n.id === replyNote.id)).toBe(true); // Reply note should remain + }); + + // リプライがクリップされているとき、どちらも削除されない + test('should not delete reply note that is clipped', async () => { + const job = createMockJob(); + + // Create old remote note that should be deleted + const olderRemoteNote = await createNote({}, bob, Date.now() - ms(`${meta.remoteNotesCleaningExpiryDaysForEachNotes} days`) - 1000); + + // Create a reply note that is old but clipped + const replyNote = await createNote({ + replyId: olderRemoteNote.id, + clippedCount: 1, // Clipped + }, carol, Date.now() - ms(`${meta.remoteNotesCleaningExpiryDaysForEachNotes} days`) - 2000); + + const result = await service.process(job as any); + + expect(result.deletedCount).toBe(0); // Both notes should be kept because reply is clipped + expect(result.skipped).toBe(false); + + const remainingNotes = await notesRepository.find(); + expect(remainingNotes.some(n => n.id === olderRemoteNote.id)).toBe(true); + expect(remainingNotes.some(n => n.id === replyNote.id)).toBe(true); + }); + + test('should handle mixed scenarios with multiple conditions', async () => { + const job = createMockJob(); + + // Create various types of notes + const oldTime = Date.now() - ms(`${meta.remoteNotesCleaningExpiryDaysForEachNotes} days`) - 1000; + + // Should be deleted: old remote note with no special conditions + const deletableNote = await createNote({}, bob, oldTime); + + // Should NOT be deleted: old remote note but favorited + const favoritedNote = await createNote({}, carol, oldTime); + await noteFavoritesRepository.save({ + id: idService.gen(), + userId: alice.id, + noteId: favoritedNote.id, + }); + + // Should NOT be deleted: old remote note but pinned + const pinnedNote = await createNote({}, bob, oldTime); + await userNotePiningsRepository.save({ + id: idService.gen(), + userId: bob.id, + noteId: pinnedNote.id, + }); + + // Should NOT be deleted: old remote note but clipped + const clippedNote = await createNote({ + clippedCount: 2, + }, carol, oldTime); + + // Should NOT be deleted: old local note + const localNote = await createNote({}, alice, oldTime); + + // Should NOT be deleted: new remote note + const newerRemoteNote = await createNote({}, bob); + + const result = await service.process(job as any); + + expect(result.deletedCount).toBe(1); // Only deletableNote should be deleted + expect(result.skipped).toBe(false); + + const remainingNotes = await notesRepository.find(); + expect(remainingNotes.length).toBe(5); + expect(remainingNotes.some(n => n.id === deletableNote.id)).toBe(false); // Deleted + expect(remainingNotes.some(n => n.id === favoritedNote.id)).toBe(true); // Kept + expect(remainingNotes.some(n => n.id === pinnedNote.id)).toBe(true); // Kept + expect(remainingNotes.some(n => n.id === clippedNote.id)).toBe(true); // Kept + expect(remainingNotes.some(n => n.id === localNote.id)).toBe(true); // Kept + expect(remainingNotes.some(n => n.id === newerRemoteNote.id)).toBe(true); // Kept + }); + + // 大量のノート + test('should handle large number of notes correctly', async () => { + const AMOUNT = 130; + const job = createMockJob(); + + const oldTime = Date.now() - ms(`${meta.remoteNotesCleaningExpiryDaysForEachNotes} days`) - 1000; + const noteIds = []; + for (let i = 0; i < AMOUNT; i++) { + const note = await createNote({}, bob, oldTime - i); + noteIds.push(note.id); + } + + const result = await service.process(job as any); + + // Should delete all notes, but may require multiple batches + expect(result.deletedCount).toBe(AMOUNT); + expect(result.skipped).toBe(false); + + const remainingNotes = await notesRepository.find(); + expect(remainingNotes.length).toBe(0); + }); + + // 大量のノート + リプライ or リノート + test('should handle large number of notes with replies correctly', async () => { + const AMOUNT = 130; + const job = createMockJob(); + + const oldTime = Date.now() - ms(`${meta.remoteNotesCleaningExpiryDaysForEachNotes} days`) - 1000; + const noteIds = []; + for (let i = 0; i < AMOUNT; i++) { + const note = await createNote({}, bob, oldTime - i - AMOUNT); + noteIds.push(note.id); + if (i % 2 === 0) { + // Create a reply for every second note + await createNote({ replyId: note.id }, carol, oldTime - i); + } else { + // Create a renote for every second note + await createNote({ renoteId: note.id }, bob, oldTime - i); + } + } + + const result = await service.process(job as any); + // Should delete all notes, but may require multiple batches + expect(result.deletedCount).toBe(AMOUNT * 2); + expect(result.skipped).toBe(false); + }); + + // 大量の古いノート + 新しいリプライ or リノート + test('should handle large number of old notes with new replies correctly', async () => { + const AMOUNT = 130; + const job = createMockJob(); + + const oldTime = Date.now() - ms(`${meta.remoteNotesCleaningExpiryDaysForEachNotes} days`) - 1000; + const newTime = Date.now(); + const noteIds = []; + for (let i = 0; i < AMOUNT; i++) { + const note = await createNote({}, bob, oldTime - i); + noteIds.push(note.id); + if (i % 2 === 0) { + // Create a reply for every second note + await createNote({ replyId: note.id }, carol, newTime + i); + } else { + // Create a renote for every second note + await createNote({ renoteId: note.id }, bob, newTime + i); + } + } + const result = await service.process(job as any); + + expect(result.deletedCount).toBe(0); + expect(result.skipped).toBe(false); + }); + + // 大量の残す対象(clippedCount: 1)と大量の削除対象 + test('should handle large number of notes, mixed conditions with clippedCount', async () => { + const AMOUNT_BASE = 70; + const job = createMockJob(); + + const oldTime = Date.now() - ms(`${meta.remoteNotesCleaningExpiryDaysForEachNotes} days`) - 1000; + const noteIds = []; + for (let i = 0; i < AMOUNT_BASE; i++) { + const note = await createNote({ clippedCount: 1 }, bob, oldTime - i - AMOUNT_BASE); + noteIds.push(note.id); + } + for (let i = 0; i < AMOUNT_BASE; i++) { + const note = await createNote({}, carol, oldTime - i); + noteIds.push(note.id); + } + + const result = await service.process(job as any); + + expect(result.deletedCount).toBe(AMOUNT_BASE); // Assuming half are deletable + expect(result.skipped).toBe(false); + }); + + // 大量の残す対象(リプライ)と大量の削除対象 + test('should handle large number of notes, mixed conditions with replies', async () => { + const AMOUNT_BASE = 70; + const job = createMockJob(); + const oldTime = Date.now() - ms(`${meta.remoteNotesCleaningExpiryDaysForEachNotes} days`) - 1000; + const newTime = Date.now(); + for (let i = 0; i < AMOUNT_BASE; i++) { + // should remain + const note = await createNote({}, carol, oldTime - AMOUNT_BASE - i); + // should remain + await createNote({ replyId: note.id }, bob, newTime + i); + } + + const noteIdsExpectedToBeDeleted = []; + for (let i = 0; i < AMOUNT_BASE; i++) { + // should be deleted + const note = await createNote({}, bob, oldTime - i); + noteIdsExpectedToBeDeleted.push(note.id); + } + + const result = await service.process(job as any); + expect(result.deletedCount).toBe(AMOUNT_BASE); // Assuming all replies are deletable + expect(result.skipped).toBe(false); + + const remainingNotes = await notesRepository.find(); + expect(remainingNotes.length).toBe(AMOUNT_BASE * 2); // Only replies should remain + noteIdsExpectedToBeDeleted.forEach(id => { + expect(remainingNotes.some(n => n.id === id)).toBe(false); // All original notes should be deleted + }); + }); + + test('should update cursor correctly during batch processing', async () => { + const job = createMockJob(); + + // Create notes with specific timing to test cursor behavior + const baseTime = Date.now() - ms(`${meta.remoteNotesCleaningExpiryDaysForEachNotes} days`) - 10000; + + const note1 = await createNote({}, bob, baseTime); + const note2 = await createNote({}, carol, baseTime - 1000); + const note3 = await createNote({}, bob, baseTime - 2000); + + const result = await service.process(job as any); + + expect(result.deletedCount).toBe(3); + expect(result.newest).toBe(idService.parse(note1.id).date.getTime()); + expect(result.oldest).toBe(idService.parse(note3.id).date.getTime()); + expect(result.skipped).toBe(false); + }); + }); +});