fix(backend): Fix and create unit test of CleanRemoteNotesProcessorService (#16368)
* wip
* test(backend): CleanRemoteNotesProcessorService (basic)
* test(backend): CleanRemoteNotesProcessorService (advanced)
* ✌️
* a
* split initiator query
* no order by
* ???
* old → older
This commit is contained in:
tamaina
GitHub
parent
076a83466e
commit
85e3e49688
|
|
@ -5,9 +5,8 @@
|
|||
|
||||
import { setTimeout } from 'node:timers/promises';
|
||||
import { Inject, Injectable } from '@nestjs/common';
|
||||
import { And, Brackets, In, IsNull, LessThan, MoreThan, Not } from 'typeorm';
|
||||
import { DI } from '@/di-symbols.js';
|
||||
import type { MiMeta, MiNote, NoteFavoritesRepository, NotesRepository, UserNotePiningsRepository } from '@/models/_.js';
|
||||
import type { MiMeta, MiNote, NotesRepository } from '@/models/_.js';
|
||||
import type Logger from '@/logger.js';
|
||||
import { bindThis } from '@/decorators.js';
|
||||
import { IdService } from '@/core/IdService.js';
|
||||
|
|
@ -25,12 +24,6 @@ export class CleanRemoteNotesProcessorService {
|
|||
@Inject(DI.notesRepository)
|
||||
private notesRepository: NotesRepository,
|
||||
|
||||
@Inject(DI.noteFavoritesRepository)
|
||||
private noteFavoritesRepository: NoteFavoritesRepository,
|
||||
|
||||
@Inject(DI.userNotePiningsRepository)
|
||||
private userNotePiningsRepository: UserNotePiningsRepository,
|
||||
|
||||
private idService: IdService,
|
||||
private queueLoggerService: QueueLoggerService,
|
||||
) {
|
||||
|
|
@ -61,6 +54,69 @@ export class CleanRemoteNotesProcessorService {
|
|||
|
||||
const MAX_NOTE_COUNT_PER_QUERY = 50;
|
||||
|
||||
//#retion queries
|
||||
// We use string literals instead of query builder for several reasons:
|
||||
// - for removeCondition, we need to use it in having clause, which is not supported by Brackets.
|
||||
// - for recursive part, we need to preserve the order of columns, but typeorm query builder does not guarantee the order of columns in the result query
|
||||
|
||||
// The condition for removing the notes.
|
||||
// The note must be:
|
||||
// - old enough (older than the newestLimit)
|
||||
// - a remote note (userHost is not null).
|
||||
// - not have clipped
|
||||
// - not have pinned on the user profile
|
||||
// - not has been favorite by any user
|
||||
const removeCondition = 'note.id < :newestLimit'
|
||||
+ ' AND note."clippedCount" = 0'
|
||||
+ ' AND note."userHost" IS NOT NULL'
|
||||
// using both userId and noteId instead of just noteId to use index on user_note_pining table.
|
||||
// This is safe because notes are only pinned by the user who created them.
|
||||
+ ' AND NOT EXISTS(SELECT 1 FROM "user_note_pining" WHERE "noteId" = note."id" AND "userId" = note."userId")'
|
||||
// We cannot use userId trick because users can favorite notes from other users.
|
||||
+ ' AND NOT EXISTS(SELECT 1 FROM "note_favorite" WHERE "noteId" = note."id")'
|
||||
;
|
||||
|
||||
// The initiator query contains the oldest ${MAX_NOTE_COUNT_PER_QUERY} remote non-clipped notes
|
||||
const initiatorQuery = this.notesRepository.createQueryBuilder('note')
|
||||
.select('note.id', 'id')
|
||||
.where(removeCondition)
|
||||
.andWhere('note.id > :cursor')
|
||||
.orderBy('note.id', 'ASC')
|
||||
.limit(MAX_NOTE_COUNT_PER_QUERY);
|
||||
|
||||
// The union query queries the related notes and replies related to the initiator query
|
||||
const unionQuery = `
|
||||
SELECT "note"."id", "note"."replyId", "note"."renoteId", rn."initiatorId"
|
||||
FROM "note" "note"
|
||||
INNER JOIN "related_notes" "rn"
|
||||
ON "note"."replyId" = rn.id
|
||||
OR "note"."renoteId" = rn.id
|
||||
OR "note"."id" = rn."replyId"
|
||||
OR "note"."id" = rn."renoteId"
|
||||
`;
|
||||
|
||||
const selectRelatedNotesFromInitiatorIdsQuery = `
|
||||
SELECT "note"."id" AS "id", "note"."replyId" AS "replyId", "note"."renoteId" AS "renoteId", "note"."id" AS "initiatorId"
|
||||
FROM "note" "note" WHERE "note"."id" IN (:...initiatorIds)
|
||||
`;
|
||||
|
||||
const recursiveQuery = `(${selectRelatedNotesFromInitiatorIdsQuery}) UNION (${unionQuery})`;
|
||||
|
||||
const removableInitiatorNotesQuery = this.notesRepository.createQueryBuilder('note')
|
||||
.select('rn."initiatorId"')
|
||||
.innerJoin('related_notes', 'rn', 'note.id = rn.id')
|
||||
.groupBy('rn."initiatorId"')
|
||||
.having(`bool_and(${removeCondition})`);
|
||||
|
||||
const notesQuery = this.notesRepository.createQueryBuilder('note')
|
||||
.addCommonTableExpression(recursiveQuery, 'related_notes', { recursive: true })
|
||||
.select('note.id', 'id')
|
||||
.addSelect('rn."initiatorId"')
|
||||
.innerJoin('related_notes', 'rn', 'note.id = rn.id')
|
||||
.where(`rn."initiatorId" IN (${removableInitiatorNotesQuery.getQuery()})`)
|
||||
.distinctOn(['note.id']);
|
||||
//#endregion
|
||||
|
||||
const stats = {
|
||||
deletedCount: 0,
|
||||
oldest: null as number | null,
|
||||
|
|
@ -74,77 +130,45 @@ export class CleanRemoteNotesProcessorService {
|
|||
let cursor = '0'; // oldest note ID to start from
|
||||
|
||||
while (true) {
|
||||
//#region check time
|
||||
const batchBeginAt = Date.now();
|
||||
|
||||
// We use string literals instead of query builder for several reasons:
|
||||
// - for removeCondition, we need to use it in having clause, which is not supported by Brackets.
|
||||
// - for recursive part, we need to preserve the order of columns, but typeorm query builder does not guarantee the order of columns in the result query
|
||||
const elapsed = batchBeginAt - startAt;
|
||||
|
||||
// The condition for removing the notes.
|
||||
// The note must be:
|
||||
// - old enough (older than the newestLimit)
|
||||
// - a remote note (userHost is not null).
|
||||
// - not have clipped
|
||||
// - not have pinned on the user profile
|
||||
// - not has been favorite by any user
|
||||
const removeCondition = 'note.id < :newestLimit'
|
||||
+ ' AND note."clippedCount" = 0'
|
||||
+ ' AND note."userHost" IS NOT NULL'
|
||||
// using both userId and noteId instead of just noteId to use index on user_note_pining table.
|
||||
// This is safe because notes are only pinned by the user who created them.
|
||||
+ ' AND NOT EXISTS(SELECT 1 FROM "user_note_pining" WHERE "noteId" = note."id" AND "userId" = note."userId")'
|
||||
// We cannot use userId trick because users can favorite notes from other users.
|
||||
+ ' AND NOT EXISTS(SELECT 1 FROM "note_favorite" WHERE "noteId" = note."id")'
|
||||
;
|
||||
if (elapsed >= maxDuration) {
|
||||
this.logger.info(`Reached maximum duration of ${maxDuration}ms, stopping...`);
|
||||
job.log('Reached maximum duration, stopping cleaning.');
|
||||
job.updateProgress(100);
|
||||
break;
|
||||
}
|
||||
|
||||
// The initiator query contains the oldest ${MAX_NOTE_COUNT_PER_QUERY} remote non-clipped notes
|
||||
const initiatorQuery = `
|
||||
SELECT "note"."id" AS "id", "note"."replyId" AS "replyId", "note"."renoteId" AS "renoteId", "note"."id" AS "initiatorId"
|
||||
FROM "note" "note" WHERE ${removeCondition} AND "note"."id" > :cursor ORDER BY "note"."id" ASC LIMIT ${MAX_NOTE_COUNT_PER_QUERY}`;
|
||||
job.updateProgress((elapsed / maxDuration) * 100);
|
||||
//#endregion
|
||||
|
||||
// The union query queries the related notes and replies related to the initiator query
|
||||
const unionQuery = `
|
||||
SELECT "note"."id", "note"."replyId", "note"."renoteId", rn."initiatorId"
|
||||
FROM "note" "note"
|
||||
INNER JOIN "related_notes" "rn"
|
||||
ON "note"."replyId" = rn.id
|
||||
OR "note"."renoteId" = rn.id
|
||||
OR "note"."id" = rn."replyId"
|
||||
OR "note"."id" = rn."renoteId"
|
||||
`;
|
||||
const recursiveQuery = `(${initiatorQuery}) UNION (${unionQuery})`;
|
||||
|
||||
const removableInitiatorNotesQuery = this.notesRepository.createQueryBuilder('note')
|
||||
.select('rn."initiatorId"')
|
||||
.innerJoin('related_notes', 'rn', 'note.id = rn.id')
|
||||
.groupBy('rn."initiatorId"')
|
||||
.having(`bool_and(${removeCondition})`);
|
||||
|
||||
const notesQuery = this.notesRepository.createQueryBuilder('note')
|
||||
.addCommonTableExpression(recursiveQuery, 'related_notes', { recursive: true })
|
||||
.select('note.id', 'id')
|
||||
.addSelect('rn."initiatorId"')
|
||||
.innerJoin('related_notes', 'rn', 'note.id = rn.id')
|
||||
.where(`rn."initiatorId" IN (${ removableInitiatorNotesQuery.getQuery() })`)
|
||||
.setParameters({ cursor, newestLimit });
|
||||
|
||||
const notes: { id: MiNote['id'], initiatorId: MiNote['id'] }[] = await notesQuery.getRawMany();
|
||||
|
||||
const fetchedCount = notes.length;
|
||||
// First, we fetch the initiator notes that are older than the newestLimit.
|
||||
const initiatorNotes: { id: MiNote['id'] }[] = await initiatorQuery.setParameters({ cursor, newestLimit }).getRawMany();
|
||||
|
||||
// update the cursor to the newest initiatorId found in the fetched notes.
|
||||
// We don't use 'id' since the note can be newer than the initiator note.
|
||||
for (const note of notes) {
|
||||
if (cursor < note.initiatorId) {
|
||||
cursor = note.initiatorId;
|
||||
}
|
||||
const newCursor = initiatorNotes.reduce((max, note) => note.id > max ? note.id : max, cursor);
|
||||
|
||||
if (initiatorNotes.length === 0 || cursor === newCursor || newCursor >= newestLimit) {
|
||||
// If no notes were found or the cursor did not change, we can stop.
|
||||
job.log('No more notes to clean. (no initiator notes found or cursor did not change.)');
|
||||
break;
|
||||
}
|
||||
|
||||
const notes: { id: MiNote['id'], initiatorId: MiNote['id'] }[] = await notesQuery.setParameters({
|
||||
initiatorIds: initiatorNotes.map(note => note.id),
|
||||
newestLimit,
|
||||
}).getRawMany();
|
||||
|
||||
cursor = newCursor;
|
||||
|
||||
if (notes.length > 0) {
|
||||
await this.notesRepository.delete(notes.map(note => note.id));
|
||||
|
||||
for (const note of notes) {
|
||||
const t = this.idService.parse(note.id).date.getTime();
|
||||
for (const { id } of notes) {
|
||||
const t = this.idService.parse(id).date.getTime();
|
||||
if (stats.oldest === null || t < stats.oldest) {
|
||||
stats.oldest = t;
|
||||
}
|
||||
|
|
@ -156,19 +180,14 @@ export class CleanRemoteNotesProcessorService {
|
|||
stats.deletedCount += notes.length;
|
||||
}
|
||||
|
||||
job.log(`Deleted ${notes.length} of ${fetchedCount}; ${Date.now() - batchBeginAt}ms`);
|
||||
job.log(`Deleted ${notes.length} from ${initiatorNotes.length} initiators; ${Date.now() - batchBeginAt}ms`);
|
||||
|
||||
const elapsed = Date.now() - startAt;
|
||||
|
||||
if (elapsed >= maxDuration) {
|
||||
this.logger.info(`Reached maximum duration of ${maxDuration}ms, stopping...`);
|
||||
job.log('Reached maximum duration, stopping cleaning.');
|
||||
job.updateProgress(100);
|
||||
if (initiatorNotes.length < MAX_NOTE_COUNT_PER_QUERY) {
|
||||
// If we fetched less than the maximum, it means there are no more notes to process.
|
||||
job.log(`No more notes to clean. (fewer than MAX_NOTE_COUNT_PER_QUERY =${MAX_NOTE_COUNT_PER_QUERY}.)`);
|
||||
break;
|
||||
}
|
||||
|
||||
job.updateProgress((elapsed / maxDuration) * 100);
|
||||
|
||||
await setTimeout(1000 * 5); // Wait a moment to avoid overwhelming the db
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,631 @@
|
|||
/*
|
||||
* SPDX-FileCopyrightText: syuilo and misskey-project
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
import { jest } from '@jest/globals';
|
||||
import { Test, TestingModule } from '@nestjs/testing';
|
||||
import ms from 'ms';
|
||||
import {
|
||||
type MiNote,
|
||||
type MiUser,
|
||||
type NotesRepository,
|
||||
type NoteFavoritesRepository,
|
||||
type UserNotePiningsRepository,
|
||||
type UsersRepository,
|
||||
type UserProfilesRepository,
|
||||
MiMeta,
|
||||
} from '@/models/_.js';
|
||||
import { CleanRemoteNotesProcessorService } from '@/queue/processors/CleanRemoteNotesProcessorService.js';
|
||||
import { DI } from '@/di-symbols.js';
|
||||
import { IdService } from '@/core/IdService.js';
|
||||
import { QueueLoggerService } from '@/queue/QueueLoggerService.js';
|
||||
import { GlobalModule } from '@/GlobalModule.js';
|
||||
import { secureRndstr } from '@/misc/secure-rndstr.js';
|
||||
|
||||
describe('CleanRemoteNotesProcessorService', () => {
|
||||
let app: TestingModule;
|
||||
let service: CleanRemoteNotesProcessorService;
|
||||
let idService: IdService;
|
||||
let notesRepository: NotesRepository;
|
||||
let noteFavoritesRepository: NoteFavoritesRepository;
|
||||
let userNotePiningsRepository: UserNotePiningsRepository;
|
||||
let usersRepository: UsersRepository;
|
||||
let userProfilesRepository: UserProfilesRepository;
|
||||
|
||||
// Local user
|
||||
let alice: MiUser;
|
||||
// Remote user 1
|
||||
let bob: MiUser;
|
||||
// Remote user 2
|
||||
let carol: MiUser;
|
||||
|
||||
const meta = new MiMeta();
|
||||
|
||||
// Mock job object
|
||||
const createMockJob = () => ({
|
||||
log: jest.fn(),
|
||||
updateProgress: jest.fn(),
|
||||
});
|
||||
|
||||
async function createUser(data: Partial<MiUser> = {}) {
|
||||
const id = idService.gen();
|
||||
const un = data.username || secureRndstr(16);
|
||||
const user = await usersRepository
|
||||
.insert({
|
||||
id,
|
||||
username: un,
|
||||
usernameLower: un.toLowerCase(),
|
||||
...data,
|
||||
})
|
||||
.then(x => usersRepository.findOneByOrFail(x.identifiers[0]));
|
||||
|
||||
await userProfilesRepository.save({
|
||||
userId: id,
|
||||
});
|
||||
|
||||
return user;
|
||||
}
|
||||
|
||||
async function createNote(data: Partial<MiNote>, user: MiUser, time?: number): Promise<MiNote> {
|
||||
const id = idService.gen(time);
|
||||
const note = await notesRepository
|
||||
.insert({
|
||||
id: id,
|
||||
text: `note_${id}`,
|
||||
userId: user.id,
|
||||
userHost: user.host,
|
||||
visibility: 'public',
|
||||
...data,
|
||||
})
|
||||
.then(x => notesRepository.findOneByOrFail(x.identifiers[0]));
|
||||
return note;
|
||||
}
|
||||
|
||||
beforeAll(async () => {
|
||||
app = await Test
|
||||
.createTestingModule({
|
||||
imports: [
|
||||
GlobalModule,
|
||||
],
|
||||
providers: [
|
||||
CleanRemoteNotesProcessorService,
|
||||
IdService,
|
||||
{
|
||||
provide: QueueLoggerService,
|
||||
useFactory: () => ({
|
||||
logger: {
|
||||
createSubLogger: () => ({
|
||||
info: jest.fn(),
|
||||
warn: jest.fn(),
|
||||
succ: jest.fn(),
|
||||
}),
|
||||
},
|
||||
}),
|
||||
},
|
||||
],
|
||||
})
|
||||
.overrideProvider(DI.meta).useFactory({ factory: () => meta })
|
||||
.compile();
|
||||
|
||||
service = app.get(CleanRemoteNotesProcessorService);
|
||||
idService = app.get(IdService);
|
||||
notesRepository = app.get(DI.notesRepository);
|
||||
noteFavoritesRepository = app.get(DI.noteFavoritesRepository);
|
||||
userNotePiningsRepository = app.get(DI.userNotePiningsRepository);
|
||||
usersRepository = app.get(DI.usersRepository);
|
||||
userProfilesRepository = app.get(DI.userProfilesRepository);
|
||||
|
||||
alice = await createUser({ username: 'alice', host: null });
|
||||
bob = await createUser({ username: 'bob', host: 'remote1.example.com' });
|
||||
carol = await createUser({ username: 'carol', host: 'remote2.example.com' });
|
||||
|
||||
app.enableShutdownHooks();
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
// Reset mocks
|
||||
jest.clearAllMocks();
|
||||
|
||||
// Set default meta values
|
||||
meta.enableRemoteNotesCleaning = true;
|
||||
meta.remoteNotesCleaningMaxProcessingDurationInMinutes = 0.3;
|
||||
meta.remoteNotesCleaningExpiryDaysForEachNotes = 30;
|
||||
}, 60 * 1000);
|
||||
|
||||
afterEach(async () => {
|
||||
// Clean up test data
|
||||
await Promise.all([
|
||||
notesRepository.createQueryBuilder().delete().execute(),
|
||||
userNotePiningsRepository.createQueryBuilder().delete().execute(),
|
||||
noteFavoritesRepository.createQueryBuilder().delete().execute(),
|
||||
]);
|
||||
}, 60 * 1000);
|
||||
|
||||
afterAll(async () => {
|
||||
await app.close();
|
||||
});
|
||||
|
||||
describe('basic', () => {
|
||||
test('should skip cleaning when enableRemoteNotesCleaning is false', async () => {
|
||||
meta.enableRemoteNotesCleaning = false;
|
||||
const job = createMockJob();
|
||||
|
||||
const result = await service.process(job as any);
|
||||
|
||||
expect(result).toEqual({
|
||||
deletedCount: 0,
|
||||
oldest: null,
|
||||
newest: null,
|
||||
skipped: true,
|
||||
});
|
||||
});
|
||||
|
||||
test('should return success result when enableRemoteNotesCleaning is true and no notes to clean', async () => {
|
||||
const job = createMockJob();
|
||||
|
||||
await createNote({}, alice);
|
||||
const result = await service.process(job as any);
|
||||
|
||||
expect(result).toEqual({
|
||||
deletedCount: 0,
|
||||
oldest: null,
|
||||
newest: null,
|
||||
skipped: false,
|
||||
});
|
||||
}, 3000);
|
||||
|
||||
test('should clean remote notes and return stats', async () => {
|
||||
// Remote notes
|
||||
const remoteNotes = await Promise.all([
|
||||
createNote({}, bob),
|
||||
createNote({}, carol),
|
||||
createNote({}, bob, Date.now() - ms(`${meta.remoteNotesCleaningExpiryDaysForEachNotes} days`) - 1000),
|
||||
createNote({}, carol, Date.now() - ms(`${meta.remoteNotesCleaningExpiryDaysForEachNotes} days`) - 2000), // Note older than expiry
|
||||
]);
|
||||
|
||||
// Local notes
|
||||
const localNotes = await Promise.all([
|
||||
createNote({}, alice),
|
||||
createNote({}, alice, Date.now() - ms(`${meta.remoteNotesCleaningExpiryDaysForEachNotes} days`) - 1000),
|
||||
]);
|
||||
|
||||
const job = createMockJob();
|
||||
|
||||
const result = await service.process(job as any);
|
||||
|
||||
expect(result).toEqual({
|
||||
deletedCount: 2,
|
||||
oldest: expect.any(Number),
|
||||
newest: expect.any(Number),
|
||||
skipped: false,
|
||||
});
|
||||
|
||||
// Check side-by-side from all notes
|
||||
const remainingNotes = await notesRepository.find();
|
||||
expect(remainingNotes.length).toBe(4);
|
||||
expect(remainingNotes.some(n => n.id === remoteNotes[0].id)).toBe(true);
|
||||
expect(remainingNotes.some(n => n.id === remoteNotes[1].id)).toBe(true);
|
||||
expect(remainingNotes.some(n => n.id === remoteNotes[2].id)).toBe(false);
|
||||
expect(remainingNotes.some(n => n.id === remoteNotes[3].id)).toBe(false);
|
||||
expect(remainingNotes.some(n => n.id === localNotes[0].id)).toBe(true);
|
||||
expect(remainingNotes.some(n => n.id === localNotes[1].id)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('advanced', () => {
|
||||
// お気に入り
|
||||
test('should not delete note that is favorited by any user', async () => {
|
||||
const job = createMockJob();
|
||||
|
||||
// Create old remote note that should be deleted
|
||||
const olderRemoteNote = await createNote({}, bob, Date.now() - ms(`${meta.remoteNotesCleaningExpiryDaysForEachNotes} days`) - 1000);
|
||||
|
||||
// Favorite the note
|
||||
await noteFavoritesRepository.save({
|
||||
id: idService.gen(),
|
||||
userId: alice.id,
|
||||
noteId: olderRemoteNote.id,
|
||||
});
|
||||
|
||||
const result = await service.process(job as any);
|
||||
|
||||
expect(result.deletedCount).toBe(0);
|
||||
expect(result.skipped).toBe(false);
|
||||
|
||||
const remainingNote = await notesRepository.findOneBy({ id: olderRemoteNote.id });
|
||||
expect(remainingNote).not.toBeNull();
|
||||
});
|
||||
|
||||
// ピン留め
|
||||
test('should not delete note that is pinned by the user', async () => {
|
||||
const job = createMockJob();
|
||||
|
||||
// Create old remote note that should be deleted
|
||||
const olderRemoteNote = await createNote({}, bob, Date.now() - ms(`${meta.remoteNotesCleaningExpiryDaysForEachNotes} days`) - 1000);
|
||||
|
||||
// Pin the note by the user who created it
|
||||
await userNotePiningsRepository.save({
|
||||
id: idService.gen(),
|
||||
userId: bob.id, // Same user as the note creator
|
||||
noteId: olderRemoteNote.id,
|
||||
});
|
||||
|
||||
const result = await service.process(job as any);
|
||||
|
||||
expect(result.deletedCount).toBe(0);
|
||||
expect(result.skipped).toBe(false);
|
||||
|
||||
const remainingNote = await notesRepository.findOneBy({ id: olderRemoteNote.id });
|
||||
expect(remainingNote).not.toBeNull();
|
||||
});
|
||||
|
||||
// クリップ
|
||||
test('should not delete note that is clipped', async () => {
|
||||
const job = createMockJob();
|
||||
|
||||
// Create old remote note that is clipped
|
||||
const clippedNote = await createNote({
|
||||
clippedCount: 1, // Clipped
|
||||
}, bob, Date.now() - ms(`${meta.remoteNotesCleaningExpiryDaysForEachNotes} days`) - 1000);
|
||||
|
||||
const result = await service.process(job as any);
|
||||
|
||||
expect(result.deletedCount).toBe(0);
|
||||
expect(result.skipped).toBe(false);
|
||||
|
||||
const remainingNote = await notesRepository.findOneBy({ id: clippedNote.id });
|
||||
expect(remainingNote).not.toBeNull();
|
||||
});
|
||||
|
||||
// 古いreply, renoteが含まれている時の挙動
|
||||
test('should handle reply/renote relationships correctly', async () => {
|
||||
const job = createMockJob();
|
||||
|
||||
// Create old remote notes with reply/renote relationships
|
||||
const originalNote = await createNote({}, bob, Date.now() - ms(`${meta.remoteNotesCleaningExpiryDaysForEachNotes} days`) - 1000);
|
||||
const replyNote = await createNote({
|
||||
replyId: originalNote.id,
|
||||
}, carol, Date.now() - ms(`${meta.remoteNotesCleaningExpiryDaysForEachNotes} days`) - 2000);
|
||||
const renoteNote = await createNote({
|
||||
renoteId: originalNote.id,
|
||||
}, bob, Date.now() - ms(`${meta.remoteNotesCleaningExpiryDaysForEachNotes} days`) - 3000);
|
||||
|
||||
const result = await service.process(job as any);
|
||||
|
||||
// Should delete all three notes as they are all old and remote
|
||||
expect(result.deletedCount).toBe(3);
|
||||
expect(result.skipped).toBe(false);
|
||||
|
||||
const remainingNotes = await notesRepository.find();
|
||||
expect(remainingNotes.some(n => n.id === originalNote.id)).toBe(false);
|
||||
expect(remainingNotes.some(n => n.id === replyNote.id)).toBe(false);
|
||||
expect(remainingNotes.some(n => n.id === renoteNote.id)).toBe(false);
|
||||
});
|
||||
|
||||
// 古いリモートノートに新しいリプライがある時、どちらも削除されない
|
||||
test('should not delete both old remote note with new reply', async () => {
|
||||
const job = createMockJob();
|
||||
|
||||
// Create old remote note that should be deleted
|
||||
const oldNote = await createNote({}, bob, Date.now() - ms(`${meta.remoteNotesCleaningExpiryDaysForEachNotes} days`) - 1000);
|
||||
|
||||
// Create a reply note that is newer than the expiry period
|
||||
const recentReplyNote = await createNote({
|
||||
replyId: oldNote.id,
|
||||
}, carol, Date.now() - ms(`${meta.remoteNotesCleaningExpiryDaysForEachNotes} days`) + 1000);
|
||||
|
||||
const result = await service.process(job as any);
|
||||
|
||||
expect(result.deletedCount).toBe(0); // Only the old note should be deleted
|
||||
expect(result.skipped).toBe(false);
|
||||
|
||||
const remainingNotes = await notesRepository.find();
|
||||
expect(remainingNotes.some(n => n.id === oldNote.id)).toBe(true);
|
||||
expect(remainingNotes.some(n => n.id === recentReplyNote.id)).toBe(true); // Recent reply note should remain
|
||||
});
|
||||
|
||||
// 古いリモートノートに新しいリプライと古いリプライがある時、全て残る
|
||||
test('should not delete old remote note with new reply and old reply', async () => {
|
||||
const job = createMockJob();
|
||||
|
||||
// Create old remote note that should be deleted
|
||||
const oldNote = await createNote({}, bob, Date.now() - ms(`${meta.remoteNotesCleaningExpiryDaysForEachNotes} days`) - 1000);
|
||||
|
||||
// Create a reply note that is newer than the expiry period
|
||||
const recentReplyNote = await createNote({
|
||||
replyId: oldNote.id,
|
||||
}, carol, Date.now() - ms(`${meta.remoteNotesCleaningExpiryDaysForEachNotes} days`) + 1000);
|
||||
|
||||
// Create an old reply note that should be deleted
|
||||
const oldReplyNote = await createNote({
|
||||
replyId: oldNote.id,
|
||||
}, carol, Date.now() - ms(`${meta.remoteNotesCleaningExpiryDaysForEachNotes} days`) - 2000);
|
||||
|
||||
const result = await service.process(job as any);
|
||||
|
||||
expect(result.deletedCount).toBe(0);
|
||||
expect(result.skipped).toBe(false);
|
||||
|
||||
const remainingNotes = await notesRepository.find();
|
||||
expect(remainingNotes.some(n => n.id === oldNote.id)).toBe(true);
|
||||
expect(remainingNotes.some(n => n.id === recentReplyNote.id)).toBe(true); // Recent reply note should remain
|
||||
expect(remainingNotes.some(n => n.id === oldReplyNote.id)).toBe(true); // Old reply note should be deleted
|
||||
});
|
||||
|
||||
// リプライがお気に入りされているとき、どちらも削除されない
|
||||
test('should not delete reply note that is favorited', async () => {
|
||||
const job = createMockJob();
|
||||
|
||||
// Create old remote note that should be deleted
|
||||
const olderRemoteNote = await createNote({}, bob, Date.now() - ms(`${meta.remoteNotesCleaningExpiryDaysForEachNotes} days`) - 1000);
|
||||
|
||||
// Create a reply note that is newer than the expiry period
|
||||
const replyNote = await createNote({
|
||||
replyId: olderRemoteNote.id,
|
||||
}, carol, Date.now() - ms(`${meta.remoteNotesCleaningExpiryDaysForEachNotes} days`) - 2000);
|
||||
|
||||
// Favorite the reply note
|
||||
await noteFavoritesRepository.save({
|
||||
id: idService.gen(),
|
||||
userId: alice.id,
|
||||
noteId: replyNote.id,
|
||||
});
|
||||
|
||||
const result = await service.process(job as any);
|
||||
|
||||
expect(result.deletedCount).toBe(0); // Only the old note should be deleted
|
||||
expect(result.skipped).toBe(false);
|
||||
|
||||
const remainingNotes = await notesRepository.find();
|
||||
expect(remainingNotes.some(n => n.id === olderRemoteNote.id)).toBe(true);
|
||||
expect(remainingNotes.some(n => n.id === replyNote.id)).toBe(true); // Recent reply note should remain
|
||||
});
|
||||
|
||||
// リプライがピン留めされているとき、どちらも削除されない
|
||||
test('should not delete reply note that is pinned', async () => {
|
||||
const job = createMockJob();
|
||||
|
||||
// Create old remote note that should be deleted
|
||||
const olderRemoteNote = await createNote({}, bob, Date.now() - ms(`${meta.remoteNotesCleaningExpiryDaysForEachNotes} days`) - 1000);
|
||||
|
||||
// Create a reply note that is newer than the expiry period
|
||||
const replyNote = await createNote({
|
||||
replyId: olderRemoteNote.id,
|
||||
}, carol, Date.now() - ms(`${meta.remoteNotesCleaningExpiryDaysForEachNotes} days`) - 2000);
|
||||
|
||||
// Pin the reply note
|
||||
await userNotePiningsRepository.save({
|
||||
id: idService.gen(),
|
||||
userId: carol.id,
|
||||
noteId: replyNote.id,
|
||||
});
|
||||
|
||||
const result = await service.process(job as any);
|
||||
|
||||
expect(result.deletedCount).toBe(0); // Only the old note should be deleted
|
||||
expect(result.skipped).toBe(false);
|
||||
|
||||
const remainingNotes = await notesRepository.find();
|
||||
expect(remainingNotes.some(n => n.id === olderRemoteNote.id)).toBe(true);
|
||||
expect(remainingNotes.some(n => n.id === replyNote.id)).toBe(true); // Reply note should remain
|
||||
});
|
||||
|
||||
// リプライがクリップされているとき、どちらも削除されない
|
||||
test('should not delete reply note that is clipped', async () => {
|
||||
const job = createMockJob();
|
||||
|
||||
// Create old remote note that should be deleted
|
||||
const olderRemoteNote = await createNote({}, bob, Date.now() - ms(`${meta.remoteNotesCleaningExpiryDaysForEachNotes} days`) - 1000);
|
||||
|
||||
// Create a reply note that is old but clipped
|
||||
const replyNote = await createNote({
|
||||
replyId: olderRemoteNote.id,
|
||||
clippedCount: 1, // Clipped
|
||||
}, carol, Date.now() - ms(`${meta.remoteNotesCleaningExpiryDaysForEachNotes} days`) - 2000);
|
||||
|
||||
const result = await service.process(job as any);
|
||||
|
||||
expect(result.deletedCount).toBe(0); // Both notes should be kept because reply is clipped
|
||||
expect(result.skipped).toBe(false);
|
||||
|
||||
const remainingNotes = await notesRepository.find();
|
||||
expect(remainingNotes.some(n => n.id === olderRemoteNote.id)).toBe(true);
|
||||
expect(remainingNotes.some(n => n.id === replyNote.id)).toBe(true);
|
||||
});
|
||||
|
||||
test('should handle mixed scenarios with multiple conditions', async () => {
|
||||
const job = createMockJob();
|
||||
|
||||
// Create various types of notes
|
||||
const oldTime = Date.now() - ms(`${meta.remoteNotesCleaningExpiryDaysForEachNotes} days`) - 1000;
|
||||
|
||||
// Should be deleted: old remote note with no special conditions
|
||||
const deletableNote = await createNote({}, bob, oldTime);
|
||||
|
||||
// Should NOT be deleted: old remote note but favorited
|
||||
const favoritedNote = await createNote({}, carol, oldTime);
|
||||
await noteFavoritesRepository.save({
|
||||
id: idService.gen(),
|
||||
userId: alice.id,
|
||||
noteId: favoritedNote.id,
|
||||
});
|
||||
|
||||
// Should NOT be deleted: old remote note but pinned
|
||||
const pinnedNote = await createNote({}, bob, oldTime);
|
||||
await userNotePiningsRepository.save({
|
||||
id: idService.gen(),
|
||||
userId: bob.id,
|
||||
noteId: pinnedNote.id,
|
||||
});
|
||||
|
||||
// Should NOT be deleted: old remote note but clipped
|
||||
const clippedNote = await createNote({
|
||||
clippedCount: 2,
|
||||
}, carol, oldTime);
|
||||
|
||||
// Should NOT be deleted: old local note
|
||||
const localNote = await createNote({}, alice, oldTime);
|
||||
|
||||
// Should NOT be deleted: new remote note
|
||||
const newerRemoteNote = await createNote({}, bob);
|
||||
|
||||
const result = await service.process(job as any);
|
||||
|
||||
expect(result.deletedCount).toBe(1); // Only deletableNote should be deleted
|
||||
expect(result.skipped).toBe(false);
|
||||
|
||||
const remainingNotes = await notesRepository.find();
|
||||
expect(remainingNotes.length).toBe(5);
|
||||
expect(remainingNotes.some(n => n.id === deletableNote.id)).toBe(false); // Deleted
|
||||
expect(remainingNotes.some(n => n.id === favoritedNote.id)).toBe(true); // Kept
|
||||
expect(remainingNotes.some(n => n.id === pinnedNote.id)).toBe(true); // Kept
|
||||
expect(remainingNotes.some(n => n.id === clippedNote.id)).toBe(true); // Kept
|
||||
expect(remainingNotes.some(n => n.id === localNote.id)).toBe(true); // Kept
|
||||
expect(remainingNotes.some(n => n.id === newerRemoteNote.id)).toBe(true); // Kept
|
||||
});
|
||||
|
||||
// 大量のノート
|
||||
test('should handle large number of notes correctly', async () => {
|
||||
const AMOUNT = 130;
|
||||
const job = createMockJob();
|
||||
|
||||
const oldTime = Date.now() - ms(`${meta.remoteNotesCleaningExpiryDaysForEachNotes} days`) - 1000;
|
||||
const noteIds = [];
|
||||
for (let i = 0; i < AMOUNT; i++) {
|
||||
const note = await createNote({}, bob, oldTime - i);
|
||||
noteIds.push(note.id);
|
||||
}
|
||||
|
||||
const result = await service.process(job as any);
|
||||
|
||||
// Should delete all notes, but may require multiple batches
|
||||
expect(result.deletedCount).toBe(AMOUNT);
|
||||
expect(result.skipped).toBe(false);
|
||||
|
||||
const remainingNotes = await notesRepository.find();
|
||||
expect(remainingNotes.length).toBe(0);
|
||||
});
|
||||
|
||||
// 大量のノート + リプライ or リノート
|
||||
test('should handle large number of notes with replies correctly', async () => {
|
||||
const AMOUNT = 130;
|
||||
const job = createMockJob();
|
||||
|
||||
const oldTime = Date.now() - ms(`${meta.remoteNotesCleaningExpiryDaysForEachNotes} days`) - 1000;
|
||||
const noteIds = [];
|
||||
for (let i = 0; i < AMOUNT; i++) {
|
||||
const note = await createNote({}, bob, oldTime - i - AMOUNT);
|
||||
noteIds.push(note.id);
|
||||
if (i % 2 === 0) {
|
||||
// Create a reply for every second note
|
||||
await createNote({ replyId: note.id }, carol, oldTime - i);
|
||||
} else {
|
||||
// Create a renote for every second note
|
||||
await createNote({ renoteId: note.id }, bob, oldTime - i);
|
||||
}
|
||||
}
|
||||
|
||||
const result = await service.process(job as any);
|
||||
// Should delete all notes, but may require multiple batches
|
||||
expect(result.deletedCount).toBe(AMOUNT * 2);
|
||||
expect(result.skipped).toBe(false);
|
||||
});
|
||||
|
||||
// 大量の古いノート + 新しいリプライ or リノート
|
||||
test('should handle large number of old notes with new replies correctly', async () => {
|
||||
const AMOUNT = 130;
|
||||
const job = createMockJob();
|
||||
|
||||
const oldTime = Date.now() - ms(`${meta.remoteNotesCleaningExpiryDaysForEachNotes} days`) - 1000;
|
||||
const newTime = Date.now();
|
||||
const noteIds = [];
|
||||
for (let i = 0; i < AMOUNT; i++) {
|
||||
const note = await createNote({}, bob, oldTime - i);
|
||||
noteIds.push(note.id);
|
||||
if (i % 2 === 0) {
|
||||
// Create a reply for every second note
|
||||
await createNote({ replyId: note.id }, carol, newTime + i);
|
||||
} else {
|
||||
// Create a renote for every second note
|
||||
await createNote({ renoteId: note.id }, bob, newTime + i);
|
||||
}
|
||||
}
|
||||
const result = await service.process(job as any);
|
||||
|
||||
expect(result.deletedCount).toBe(0);
|
||||
expect(result.skipped).toBe(false);
|
||||
});
|
||||
|
||||
// 大量の残す対象(clippedCount: 1)と大量の削除対象
|
||||
test('should handle large number of notes, mixed conditions with clippedCount', async () => {
|
||||
const AMOUNT_BASE = 70;
|
||||
const job = createMockJob();
|
||||
|
||||
const oldTime = Date.now() - ms(`${meta.remoteNotesCleaningExpiryDaysForEachNotes} days`) - 1000;
|
||||
const noteIds = [];
|
||||
for (let i = 0; i < AMOUNT_BASE; i++) {
|
||||
const note = await createNote({ clippedCount: 1 }, bob, oldTime - i - AMOUNT_BASE);
|
||||
noteIds.push(note.id);
|
||||
}
|
||||
for (let i = 0; i < AMOUNT_BASE; i++) {
|
||||
const note = await createNote({}, carol, oldTime - i);
|
||||
noteIds.push(note.id);
|
||||
}
|
||||
|
||||
const result = await service.process(job as any);
|
||||
|
||||
expect(result.deletedCount).toBe(AMOUNT_BASE); // Assuming half are deletable
|
||||
expect(result.skipped).toBe(false);
|
||||
});
|
||||
|
||||
// 大量の残す対象(リプライ)と大量の削除対象
|
||||
test('should handle large number of notes, mixed conditions with replies', async () => {
|
||||
const AMOUNT_BASE = 70;
|
||||
const job = createMockJob();
|
||||
const oldTime = Date.now() - ms(`${meta.remoteNotesCleaningExpiryDaysForEachNotes} days`) - 1000;
|
||||
const newTime = Date.now();
|
||||
for (let i = 0; i < AMOUNT_BASE; i++) {
|
||||
// should remain
|
||||
const note = await createNote({}, carol, oldTime - AMOUNT_BASE - i);
|
||||
// should remain
|
||||
await createNote({ replyId: note.id }, bob, newTime + i);
|
||||
}
|
||||
|
||||
const noteIdsExpectedToBeDeleted = [];
|
||||
for (let i = 0; i < AMOUNT_BASE; i++) {
|
||||
// should be deleted
|
||||
const note = await createNote({}, bob, oldTime - i);
|
||||
noteIdsExpectedToBeDeleted.push(note.id);
|
||||
}
|
||||
|
||||
const result = await service.process(job as any);
|
||||
expect(result.deletedCount).toBe(AMOUNT_BASE); // Assuming all replies are deletable
|
||||
expect(result.skipped).toBe(false);
|
||||
|
||||
const remainingNotes = await notesRepository.find();
|
||||
expect(remainingNotes.length).toBe(AMOUNT_BASE * 2); // Only replies should remain
|
||||
noteIdsExpectedToBeDeleted.forEach(id => {
|
||||
expect(remainingNotes.some(n => n.id === id)).toBe(false); // All original notes should be deleted
|
||||
});
|
||||
});
|
||||
|
||||
test('should update cursor correctly during batch processing', async () => {
|
||||
const job = createMockJob();
|
||||
|
||||
// Create notes with specific timing to test cursor behavior
|
||||
const baseTime = Date.now() - ms(`${meta.remoteNotesCleaningExpiryDaysForEachNotes} days`) - 10000;
|
||||
|
||||
const note1 = await createNote({}, bob, baseTime);
|
||||
const note2 = await createNote({}, carol, baseTime - 1000);
|
||||
const note3 = await createNote({}, bob, baseTime - 2000);
|
||||
|
||||
const result = await service.process(job as any);
|
||||
|
||||
expect(result.deletedCount).toBe(3);
|
||||
expect(result.newest).toBe(idService.parse(note1.id).date.getTime());
|
||||
expect(result.oldest).toBe(idService.parse(note3.id).date.getTime());
|
||||
expect(result.skipped).toBe(false);
|
||||
});
|
||||
});
|
||||
});
|
||||
Loading…
Reference in New Issue