usbharu
/
misskey
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

enhance: performance for CleanRemoteNotesProcessorService (#16404) 

* enhance: performance for CleanRemoteNotesProcessorService

Signed-off-by: eternal-flame-AD <yume@yumechi.jp>

* suggestions

Signed-off-by: eternal-flame-AD <yume@yumechi.jp>

* docs

Signed-off-by: eternal-flame-AD <yume@yumechi.jp>

* change initial limit to 100

Signed-off-by: eternal-flame-AD <yume@yumechi.jp>

* robustness for transient race conditions

Signed-off-by: eternal-flame-AD <yume@yumechi.jp>

* handle cursors in postgres

Signed-off-by: eternal-flame-AD <yume@yumechi.jp>

* robustness: transient errors and timeout handling

Signed-off-by: eternal-flame-AD <yume@yumechi.jp>

* use '0' as initial cursor

Signed-off-by: eternal-flame-AD <yume@yumechi.jp>

---------

Signed-off-by: eternal-flame-AD <yume@yumechi.jp>
This commit is contained in:
饺子w (Yumechi) 2025-08-14 07:54:28 +00:00 committed by GitHub
parent c25a922928
commit 90b9609341
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 181 additions and 94 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

272
packages/backend/src/queue/processors/CleanRemoteNotesProcessorService.ts
View File

@ -5,6 +5,7 @@
import { setTimeout } from 'node:timers/promises'; import { setTimeout } from 'node:timers/promises';
import { Inject, Injectable } from '@nestjs/common'; import { Inject, Injectable } from '@nestjs/common';
import { DataSource, IsNull, LessThan, QueryFailedError, Not } from 'typeorm';
import { DI } from '@/di-symbols.js'; import { DI } from '@/di-symbols.js';
import type { MiMeta, MiNote, NotesRepository } from '@/models/_.js'; import type { MiMeta, MiNote, NotesRepository } from '@/models/_.js';
import type Logger from '@/logger.js'; import type Logger from '@/logger.js';
@ -24,18 +25,31 @@ export class CleanRemoteNotesProcessorService {
@Inject(DI.notesRepository) @Inject(DI.notesRepository)
private notesRepository: NotesRepository, private notesRepository: NotesRepository,
@Inject(DI.db)
private db: DataSource,
private idService: IdService, private idService: IdService,
private queueLoggerService: QueueLoggerService, private queueLoggerService: QueueLoggerService,
) { ) {
this.logger = this.queueLoggerService.logger.createSubLogger('clean-remote-notes'); this.logger = this.queueLoggerService.logger.createSubLogger('clean-remote-notes');
} }
@bindThis
private computeProgress(minId: string, maxId: string, cursorLeft: string) {
const minTs = this.idService.parse(minId).date.getTime();
const maxTs = this.idService.parse(maxId).date.getTime();
const cursorTs = this.idService.parse(cursorLeft).date.getTime();
return ((cursorTs - minTs) / (maxTs - minTs)) * 100;
}
@bindThis @bindThis
public async process(job: Bull.Job<Record<string, unknown>>): Promise<{ public async process(job: Bull.Job<Record<string, unknown>>): Promise<{
deletedCount: number; deletedCount: number;
oldest: number | null; oldest: number | null;
newest: number | null; newest: number | null;
skipped?: boolean; skipped: boolean;
transientErrors: number;
}> { }> {
if (!this.meta.enableRemoteNotesCleaning) { if (!this.meta.enableRemoteNotesCleaning) {
this.logger.info('Remote notes cleaning is disabled, skipping...'); this.logger.info('Remote notes cleaning is disabled, skipping...');
@ -44,6 +58,7 @@ export class CleanRemoteNotesProcessorService {
oldest: null, oldest: null,
newest: null, newest: null,
skipped: true, skipped: true,
transientErrors: 0,
}; };
} }
@ -52,12 +67,10 @@ export class CleanRemoteNotesProcessorService {
const maxDuration = this.meta.remoteNotesCleaningMaxProcessingDurationInMinutes * 60 * 1000; // Convert minutes to milliseconds const maxDuration = this.meta.remoteNotesCleaningMaxProcessingDurationInMinutes * 60 * 1000; // Convert minutes to milliseconds
const startAt = Date.now(); const startAt = Date.now();
const MAX_NOTE_COUNT_PER_QUERY = 50; //#region queries
// The date limit for the newest note to be considered for deletion.
//#retion queries // All notes newer than this limit will always be retained.
// We use string literals instead of query builder for several reasons: const newestLimit = this.idService.gen(Date.now() - (1000 * 60 * 60 * 24 * this.meta.remoteNotesCleaningExpiryDaysForEachNotes));
// - for removeCondition, we need to use it in having clause, which is not supported by Brackets.
// - for recursive part, we need to preserve the order of columns, but typeorm query builder does not guarantee the order of columns in the result query
// The condition for removing the notes. // The condition for removing the notes.
// The note must be: // The note must be:
@ -66,56 +79,93 @@ export class CleanRemoteNotesProcessorService {
// - not have clipped // - not have clipped
// - not have pinned on the user profile // - not have pinned on the user profile
// - not has been favorite by any user // - not has been favorite by any user
const removeCondition = 'note.id < :newestLimit' const removalCriteria = [
+ ' AND note."clippedCount" = 0' 'note."id" < :newestLimit',
+ ' AND note."userHost" IS NOT NULL' 'note."clippedCount" = 0',
// using both userId and noteId instead of just noteId to use index on user_note_pining table. 'note."userHost" IS NOT NULL',
// This is safe because notes are only pinned by the user who created them. 'NOT EXISTS (SELECT 1 FROM user_note_pining WHERE "noteId" = note."id")',
+ ' AND NOT EXISTS(SELECT 1 FROM "user_note_pining" WHERE "noteId" = note."id" AND "userId" = note."userId")' 'NOT EXISTS (SELECT 1 FROM note_favorite WHERE "noteId" = note."id")',
// We cannot use userId trick because users can favorite notes from other users. ].join(' AND ');
+ ' AND NOT EXISTS(SELECT 1 FROM "note_favorite" WHERE "noteId" = note."id")'
;
// The initiator query contains the oldest ${MAX_NOTE_COUNT_PER_QUERY} remote non-clipped notes const minId = (await this.notesRepository.createQueryBuilder('note')
const initiatorQuery = this.notesRepository.createQueryBuilder('note') .select('MIN(note.id)', 'minId')
.where({
id: LessThan(newestLimit),
userHost: Not(IsNull()),
replyId: IsNull(),
renoteId: IsNull(),
})
.getRawOne<{ minId?: MiNote['id'] }>())?.minId;
if (!minId) {
this.logger.info('No notes can possibly be deleted, skipping...');
return {
deletedCount: 0,
oldest: null,
newest: null,
skipped: false,
transientErrors: 0,
};
}
// start with a conservative limit and adjust it based on the query duration
const minimumLimit = 10;
let currentLimit = 100;
let cursorLeft = '0';
const candidateNotesCteName = 'candidate_notes';
// tree walk down all root notes, short-circuit when the first unremovable note is found
const candidateNotesQueryBase = this.notesRepository.createQueryBuilder('note')
.select('note."id"', 'id')
.addSelect('note."replyId"', 'replyId')
.addSelect('note."renoteId"', 'renoteId')
.addSelect('note."id"', 'rootId')
.addSelect('TRUE', 'isRemovable')
.addSelect('TRUE', 'isBase')
.where('note."id" > :cursorLeft')
.andWhere(removalCriteria)
.andWhere({ replyId: IsNull(), renoteId: IsNull() });
const candidateNotesQueryInductive = this.notesRepository.createQueryBuilder('note')
.select('note.id', 'id') .select('note.id', 'id')
.where(removeCondition) .addSelect('note."replyId"', 'replyId')
.andWhere('note.id > :cursor') .addSelect('note."renoteId"', 'renoteId')
.orderBy('note.id', 'ASC') .addSelect('parent."rootId"', 'rootId')
.limit(MAX_NOTE_COUNT_PER_QUERY); .addSelect(removalCriteria, 'isRemovable')
.addSelect('FALSE', 'isBase')
.innerJoin(candidateNotesCteName, 'parent', 'parent."id" = note."replyId" OR parent."id" = note."renoteId"')
.where('parent."isRemovable" = TRUE');
// The union query queries the related notes and replies related to the initiator query // A note tree can be deleted if there are no unremovable rows with the same rootId.
const unionQuery = ` //
SELECT "note"."id", "note"."replyId", "note"."renoteId", rn."initiatorId" // `candidate_notes` will have the following structure after recursive query (some columns omitted):
FROM "note" "note" // After performing a LEFT JOIN with `candidate_notes` as `unremovable`,
INNER JOIN "related_notes" "rn" // the note tree containing unremovable notes will be anti-joined.
ON "note"."replyId" = rn.id // For removable rows, the `unremovable` columns will have `NULL` values.
OR "note"."renoteId" = rn.id // | id | rootId | isRemovable |
OR "note"."id" = rn."replyId" // |-----|--------|-------------|
OR "note"."id" = rn."renoteId" // | aaa | aaa | TRUE |
`; // | bbb | aaa | FALSE |
// | ccc | aaa | FALSE |
const selectRelatedNotesFromInitiatorIdsQuery = ` // | ddd | ddd | TRUE |
SELECT "note"."id" AS "id", "note"."replyId" AS "replyId", "note"."renoteId" AS "renoteId", "note"."id" AS "initiatorId" // | eee | ddd | TRUE |
FROM "note" "note" WHERE "note"."id" IN (:...initiatorIds) // | fff | fff | TRUE |
`; // | ggg | ggg | FALSE |
//
const recursiveQuery = `(${selectRelatedNotesFromInitiatorIdsQuery}) UNION (${unionQuery})`; const candidateNotesQuery = this.db.createQueryBuilder()
.select(`"${candidateNotesCteName}"."id"`, 'id')
const removableInitiatorNotesQuery = this.notesRepository.createQueryBuilder('note') .addSelect('unremovable."id" IS NULL', 'isRemovable')
.select('rn."initiatorId"') .addSelect(`BOOL_OR("${candidateNotesCteName}"."isBase")`, 'isBase')
.innerJoin('related_notes', 'rn', 'note.id = rn.id') .addCommonTableExpression(
.groupBy('rn."initiatorId"') `((SELECT "base".* FROM (${candidateNotesQueryBase.orderBy('note.id', 'ASC').limit(currentLimit).getQuery()}) AS "base") UNION ${candidateNotesQueryInductive.getQuery()})`,
.having(`bool_and(${removeCondition})`); candidateNotesCteName,
{ recursive: true },
const notesQuery = this.notesRepository.createQueryBuilder('note') )
.addCommonTableExpression(recursiveQuery, 'related_notes', { recursive: true }) .from(candidateNotesCteName, candidateNotesCteName)
.select('note.id', 'id') .leftJoin(candidateNotesCteName, 'unremovable', `unremovable."rootId" = "${candidateNotesCteName}"."rootId" AND unremovable."isRemovable" = FALSE`)
.addSelect('rn."initiatorId"') .groupBy(`"${candidateNotesCteName}"."id"`)
.innerJoin('related_notes', 'rn', 'note.id = rn.id') .addGroupBy('unremovable."id" IS NULL');
.where(`rn."initiatorId" IN (${removableInitiatorNotesQuery.getQuery()})`)
.distinctOn(['note.id']);
//#endregion
const stats = { const stats = {
deletedCount: 0, deletedCount: 0,
@ -123,74 +173,107 @@ export class CleanRemoteNotesProcessorService {
newest: null as number | null, newest: null as number | null,
}; };
// The date limit for the newest note to be considered for deletion. let lowThroughputWarned = false;
// All notes newer than this limit will always be retained. let transientErrors = 0;
const newestLimit = this.idService.gen(Date.now() - (1000 * 60 * 60 * 24 * this.meta.remoteNotesCleaningExpiryDaysForEachNotes)); for (;;) {
let cursor = '0'; // oldest note ID to start from
while (true) {
//#region check time //#region check time
const batchBeginAt = Date.now(); const batchBeginAt = Date.now();
const elapsed = batchBeginAt - startAt; const elapsed = batchBeginAt - startAt;
const progress = this.computeProgress(minId, newestLimit, cursorLeft > minId ? cursorLeft : minId);
if (elapsed >= maxDuration) { if (elapsed >= maxDuration) {
this.logger.info(`Reached maximum duration of ${maxDuration}ms, stopping...`); job.log(`Reached maximum duration of ${maxDuration}ms, stopping... (last cursor: ${cursorLeft}, final progress ${progress}%)`);
job.log('Reached maximum duration, stopping cleaning.');
job.updateProgress(100); job.updateProgress(100);
break; break;
} }
job.updateProgress((elapsed / maxDuration) * 100); const wallClockUsage = elapsed / maxDuration;
if (wallClockUsage > 0.5 && progress < 50 && !lowThroughputWarned) {
const msg = `Not projected to finish in time! (wall clock usage ${wallClockUsage * 100}% at ${progress}%, current limit ${currentLimit})`;
this.logger.warn(msg);
job.log(msg);
lowThroughputWarned = true;
}
job.updateProgress(progress);
//#endregion //#endregion
// First, we fetch the initiator notes that are older than the newestLimit. const queryBegin = performance.now();
const initiatorNotes: { id: MiNote['id'] }[] = await initiatorQuery.setParameters({ cursor, newestLimit }).getRawMany(); let noteIds = null;
// update the cursor to the newest initiatorId found in the fetched notes. try {
const newCursor = initiatorNotes.reduce((max, note) => note.id > max ? note.id : max, cursor); noteIds = await candidateNotesQuery.setParameters(
{ newestLimit, cursorLeft },
).getRawMany<{ id: MiNote['id'], isRemovable: boolean, isBase: boolean }>();
} catch (e) {
if (currentLimit > minimumLimit && e instanceof QueryFailedError && e.driverError?.code === '57014') {
// Statement timeout (maybe suddenly hit a large note tree), reduce the limit and try again
// continuous failures will eventually converge to currentLimit == minimumLimit and then throw
currentLimit = Math.max(minimumLimit, Math.floor(currentLimit * 0.25));
continue;
}
throw e;
}
if (initiatorNotes.length === 0 || cursor === newCursor || newCursor >= newestLimit) { if (noteIds.length === 0) {
// If no notes were found or the cursor did not change, we can stop. job.log('No more notes to clean.');
job.log('No more notes to clean. (no initiator notes found or cursor did not change.)');
break; break;
} }
const notes: { id: MiNote['id'], initiatorId: MiNote['id'] }[] = await notesQuery.setParameters({ const queryDuration = performance.now() - queryBegin;
initiatorIds: initiatorNotes.map(note => note.id), // try to adjust such that each query takes about 1~5 seconds and reasonable NodeJS heap so the task stays responsive
newestLimit, // this should not oscillate..
}).getRawMany(); if (queryDuration > 5000 || noteIds.length > 5000) {
currentLimit = Math.floor(currentLimit * 0.5);
} else if (queryDuration < 1000 && noteIds.length < 1000) {
currentLimit = Math.floor(currentLimit * 1.5);
}
// clamp to a sane range
currentLimit = Math.min(Math.max(currentLimit, minimumLimit), 5000);
cursor = newCursor; const deletableNoteIds = noteIds.filter(result => result.isRemovable).map(result => result.id);
if (deletableNoteIds.length > 0) {
try {
await this.notesRepository.delete(deletableNoteIds);
if (notes.length > 0) { for (const id of deletableNoteIds) {
await this.notesRepository.delete(notes.map(note => note.id)); const t = this.idService.parse(id).date.getTime();
if (stats.oldest === null || t < stats.oldest) {
for (const { id } of notes) { stats.oldest = t;
const t = this.idService.parse(id).date.getTime(); }
if (stats.oldest === null || t < stats.oldest) { if (stats.newest === null || t > stats.newest) {
stats.oldest = t; stats.newest = t;
}
} }
if (stats.newest === null || t > stats.newest) {
stats.newest = t; stats.deletedCount += deletableNoteIds.length;
} catch (e) {
// check for integrity violation errors (class 23) that might have occurred between the check and the delete
// we can safely continue to the next batch
if (e instanceof QueryFailedError && e.driverError?.code?.startsWith('23')) {
transientErrors++;
job.log(`Error deleting notes: ${e} (transient race condition?)`);
} else {
throw e;
} }
} }
stats.deletedCount += notes.length;
} }
job.log(`Deleted ${notes.length} from ${initiatorNotes.length} initiators; ${Date.now() - batchBeginAt}ms`); cursorLeft = noteIds.filter(result => result.isBase).reduce((max, { id }) => id > max ? id : max, cursorLeft);
if (initiatorNotes.length < MAX_NOTE_COUNT_PER_QUERY) { job.log(`Deleted ${noteIds.length} notes; ${Date.now() - batchBeginAt}ms`);
// If we fetched less than the maximum, it means there are no more notes to process.
job.log(`No more notes to clean. (fewer than MAX_NOTE_COUNT_PER_QUERY =${MAX_NOTE_COUNT_PER_QUERY}.)`); if (process.env.NODE_ENV !== 'test') {
break; await setTimeout(Math.min(1000 * 5, queryDuration)); // Wait a moment to avoid overwhelming the db
} }
};
await setTimeout(1000 * 5); // Wait a moment to avoid overwhelming the db if (transientErrors > 0) {
const msg = `${transientErrors} transient errors occurred while cleaning remote notes. You may need a second pass to complete the cleaning.`;
this.logger.warn(msg);
job.log(msg);
} }
this.logger.succ('cleaning of remote notes completed.'); this.logger.succ('cleaning of remote notes completed.');
return { return {
@ -198,6 +281,7 @@ export class CleanRemoteNotesProcessorService {
oldest: stats.oldest, oldest: stats.oldest,
newest: stats.newest, newest: stats.newest,
skipped: false, skipped: false,
transientErrors,
}; };
} }
} }

3
packages/backend/test/unit/queue/processors/CleanRemoteNotesProcessorService.ts
View File

@ -158,6 +158,7 @@ describe('CleanRemoteNotesProcessorService', () => {
oldest: null, oldest: null,
newest: null, newest: null,
skipped: true, skipped: true,
transientErrors: 0,
}); });
}); });
@ -172,6 +173,7 @@ describe('CleanRemoteNotesProcessorService', () => {
oldest: null, oldest: null,
newest: null, newest: null,
skipped: false, skipped: false,
transientErrors: 0,
}); });
}, 3000); }, 3000);
@ -199,6 +201,7 @@ describe('CleanRemoteNotesProcessorService', () => {
oldest: expect.any(Number), oldest: expect.any(Number),
newest: expect.any(Number), newest: expect.any(Number),
skipped: false, skipped: false,
transientErrors: 0,
}); });
// Check side-by-side from all notes // Check side-by-side from all notes