From 3e93450fd4b8f8a08110ccb2cda17297ae1c7c7f Mon Sep 17 00:00:00 2001
From: riku6460 <17585784+riku6460@users.noreply.github.com>
Date: Wed, 5 Jul 2023 23:59:32 +0900
Subject: [PATCH] perf(backend): Use addBulk to add deliver queues (#96)

* perf(backend): Use addBulk to add deliver queues (#11114)

* BullMQ -> Bull

---------

Co-authored-by: tamaina <tamaina@hotmail.co.jp>
---
 packages/backend/src/core/QueueService.ts     | 37 ++++++++++++++++++-
 .../activitypub/ApDeliverManagerService.ts    | 19 ++++++----
 2 files changed, 46 insertions(+), 10 deletions(-)

diff --git a/packages/backend/src/core/QueueService.ts b/packages/backend/src/core/QueueService.ts
index b4ffffecc0..1c32208c05 100644
--- a/packages/backend/src/core/QueueService.ts
+++ b/packages/backend/src/core/QueueService.ts
@@ -9,7 +9,7 @@ import { DI } from '@/di-symbols.js';
 import { bindThis } from '@/decorators.js';
 import type { Antenna } from '@/server/api/endpoints/i/import-antennas.js';
 import type { DbQueue, DeliverQueue, EndedPollNotificationQueue, InboxQueue, ObjectStorageQueue, RelationshipQueue, SystemQueue, WebhookDeliverQueue } from './QueueModule.js';
-import type { DbJobData, RelationshipJobData, ThinUser } from '../queue/types.js';
+import type { DbJobData, DeliverJobData, RelationshipJobData, ThinUser } from '../queue/types.js';
 import type httpSignature from '@peertube/http-signature';
 
 @Injectable()
@@ -33,7 +33,7 @@ export class QueueService {
 		if (content == null) return null;
 		if (to == null) return null;
 
-		const data = {
+		const data: DeliverJobData = {
 			user: {
 				id: user.id,
 			},
@@ -53,6 +53,39 @@ export class QueueService {
 		});
 	}
 
+	/**
+	 * ApDeliverManager-DeliverManager.execute()からinboxesを突っ込んでaddBulkしたい
+	 * @param user `{ id: string; }` この関数ではThinUserに変換しないので前もって変換してください
+	 * @param content IActivity | null
+	 * @param inboxes `Map<string, boolean>` / key: to (inbox url), value: isSharedInbox (whether it is sharedInbox)
+	 * @returns void
+	 */
+	@bindThis
+	public async deliverMany(user: ThinUser, content: IActivity | null, inboxes: Map<string, boolean>) {
+		const opts = {
+			attempts: this.config.deliverJobMaxAttempts ?? 12,
+			timeout: 1 * 60 * 1000,	// 1min
+			backoff: {
+				type: 'apBackoff',
+			},
+			removeOnComplete: true,
+			removeOnFail: true,
+		};
+
+		await this.deliverQueue.addBulk(Array.from(inboxes.entries()).map(d => ({
+			name: d[0],
+			data: {
+				user,
+				content,
+				to: d[0],
+				isSharedInbox: d[1],
+			} as DeliverJobData,
+			opts,
+		})));
+
+		return;
+	}
+
 	@bindThis
 	public inbox(activity: IActivity, signature: httpSignature.IParsedSignature) {
 		const data = {
diff --git a/packages/backend/src/core/activitypub/ApDeliverManagerService.ts b/packages/backend/src/core/activitypub/ApDeliverManagerService.ts
index 62a2a33a19..66e7761187 100644
--- a/packages/backend/src/core/activitypub/ApDeliverManagerService.ts
+++ b/packages/backend/src/core/activitypub/ApDeliverManagerService.ts
@@ -7,6 +7,7 @@ import type { LocalUser, RemoteUser, User } from '@/models/entities/User.js';
 import { QueueService } from '@/core/QueueService.js';
 import { UserEntityService } from '@/core/entities/UserEntityService.js';
 import { bindThis } from '@/decorators.js';
+import { ThinUser } from '@/queue/types.js';
 
 interface IRecipe {
 	type: string;
@@ -94,7 +95,7 @@ export class ApDeliverManagerService {
 }
 
 class DeliverManager {
-	private actor: { id: User['id']; host: null; };
+	private actor: ThinUser;
 	private activity: any;
 	private recipes: IRecipe[] = [];
 
@@ -111,7 +112,13 @@ class DeliverManager {
 		actor: { id: User['id']; host: null; },
 		activity: any,
 	) {
-		this.actor = actor;
+		// 型で弾いてはいるが一応ローカルユーザーかチェック
+		if (actor.host != null) throw new Error('actor.host must be null');
+
+		// パフォーマンス向上のためキューに突っ込むのはidのみに絞る
+		this.actor = {
+			id: actor.id,
+		};
 		this.activity = activity;
 	}
 
@@ -155,9 +162,8 @@ class DeliverManager {
 	 */
 	@bindThis
 	public async execute() {
-		if (!this.userEntityService.isLocalUser(this.actor)) return;
-
 		// The value flags whether it is shared or not.
+		// key: inbox URL, value: whether it is sharedInbox
 		const inboxes = new Map<string, boolean>();
 
 		/*
@@ -201,9 +207,6 @@ class DeliverManager {
 			.forEach(recipe => inboxes.set(recipe.to.inbox!, false));
 
 		// deliver
-		for (const inbox of inboxes) {
-			// inbox[0]: inbox, inbox[1]: whether it is sharedInbox
-			this.queueService.deliver(this.actor, this.activity, inbox[0], inbox[1]);
-		}
+		this.queueService.deliverMany(this.actor, this.activity, inboxes);
 	}
 }