From a697a7f97b401d1545a7f61694b2704b4d3ac9fc Mon Sep 17 00:00:00 2001
From: syuilo <4439005+syuilo@users.noreply.github.com>
Date: Thu, 6 Jun 2024 11:38:34 +0900
Subject: [PATCH] enhance(backend): improve sentry integration

---
 .../src/queue/QueueProcessorService.ts        | 63 ++++++++++++++++---
 1 file changed, 56 insertions(+), 7 deletions(-)

diff --git a/packages/backend/src/queue/QueueProcessorService.ts b/packages/backend/src/queue/QueueProcessorService.ts
index 4f333df791..fdeb6a9518 100644
--- a/packages/backend/src/queue/QueueProcessorService.ts
+++ b/packages/backend/src/queue/QueueProcessorService.ts
@@ -165,7 +165,14 @@ export class QueueProcessorService implements OnApplicationShutdown {
 			this.systemQueueWorker
 				.on('active', (job) => systemLogger.debug(`active id=${job.id}`))
 				.on('completed', (job, result) => systemLogger.debug(`completed(${result}) id=${job.id}`))
-				.on('failed', (job, err) => systemLogger.error(`failed(${err.stack}) id=${job ? job.id : '-'}`, { job, e: renderError(err) }))
+				.on('failed', (job, err) => {
+					systemLogger.error(`failed(${err.stack}) id=${job ? job.id : '-'}`, { job, e: renderError(err) });
+					if (config.sentryForBackend) {
+						Sentry.captureMessage(`Queue: System: ${job?.name ?? '?'}`, {
+							extra: { job, err },
+						});
+					}
+				})
 				.on('error', (err: Error) => systemLogger.error(`error ${err.stack}`, { e: renderError(err) }))
 				.on('stalled', (jobId) => systemLogger.warn(`stalled id=${jobId}`));
 		}
@@ -214,7 +221,14 @@ export class QueueProcessorService implements OnApplicationShutdown {
 			this.dbQueueWorker
 				.on('active', (job) => dbLogger.debug(`active id=${job.id}`))
 				.on('completed', (job, result) => dbLogger.debug(`completed(${result}) id=${job.id}`))
-				.on('failed', (job, err) => dbLogger.error(`failed(${err.stack}) id=${job ? job.id : '-'}`, { job, e: renderError(err) }))
+				.on('failed', (job, err) => {
+					dbLogger.error(`failed(${err.stack}) id=${job ? job.id : '-'}`, { job, e: renderError(err) });
+					if (config.sentryForBackend) {
+						Sentry.captureMessage(`Queue: DB: ${job?.name ?? '?'}`, {
+							extra: { job, err },
+						});
+					}
+				})
 				.on('error', (err: Error) => dbLogger.error(`error ${err.stack}`, { e: renderError(err) }))
 				.on('stalled', (jobId) => dbLogger.warn(`stalled id=${jobId}`));
 		}
@@ -246,7 +260,14 @@ export class QueueProcessorService implements OnApplicationShutdown {
 			this.deliverQueueWorker
 				.on('active', (job) => deliverLogger.debug(`active ${getJobInfo(job, true)} to=${job.data.to}`))
 				.on('completed', (job, result) => deliverLogger.debug(`completed(${result}) ${getJobInfo(job, true)} to=${job.data.to}`))
-				.on('failed', (job, err) => deliverLogger.warn(`failed(${err.stack}) ${getJobInfo(job)} to=${job ? job.data.to : '-'}`))
+				.on('failed', (job, err) => {
+					deliverLogger.error(`failed(${err.stack}) ${getJobInfo(job)} to=${job ? job.data.to : '-'}`);
+					if (config.sentryForBackend) {
+						Sentry.captureMessage('Queue: Deliver', {
+							extra: { job, err },
+						});
+					}
+				})
 				.on('error', (err: Error) => deliverLogger.error(`error ${err.stack}`, { e: renderError(err) }))
 				.on('stalled', (jobId) => deliverLogger.warn(`stalled id=${jobId}`));
 		}
@@ -278,7 +299,14 @@ export class QueueProcessorService implements OnApplicationShutdown {
 			this.inboxQueueWorker
 				.on('active', (job) => inboxLogger.debug(`active ${getJobInfo(job, true)}`))
 				.on('completed', (job, result) => inboxLogger.debug(`completed(${result}) ${getJobInfo(job, true)}`))
-				.on('failed', (job, err) => inboxLogger.warn(`failed(${err.stack}) ${getJobInfo(job)} activity=${job ? (job.data.activity ? job.data.activity.id : 'none') : '-'}`, { job, e: renderError(err) }))
+				.on('failed', (job, err) => {
+					inboxLogger.error(`failed(${err.stack}) ${getJobInfo(job)} activity=${job ? (job.data.activity ? job.data.activity.id : 'none') : '-'}`, { job, e: renderError(err) });
+					if (config.sentryForBackend) {
+						Sentry.captureMessage('Queue: Inbox', {
+							extra: { job, err },
+						});
+					}
+				})
 				.on('error', (err: Error) => inboxLogger.error(`error ${err.stack}`, { e: renderError(err) }))
 				.on('stalled', (jobId) => inboxLogger.warn(`stalled id=${jobId}`));
 		}
@@ -310,7 +338,14 @@ export class QueueProcessorService implements OnApplicationShutdown {
 			this.webhookDeliverQueueWorker
 				.on('active', (job) => webhookLogger.debug(`active ${getJobInfo(job, true)} to=${job.data.to}`))
 				.on('completed', (job, result) => webhookLogger.debug(`completed(${result}) ${getJobInfo(job, true)} to=${job.data.to}`))
-				.on('failed', (job, err) => webhookLogger.warn(`failed(${err.stack}) ${getJobInfo(job)} to=${job ? job.data.to : '-'}`))
+				.on('failed', (job, err) => {
+					webhookLogger.error(`failed(${err.stack}) ${getJobInfo(job)} to=${job ? job.data.to : '-'}`);
+					if (config.sentryForBackend) {
+						Sentry.captureMessage('Queue: WebhookDeliver', {
+							extra: { job, err },
+						});
+					}
+				})
 				.on('error', (err: Error) => webhookLogger.error(`error ${err.stack}`, { e: renderError(err) }))
 				.on('stalled', (jobId) => webhookLogger.warn(`stalled id=${jobId}`));
 		}
@@ -349,7 +384,14 @@ export class QueueProcessorService implements OnApplicationShutdown {
 			this.relationshipQueueWorker
 				.on('active', (job) => relationshipLogger.debug(`active id=${job.id}`))
 				.on('completed', (job, result) => relationshipLogger.debug(`completed(${result}) id=${job.id}`))
-				.on('failed', (job, err) => relationshipLogger.warn(`failed(${err.stack}) id=${job ? job.id : '-'}`, { job, e: renderError(err) }))
+				.on('failed', (job, err) => {
+					relationshipLogger.error(`failed(${err.stack}) id=${job ? job.id : '-'}`, { job, e: renderError(err) });
+					if (config.sentryForBackend) {
+						Sentry.captureMessage(`Queue: Relationship: ${job?.name ?? '?'}`, {
+							extra: { job, err },
+						});
+					}
+				})
 				.on('error', (err: Error) => relationshipLogger.error(`error ${err.stack}`, { e: renderError(err) }))
 				.on('stalled', (jobId) => relationshipLogger.warn(`stalled id=${jobId}`));
 		}
@@ -382,7 +424,14 @@ export class QueueProcessorService implements OnApplicationShutdown {
 			this.objectStorageQueueWorker
 				.on('active', (job) => objectStorageLogger.debug(`active id=${job.id}`))
 				.on('completed', (job, result) => objectStorageLogger.debug(`completed(${result}) id=${job.id}`))
-				.on('failed', (job, err) => objectStorageLogger.warn(`failed(${err.stack}) id=${job ? job.id : '-'}`, { job, e: renderError(err) }))
+				.on('failed', (job, err) => {
+					objectStorageLogger.error(`failed(${err.stack}) id=${job ? job.id : '-'}`, { job, e: renderError(err) });
+					if (config.sentryForBackend) {
+						Sentry.captureMessage(`Queue: ObjectStorage: ${job?.name ?? '?'}`, {
+							extra: { job, err },
+						});
+					}
+				})
 				.on('error', (err: Error) => objectStorageLogger.error(`error ${err.stack}`, { e: renderError(err) }))
 				.on('stalled', (jobId) => objectStorageLogger.warn(`stalled id=${jobId}`));
 		}