Add external service support for sensitive media detection

- Add sensitiveMediaDetectionProxyUrl field to Meta model
- Create migration for new database field
- Update admin API endpoints to expose and update the proxy URL
- Modify AiService to support external API calls when proxy URL is configured
- Update frontend admin UI to add proxy URL configuration field
- Add i18n strings for proxy URL in English and Japanese

Co-authored-by: syuilo <4439005+syuilo@users.noreply.github.com>

locales/en-US.yml

@@ -2123,6 +2123,8 @@ _role:
     not: "NOT-Condition"
 _sensitiveMediaDetection:
   description: "Reduces the effort of server moderation through automatically recognizing sensitive media via Machine Learning. This will slightly increase the load on the server."
+  proxyUrl: "Sensitive media detection proxy URL"
+  proxyUrlDescription: "URL of the external sensitive media detection service. Leave empty to use the built-in detection (nsfwjs). Set to an external service URL to offload detection and reduce server load."
   sensitivity: "Detection sensitivity"
   sensitivityDescription: "Reducing the sensitivity will lead to fewer misdetections (false positives) whereas increasing it will lead to fewer missed detections (false negatives)."
   setSensitiveFlagAutomatically: "Mark as sensitive"

locales/ja-JP.yml

@@ -2150,6 +2150,8 @@ _role:
 
 _sensitiveMediaDetection:
   description: "機械学習を使って自動でセンシティブなメディアを検出し、モデレーションに役立てることができます。サーバーの負荷が少し増えます。"
+  proxyUrl: "センシティブメディア検出プロキシURL"
+  proxyUrlDescription: "外部のセンシティブメディア検出サービスのURL。空欄にすると内蔵の検出機能(nsfwjs)を使用します。外部サービスのURLを設定すると、検出処理を外部化してサーバーの負荷を軽減できます。"
   sensitivity: "検出感度"
   sensitivityDescription: "感度を低くすると、誤検知(偽陽性)が減ります。感度を高くすると、検知漏れ(偽陰性)が減ります。"
   setSensitiveFlagAutomatically: "センシティブフラグを設定する"

packages/backend/migration/1731916961000-SensitiveMediaDetectionProxy.js

@@ -0,0 +1,16 @@
+/*
+ * SPDX-FileCopyrightText: syuilo and misskey-project
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+export class SensitiveMediaDetectionProxy1731916961000 {
+    name = 'SensitiveMediaDetectionProxy1731916961000'
+
+    async up(queryRunner) {
+        await queryRunner.query(`ALTER TABLE "meta" ADD "sensitiveMediaDetectionProxyUrl" character varying(1024)`);
+    }
+
+    async down(queryRunner) {
+        await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "sensitiveMediaDetectionProxyUrl"`);
+    }
+}

packages/backend/src/core/AiService.ts

@@ -6,12 +6,15 @@
 import * as fs from 'node:fs';
 import { fileURLToPath } from 'node:url';
 import { dirname } from 'node:path';
-import { Injectable } from '@nestjs/common';
+import { Inject, Injectable } from '@nestjs/common';
 import * as nsfw from 'nsfwjs';
 import si from 'systeminformation';
 import { Mutex } from 'async-mutex';
 import fetch from 'node-fetch';
 import { bindThis } from '@/decorators.js';
+import { DI } from '@/di-symbols.js';
+import type { MiMeta } from '@/models/Meta.js';
+import { HttpRequestService } from '@/core/HttpRequestService.js';
 
 const _filename = fileURLToPath(import.meta.url);
 const _dirname = dirname(_filename);
@@ -25,11 +28,21 @@ export class AiService {
 	private modelLoadMutex: Mutex = new Mutex();
 
 	constructor(
+		@Inject(DI.meta)
+		private meta: MiMeta,
+
+		private httpRequestService: HttpRequestService,
 	) {
 	}
 
 	@bindThis
 	public async detectSensitive(source: string | Buffer): Promise<nsfw.PredictionType[] | null> {
+		// If external service is configured, use it
+		if (this.meta.sensitiveMediaDetectionProxyUrl) {
+			return await this.detectSensitiveWithProxy(source);
+		}
+
+		// Otherwise, use the local nsfwjs model
 		try {
 			if (isSupportedCpu === undefined) {
 				isSupportedCpu = await this.computeIsSupportedCpu();
@@ -65,6 +78,29 @@ export class AiService {
 		}
 	}
 
+	@bindThis
+	private async detectSensitiveWithProxy(source: string | Buffer): Promise<nsfw.PredictionType[] | null> {
+		try {
+			const buffer = source instanceof Buffer ? source : await fs.promises.readFile(source);
+			const base64 = buffer.toString('base64');
+			
+			const response = await this.httpRequestService.send(this.meta.sensitiveMediaDetectionProxyUrl!, {
+				method: 'POST',
+				headers: {
+					'Content-Type': 'application/json',
+				},
+				body: JSON.stringify({ image: base64 }),
+				timeout: 10000,
+			});
+
+			const json = await response.json() as { predictions: nsfw.PredictionType[] };
+			return json.predictions;
+		} catch (err) {
+			console.error('Failed to detect sensitive media with proxy:', err);
+			return null;
+		}
+	}
+
 	private async computeIsSupportedCpu(): Promise<boolean> {
 		switch (process.arch) {
 			case 'x64': {

packages/backend/src/models/Meta.ts

@@ -292,6 +292,13 @@ export class MiMeta {
 	})
 	public enableSensitiveMediaDetectionForVideos: boolean;
 
+	@Column('varchar', {
+		length: 1024,
+		nullable: true,
+		default: null,
+	})
+	public sensitiveMediaDetectionProxyUrl: string | null;
+
 	@Column('boolean', {
 		default: false,
 	})

packages/backend/src/server/api/endpoints/admin/meta.ts

@@ -238,6 +238,10 @@ export const meta = {
 				type: 'boolean',
 				optional: false, nullable: false,
 			},
+			sensitiveMediaDetectionProxyUrl: {
+				type: 'string',
+				optional: false, nullable: true,
+			},
 			proxyAccountId: {
 				type: 'string',
 				optional: false, nullable: false,
@@ -687,6 +691,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 				sensitiveMediaDetectionSensitivity: instance.sensitiveMediaDetectionSensitivity,
 				setSensitiveFlagAutomatically: instance.setSensitiveFlagAutomatically,
 				enableSensitiveMediaDetectionForVideos: instance.enableSensitiveMediaDetectionForVideos,
+				sensitiveMediaDetectionProxyUrl: instance.sensitiveMediaDetectionProxyUrl,
 				proxyAccountId: proxy.id,
 				email: instance.email,
 				smtpSecure: instance.smtpSecure,

packages/backend/src/server/api/endpoints/admin/update-meta.ts

@@ -90,6 +90,7 @@ export const paramDef = {
 		sensitiveMediaDetectionSensitivity: { type: 'string', enum: ['medium', 'low', 'high', 'veryLow', 'veryHigh'] },
 		setSensitiveFlagAutomatically: { type: 'boolean' },
 		enableSensitiveMediaDetectionForVideos: { type: 'boolean' },
+		sensitiveMediaDetectionProxyUrl: { type: 'string', nullable: true },
 		maintainerName: { type: 'string', nullable: true },
 		maintainerEmail: { type: 'string', nullable: true },
 		langs: {
@@ -422,6 +423,11 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 				set.enableSensitiveMediaDetectionForVideos = ps.enableSensitiveMediaDetectionForVideos;
 			}
 
+			if (ps.sensitiveMediaDetectionProxyUrl !== undefined) {
+				// eslint-disable-next-line @typescript-eslint/prefer-nullish-coalescing
+				set.sensitiveMediaDetectionProxyUrl = ps.sensitiveMediaDetectionProxyUrl || null;
+			}
+
 			if (ps.maintainerName !== undefined) {
 				set.maintainerName = ps.maintainerName;
 			}

packages/frontend/src/pages/admin/security.vue

@@ -25,6 +25,13 @@ SPDX-License-Identifier: AGPL-3.0-only
 						<div class="_gaps_m">
 							<div><SearchText>{{ i18n.ts._sensitiveMediaDetection.description }}</SearchText></div>
 
+							<SearchMarker :keywords="['proxy', 'url', 'external', 'service']">
+								<MkInput v-model="sensitiveMediaDetectionForm.state.sensitiveMediaDetectionProxyUrl">
+									<template #label><SearchLabel>{{ i18n.ts._sensitiveMediaDetection.proxyUrl }}</SearchLabel></template>
+									<template #caption><SearchText>{{ i18n.ts._sensitiveMediaDetection.proxyUrlDescription }}</SearchText></template>
+								</MkInput>
+							</SearchMarker>
+
 							<MkRadios v-model="sensitiveMediaDetectionForm.state.sensitiveMediaDetection">
 								<option value="none">{{ i18n.ts.none }}</option>
 								<option value="all">{{ i18n.ts.all }}</option>
@@ -185,6 +192,7 @@ const sensitiveMediaDetectionForm = useForm({
 	meta.sensitiveMediaDetectionSensitivity === 'veryHigh' ? 4 : 0,
 	setSensitiveFlagAutomatically: meta.setSensitiveFlagAutomatically,
 	enableSensitiveMediaDetectionForVideos: meta.enableSensitiveMediaDetectionForVideos,
+	sensitiveMediaDetectionProxyUrl: meta.sensitiveMediaDetectionProxyUrl,
 }, async (state) => {
 	await os.apiWithDialog('admin/update-meta', {
 		sensitiveMediaDetection: state.sensitiveMediaDetection,
@@ -197,6 +205,7 @@ const sensitiveMediaDetectionForm = useForm({
 			null as never,
 		setSensitiveFlagAutomatically: state.setSensitiveFlagAutomatically,
 		enableSensitiveMediaDetectionForVideos: state.enableSensitiveMediaDetectionForVideos,
+		sensitiveMediaDetectionProxyUrl: state.sensitiveMediaDetectionProxyUrl,
 	});
 	fetchInstance(true);
 });