Add documentation for external sensitive media detection API

- Create API documentation explaining request/response format
- Add JSDoc comments to AiService methods
- Provide example implementation using nsfwjs

Co-authored-by: syuilo <4439005+syuilo@users.noreply.github.com>

docs/sensitive-media-detection-api.md

@@ -0,0 +1,117 @@
+# Sensitive Media Detection External Service API
+
+This document describes the API contract for an external sensitive media detection service that can be used with Misskey.
+
+## Endpoint
+
+The external service should expose a single HTTP POST endpoint at the URL configured in the Misskey admin settings (`sensitiveMediaDetectionProxyUrl`).
+
+## Request
+
+### Method
+`POST`
+
+### Headers
+- `Content-Type: application/json`
+
+### Body
+JSON object with the following structure:
+```json
+{
+  "image": "base64-encoded-image-data"
+}
+```
+
+Where `image` is the base64-encoded binary data of the image file to analyze.
+
+## Response
+
+### Success (HTTP 200)
+
+JSON object with the following structure:
+```json
+{
+  "predictions": [
+    {
+      "className": "Neutral",
+      "probability": 0.95
+    },
+    {
+      "className": "Drawing",
+      "probability": 0.03
+    },
+    {
+      "className": "Sexy",
+      "probability": 0.01
+    },
+    {
+      "className": "Porn",
+      "probability": 0.005
+    },
+    {
+      "className": "Hentai",
+      "probability": 0.005
+    }
+  ]
+}
+```
+
+The `predictions` array should contain objects with:
+- `className`: One of `"Neutral"`, `"Drawing"`, `"Sexy"`, `"Porn"`, or `"Hentai"`
+- `probability`: A number between 0 and 1 indicating the confidence level
+
+The classification names and behavior should match the [nsfwjs](https://github.com/infinitered/nsfwjs) library.
+
+### Error
+
+On error, the external service can return any HTTP error status code. Misskey will treat the request as failed and will not mark the file as sensitive (allowing the upload to proceed).
+
+## Timeout
+
+Requests will timeout after 10 seconds. The external service should respond within this time limit.
+
+## Example Implementation
+
+A simple example using nsfwjs:
+
+```javascript
+const express = require('express');
+const nsfw = require('nsfwjs');
+const tf = require('@tensorflow/tfjs-node');
+
+const app = express();
+app.use(express.json({ limit: '50mb' }));
+
+let model;
+
+async function loadModel() {
+  model = await nsfw.load();
+}
+
+app.post('/', async (req, res) => {
+  try {
+    const { image } = req.body;
+    const buffer = Buffer.from(image, 'base64');
+    const tfImage = await tf.node.decodeImage(buffer, 3);
+    const predictions = await model.classify(tfImage);
+    tfImage.dispose();
+    
+    res.json({ predictions });
+  } catch (error) {
+    res.status(500).json({ error: error.message });
+  }
+});
+
+loadModel().then(() => {
+  app.listen(3000, () => {
+    console.log('Sensitive media detection service running on port 3000');
+  });
+});
+```
+
+## Benefits of External Service
+
+- **Reduced Memory Usage**: nsfwjs and TensorFlow can be memory-intensive. Running them separately reduces Misskey's memory footprint.
+- **Scalability**: The detection service can be scaled independently.
+- **Alternative Implementations**: Operators can use different detection models or services (e.g., cloud-based APIs, more efficient models).
+- **Isolation**: Issues with the detection service don't affect Misskey's core functionality.

locales/en-US.yml

@@ -2123,6 +2123,8 @@ _role:
     not: "NOT-Condition"
 _sensitiveMediaDetection:
   description: "Reduces the effort of server moderation through automatically recognizing sensitive media via Machine Learning. This will slightly increase the load on the server."
+  proxyUrl: "Sensitive media detection proxy URL"
+  proxyUrlDescription: "URL of the external sensitive media detection service. Leave empty to use the built-in detection (nsfwjs). Set to an external service URL to offload detection and reduce server load."
   sensitivity: "Detection sensitivity"
   sensitivityDescription: "Reducing the sensitivity will lead to fewer misdetections (false positives) whereas increasing it will lead to fewer missed detections (false negatives)."
   setSensitiveFlagAutomatically: "Mark as sensitive"

locales/ja-JP.yml

@@ -2150,6 +2150,8 @@ _role:
 
 _sensitiveMediaDetection:
   description: "機械学習を使って自動でセンシティブなメディアを検出し、モデレーションに役立てることができます。サーバーの負荷が少し増えます。"
+  proxyUrl: "センシティブメディア検出プロキシURL"
+  proxyUrlDescription: "外部のセンシティブメディア検出サービスのURL。空欄にすると内蔵の検出機能(nsfwjs)を使用します。外部サービスのURLを設定すると、検出処理を外部化してサーバーの負荷を軽減できます。"
   sensitivity: "検出感度"
   sensitivityDescription: "感度を低くすると、誤検知(偽陽性)が減ります。感度を高くすると、検知漏れ(偽陰性)が減ります。"
   setSensitiveFlagAutomatically: "センシティブフラグを設定する"

packages/backend/migration/1731916961000-SensitiveMediaDetectionProxy.js

@@ -0,0 +1,16 @@
+/*
+ * SPDX-FileCopyrightText: syuilo and misskey-project
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+export class SensitiveMediaDetectionProxy1731916961000 {
+    name = 'SensitiveMediaDetectionProxy1731916961000'
+
+    async up(queryRunner) {
+        await queryRunner.query(`ALTER TABLE "meta" ADD "sensitiveMediaDetectionProxyUrl" character varying(1024)`);
+    }
+
+    async down(queryRunner) {
+        await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "sensitiveMediaDetectionProxyUrl"`);
+    }
+}

packages/backend/src/core/AiService.ts

@@ -6,12 +6,15 @@
 import * as fs from 'node:fs';
 import { fileURLToPath } from 'node:url';
 import { dirname } from 'node:path';
-import { Injectable } from '@nestjs/common';
+import { Inject, Injectable } from '@nestjs/common';
 import * as nsfw from 'nsfwjs';
 import si from 'systeminformation';
 import { Mutex } from 'async-mutex';
 import fetch from 'node-fetch';
 import { bindThis } from '@/decorators.js';
+import { DI } from '@/di-symbols.js';
+import type { MiMeta } from '@/models/Meta.js';
+import { HttpRequestService } from '@/core/HttpRequestService.js';
 
 const _filename = fileURLToPath(import.meta.url);
 const _dirname = dirname(_filename);
@@ -25,11 +28,31 @@ export class AiService {
 	private modelLoadMutex: Mutex = new Mutex();
 
 	constructor(
+		@Inject(DI.meta)
+		private meta: MiMeta,
+
+		private httpRequestService: HttpRequestService,
 	) {
 	}
 
+	/**
+	 * Detect sensitive content in an image.
+	 * 
+	 * If an external proxy URL is configured (sensitiveMediaDetectionProxyUrl),
+	 * this method will use the external service. Otherwise, it will use the
+	 * built-in nsfwjs model.
+	 * 
+	 * @param source - Path to the image file or a Buffer containing the image data
+	 * @returns Array of predictions or null if detection fails
+	 */
 	@bindThis
 	public async detectSensitive(source: string | Buffer): Promise<nsfw.PredictionType[] | null> {
+		// If external service is configured, use it
+		if (this.meta.sensitiveMediaDetectionProxyUrl) {
+			return await this.detectSensitiveWithProxy(source);
+		}
+
+		// Otherwise, use the local nsfwjs model
 		try {
 			if (isSupportedCpu === undefined) {
 				isSupportedCpu = await this.computeIsSupportedCpu();
@@ -65,6 +88,39 @@ export class AiService {
 		}
 	}
 
+	/**
+	 * Detect sensitive content using an external proxy service.
+	 * 
+	 * Sends the image as base64-encoded data to the external service
+	 * and expects a response in the same format as nsfwjs.
+	 * 
+	 * @param source - Path to the image file or a Buffer containing the image data
+	 * @returns Array of predictions or null if the request fails
+	 * @see docs/sensitive-media-detection-api.md for API contract details
+	 */
+	@bindThis
+	private async detectSensitiveWithProxy(source: string | Buffer): Promise<nsfw.PredictionType[] | null> {
+		try {
+			const buffer = source instanceof Buffer ? source : await fs.promises.readFile(source);
+			const base64 = buffer.toString('base64');
+			
+			const response = await this.httpRequestService.send(this.meta.sensitiveMediaDetectionProxyUrl!, {
+				method: 'POST',
+				headers: {
+					'Content-Type': 'application/json',
+				},
+				body: JSON.stringify({ image: base64 }),
+				timeout: 10000,
+			});
+
+			const json = await response.json() as { predictions: nsfw.PredictionType[] };
+			return json.predictions;
+		} catch (err) {
+			console.error('Failed to detect sensitive media with proxy:', err);
+			return null;
+		}
+	}
+
 	private async computeIsSupportedCpu(): Promise<boolean> {
 		switch (process.arch) {
 			case 'x64': {

packages/backend/src/models/Meta.ts

@@ -292,6 +292,13 @@ export class MiMeta {
 	})
 	public enableSensitiveMediaDetectionForVideos: boolean;
 
+	@Column('varchar', {
+		length: 1024,
+		nullable: true,
+		default: null,
+	})
+	public sensitiveMediaDetectionProxyUrl: string | null;
+
 	@Column('boolean', {
 		default: false,
 	})

packages/backend/src/server/api/endpoints/admin/meta.ts

@@ -238,6 +238,10 @@ export const meta = {
 				type: 'boolean',
 				optional: false, nullable: false,
 			},
+			sensitiveMediaDetectionProxyUrl: {
+				type: 'string',
+				optional: false, nullable: true,
+			},
 			proxyAccountId: {
 				type: 'string',
 				optional: false, nullable: false,
@@ -687,6 +691,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 				sensitiveMediaDetectionSensitivity: instance.sensitiveMediaDetectionSensitivity,
 				setSensitiveFlagAutomatically: instance.setSensitiveFlagAutomatically,
 				enableSensitiveMediaDetectionForVideos: instance.enableSensitiveMediaDetectionForVideos,
+				sensitiveMediaDetectionProxyUrl: instance.sensitiveMediaDetectionProxyUrl,
 				proxyAccountId: proxy.id,
 				email: instance.email,
 				smtpSecure: instance.smtpSecure,

packages/backend/src/server/api/endpoints/admin/update-meta.ts

@@ -90,6 +90,7 @@ export const paramDef = {
 		sensitiveMediaDetectionSensitivity: { type: 'string', enum: ['medium', 'low', 'high', 'veryLow', 'veryHigh'] },
 		setSensitiveFlagAutomatically: { type: 'boolean' },
 		enableSensitiveMediaDetectionForVideos: { type: 'boolean' },
+		sensitiveMediaDetectionProxyUrl: { type: 'string', nullable: true },
 		maintainerName: { type: 'string', nullable: true },
 		maintainerEmail: { type: 'string', nullable: true },
 		langs: {
@@ -422,6 +423,11 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 				set.enableSensitiveMediaDetectionForVideos = ps.enableSensitiveMediaDetectionForVideos;
 			}
 
+			if (ps.sensitiveMediaDetectionProxyUrl !== undefined) {
+				// eslint-disable-next-line @typescript-eslint/prefer-nullish-coalescing
+				set.sensitiveMediaDetectionProxyUrl = ps.sensitiveMediaDetectionProxyUrl || null;
+			}
+
 			if (ps.maintainerName !== undefined) {
 				set.maintainerName = ps.maintainerName;
 			}

packages/frontend/src/pages/admin/security.vue

@@ -25,6 +25,13 @@ SPDX-License-Identifier: AGPL-3.0-only
 						<div class="_gaps_m">
 							<div><SearchText>{{ i18n.ts._sensitiveMediaDetection.description }}</SearchText></div>
 
+							<SearchMarker :keywords="['proxy', 'url', 'external', 'service']">
+								<MkInput v-model="sensitiveMediaDetectionForm.state.sensitiveMediaDetectionProxyUrl">
+									<template #label><SearchLabel>{{ i18n.ts._sensitiveMediaDetection.proxyUrl }}</SearchLabel></template>
+									<template #caption><SearchText>{{ i18n.ts._sensitiveMediaDetection.proxyUrlDescription }}</SearchText></template>
+								</MkInput>
+							</SearchMarker>
+
 							<MkRadios v-model="sensitiveMediaDetectionForm.state.sensitiveMediaDetection">
 								<option value="none">{{ i18n.ts.none }}</option>
 								<option value="all">{{ i18n.ts.all }}</option>
@@ -185,6 +192,7 @@ const sensitiveMediaDetectionForm = useForm({
 	meta.sensitiveMediaDetectionSensitivity === 'veryHigh' ? 4 : 0,
 	setSensitiveFlagAutomatically: meta.setSensitiveFlagAutomatically,
 	enableSensitiveMediaDetectionForVideos: meta.enableSensitiveMediaDetectionForVideos,
+	sensitiveMediaDetectionProxyUrl: meta.sensitiveMediaDetectionProxyUrl,
 }, async (state) => {
 	await os.apiWithDialog('admin/update-meta', {
 		sensitiveMediaDetection: state.sensitiveMediaDetection,
@@ -197,6 +205,7 @@ const sensitiveMediaDetectionForm = useForm({
 			null as never,
 		setSensitiveFlagAutomatically: state.setSensitiveFlagAutomatically,
 		enableSensitiveMediaDetectionForVideos: state.enableSensitiveMediaDetectionForVideos,
+		sensitiveMediaDetectionProxyUrl: state.sensitiveMediaDetectionProxyUrl,
 	});
 	fetchInstance(true);
 });