Merge add32485fe into 9871035597

* fix(backend): fix apResolver

* fix

* add comments

* tweak comment

.config/docker_example.yml

@@ -114,9 +114,27 @@ redis:
 #  #prefix: example-prefix
 #  #db: 1
 
-#   ┌───────────────────────────┐
-#───┘ MeiliSearch configuration └─────────────────────────────
+#   ┌───────────────────────────────┐
+#───┘ Fulltext search configuration └─────────────────────────────
 
+# These are the setting items for the full-text search provider.
+fulltextSearch:
+  # You can select the ID generation method.
+  # - sqlLike (default)
+  #   Use SQL-like search.
+  #   This is a standard feature of PostgreSQL, so no special extensions are required.
+  # - sqlPgroonga
+  #   Use pgroonga.
+  #   You need to install pgroonga and configure it as a PostgreSQL extension.
+  #   In addition to the above, you need to create a pgroonga index on the text column of the note table.
+  #   see: https://pgroonga.github.io/tutorial/
+  # - meilisearch
+  #   Use Meilisearch.
+  #   You need to install Meilisearch and configure.
+  provider: sqlLike
+
+# For Meilisearch settings.
+# If you select "meilisearch" for "fulltextSearch.provider", it must be set.
 # You can set scope to local (default value) or global
 # (include notes from remote).
 

.config/example.yml

@@ -196,9 +196,27 @@ redis:
 #  # You can specify more ioredis options...
 #  #username: example-username
 
-#   ┌───────────────────────────┐
-#───┘ MeiliSearch configuration └─────────────────────────────
+#   ┌───────────────────────────────┐
+#───┘ Fulltext search configuration └─────────────────────────────
 
+# These are the setting items for the full-text search provider.
+fulltextSearch:
+  # You can select the ID generation method.
+  # - sqlLike (default)
+  #   Use SQL-like search.
+  #   This is a standard feature of PostgreSQL, so no special extensions are required.
+  # - sqlPgroonga
+  #   Use pgroonga.
+  #   You need to install pgroonga and configure it as a PostgreSQL extension.
+  #   In addition to the above, you need to create a pgroonga index on the text column of the note table.
+  #   see: https://pgroonga.github.io/tutorial/
+  # - meilisearch
+  #   Use Meilisearch.
+  #   You need to install Meilisearch and configure.
+  provider: sqlLike
+
+# For Meilisearch settings.
+# If you select "meilisearch" for "fulltextSearch.provider", it must be set.
 # You can set scope to local (default value) or global
 # (include notes from remote).
 

CHANGELOG.md

@@ -1,6 +1,10 @@
 ## 2024.11.0
 
 ### Note
+- [重要] ノート検索プロバイダの追加に伴い、configファイル（default.ymlなど）の構成が少し変わります.
+  - 新しい設定項目"fulltextSearch.provider"が追加されました. sqlLike, sqlPgroonga, meilisearchのいずれかを設定出来ます.
+  - すでにMeilisearchをお使いの場合、 **"fulltextSearch.provider"を"meilisearch"に設定する必要** があります.
+  - 詳細は #14730 および `.config/example.yml` または `.config/docker_example.yml`の'Fulltext search configuration'をご参照願います.
 - Node.js 20.xは非推奨になりました。Node.js 22.x (LTS)の利用を推奨します。
   - なお、Node.js 23.xは対応していません。
 - DockerのNode.jsが22.11.0に更新されました
@@ -51,6 +55,7 @@
   (Based on https://activitypub.software/TransFem-org/Sharkey/-/merge_requests/588)  
   (Cherry-picked from https://activitypub.software/TransFem-org/Sharkey/-/merge_requests/715)
 - Enhance: リモートユーザーの照会をオリジナルにリダイレクトするように
+- Enhance: ノート検索の選択肢としてpgroongaに対応 ( #14730 )
 - Fix: sharedInboxが無いActorに紐づくリモートユーザーを照会できない
 - Fix: Aproving request from GtS appears with some delay
 - Fix: フォロワーへのメッセージの絵文字をemojisに含めるように
@@ -67,6 +72,7 @@
 - Fix: User Webhookテスト機能のMock Payloadを修正  
 - Fix: アカウント削除のモデレーションログが動作していないのを修正 (#14996)  
 - Fix: リノートミュートが新規投稿通知に対して作用していなかった問題を修正
+- Fix: セキュリティに関する修正
 
 ### Misskey.js
 - Fix: Stream初期化時、別途WebSocketを指定する場合の型定義を修正

package.json

@@ -1,6 +1,6 @@
 {
 	"name": "misskey",
-	"version": "2024.11.0-alpha.2",
+	"version": "2024.11.0-alpha.3",
 	"codename": "nasubi",
 	"repository": {
 		"type": "git",

packages/backend/src/GlobalModule.ts

@@ -7,14 +7,14 @@ import { Global, Inject, Module } from '@nestjs/common';
 import * as Redis from 'ioredis';
 import { DataSource } from 'typeorm';
 import { MeiliSearch } from 'meilisearch';
+import { MiMeta } from '@/models/Meta.js';
 import { DI } from './di-symbols.js';
 import { Config, loadConfig } from './config.js';
 import { createPostgresDataSource } from './postgres.js';
 import { RepositoryModule } from './models/RepositoryModule.js';
 import { allSettled } from './misc/promise-tracker.js';
-import type { Provider, OnApplicationShutdown } from '@nestjs/common';
-import { MiMeta } from '@/models/Meta.js';
 import { GlobalEvents } from './core/GlobalEventService.js';
+import type { Provider, OnApplicationShutdown } from '@nestjs/common';
 
 const $config: Provider = {
 	provide: DI.config,
@@ -33,7 +33,11 @@ const $db: Provider = {
 const $meilisearch: Provider = {
 	provide: DI.meilisearch,
 	useFactory: (config: Config) => {
-		if (config.meilisearch) {
+		if (config.fulltextSearch?.provider === 'meilisearch') {
+			if (!config.meilisearch) {
+				throw new Error('MeiliSearch is enabled but no configuration is provided');
+			}
+
 			return new MeiliSearch({
 				host: `${config.meilisearch.ssl ? 'https' : 'http'}://${config.meilisearch.host}:${config.meilisearch.port}`,
 				apiKey: config.meilisearch.apiKey,

packages/backend/src/config.ts

@@ -50,6 +50,9 @@ type Source = {
 	redisForJobQueue?: RedisOptionsSource;
 	redisForTimelines?: RedisOptionsSource;
 	redisForReactions?: RedisOptionsSource;
+	fulltextSearch?: {
+		provider?: FulltextSearchProvider;
+	};
 	meilisearch?: {
 		host: string;
 		port: string;
@@ -124,6 +127,9 @@ export type Config = {
 		user: string;
 		pass: string;
 	}[] | undefined;
+	fulltextSearch?: {
+		provider?: FulltextSearchProvider;
+	};
 	meilisearch: {
 		host: string;
 		port: string;
@@ -184,6 +190,8 @@ export type Config = {
 	pidFile: string;
 };
 
+export type FulltextSearchProvider = 'sqlLike' | 'sqlPgroonga' | 'meilisearch';
+
 const _filename = fileURLToPath(import.meta.url);
 const _dirname = dirname(_filename);
 
@@ -252,6 +260,7 @@ export function loadConfig(): Config {
 		db: { ...config.db, db: dbDb, user: dbUser, pass: dbPass },
 		dbReplications: config.dbReplications,
 		dbSlaves: config.dbSlaves,
+		fulltextSearch: config.fulltextSearch,
 		meilisearch: config.meilisearch,
 		redis,
 		redisForPubsub: config.redisForPubsub ? convertRedisOptions(config.redisForPubsub, host) : redis,

packages/backend/src/core/HttpRequestService.ts

@@ -54,7 +54,7 @@ class HttpRequestServiceAgent extends http.Agent {
 				}
 			});
 		return socket;
-	};
+	}
 
 	@bindThis
 	private isPrivateIp(ip: string): boolean {
@@ -93,7 +93,7 @@ class HttpsRequestServiceAgent extends https.Agent {
 				}
 			});
 		return socket;
-	};
+	}
 
 	@bindThis
 	private isPrivateIp(ip: string): boolean {

packages/backend/src/core/RemoteUserResolveService.ts

@@ -54,7 +54,7 @@ export class RemoteUserResolveService {
 			}) as MiLocalUser;
 		}
 
-		host = this.utilityService.punyHost(host);
+		host = this.utilityService.toPuny(host);
 
 		if (host === this.utilityService.toPuny(this.config.host)) {
 			this.logger.info(`return local user: ${usernameLower}`);

packages/backend/src/core/SearchService.ts

@@ -6,16 +6,17 @@
 import { Inject, Injectable } from '@nestjs/common';
 import { In } from 'typeorm';
 import { DI } from '@/di-symbols.js';
-import type { Config } from '@/config.js';
+import { type Config, FulltextSearchProvider } from '@/config.js';
 import { bindThis } from '@/decorators.js';
 import { MiNote } from '@/models/Note.js';
-import { MiUser } from '@/models/_.js';
 import type { NotesRepository } from '@/models/_.js';
+import { MiUser } from '@/models/_.js';
 import { sqlLikeEscape } from '@/misc/sql-like-escape.js';
 import { isUserRelated } from '@/misc/is-user-related.js';
 import { CacheService } from '@/core/CacheService.js';
 import { QueryService } from '@/core/QueryService.js';
 import { IdService } from '@/core/IdService.js';
+import { LoggerService } from '@/core/LoggerService.js';
 import type { Index, MeiliSearch } from 'meilisearch';
 
 type K = string;
@@ -33,6 +34,18 @@ type Q =
 	{ op: 'or', qs: Q[] } |
 	{ op: 'not', q: Q };
 
+export type SearchOpts = {
+	userId?: MiNote['userId'] | null;
+	channelId?: MiNote['channelId'] | null;
+	host?: string | null;
+};
+
+export type SearchPagination = {
+	untilId?: MiNote['id'];
+	sinceId?: MiNote['id'];
+	limit: number;
+};
+
 function compileValue(value: V): string {
 	if (typeof value === 'string') {
 		return `'${value}'`; // TODO: escape
@@ -64,7 +77,8 @@ function compileQuery(q: Q): string {
 @Injectable()
 export class SearchService {
 	private readonly meilisearchIndexScope: 'local' | 'global' | string[] = 'local';
-	private meilisearchNoteIndex: Index | null = null;
+	private readonly meilisearchNoteIndex: Index | null = null;
+	private readonly provider: FulltextSearchProvider;
 
 	constructor(
 		@Inject(DI.config)
@@ -79,6 +93,7 @@ export class SearchService {
 		private cacheService: CacheService,
 		private queryService: QueryService,
 		private idService: IdService,
+		private loggerService: LoggerService,
 	) {
 		if (meilisearch) {
 			this.meilisearchNoteIndex = meilisearch.index(`${config.meilisearch!.index}---notes`);
@@ -109,14 +124,17 @@ export class SearchService {
 		if (config.meilisearch?.scope) {
 			this.meilisearchIndexScope = config.meilisearch.scope;
 		}
+
+		this.provider = config.fulltextSearch?.provider ?? 'sqlLike';
+		this.loggerService.getLogger('SearchService').info(`-- Provider: ${this.provider}`);
 	}
 
 	@bindThis
 	public async indexNote(note: MiNote): Promise<void> {
+		if (!this.meilisearch) return;
 		if (note.text == null && note.cw == null) return;
 		if (!['home', 'public'].includes(note.visibility)) return;
 
-		if (this.meilisearch) {
 		switch (this.meilisearchIndexScope) {
 			case 'global':
 				break;
@@ -145,67 +163,47 @@ export class SearchService {
 			primaryKey: 'id',
 		});
 	}
-	}
 
 	@bindThis
 	public async unindexNote(note: MiNote): Promise<void> {
+		if (!this.meilisearch) return;
 		if (!['home', 'public'].includes(note.visibility)) return;
 
-		if (this.meilisearch) {
-			this.meilisearchNoteIndex!.deleteDocument(note.id);
+		await this.meilisearchNoteIndex?.deleteDocument(note.id);
+	}
+
+	@bindThis
+	public async searchNote(
+		q: string,
+		me: MiUser | null,
+		opts: SearchOpts,
+		pagination: SearchPagination,
+	): Promise<MiNote[]> {
+		switch (this.provider) {
+			case 'sqlLike':
+			case 'sqlPgroonga': {
+				// ほとんど内容に差がないのでsqlLikeとsqlPgroongaを同じ処理にしている.
+				// 今後の拡張で差が出る用であれば関数を分ける.
+				return this.searchNoteByLike(q, me, opts, pagination);
+			}
+			case 'meilisearch': {
+				return this.searchNoteByMeiliSearch(q, me, opts, pagination);
+			}
+			default: {
+				// eslint-disable-next-line @typescript-eslint/no-unused-vars
+				const typeCheck: never = this.provider;
+				return [];
+			}
 		}
 	}
 
 	@bindThis
-	public async searchNote(q: string, me: MiUser | null, opts: {
-		userId?: MiNote['userId'] | null;
-		channelId?: MiNote['channelId'] | null;
-		host?: string | null;
-	}, pagination: {
-		untilId?: MiNote['id'];
-		sinceId?: MiNote['id'];
-		limit?: number;
-	}): Promise<MiNote[]> {
-		if (this.meilisearch) {
-			const filter: Q = {
-				op: 'and',
-				qs: [],
-			};
-			if (pagination.untilId) filter.qs.push({ op: '<', k: 'createdAt', v: this.idService.parse(pagination.untilId).date.getTime() });
-			if (pagination.sinceId) filter.qs.push({ op: '>', k: 'createdAt', v: this.idService.parse(pagination.sinceId).date.getTime() });
-			if (opts.userId) filter.qs.push({ op: '=', k: 'userId', v: opts.userId });
-			if (opts.channelId) filter.qs.push({ op: '=', k: 'channelId', v: opts.channelId });
-			if (opts.host) {
-				if (opts.host === '.') {
-					filter.qs.push({ op: 'is null', k: 'userHost' });
-				} else {
-					filter.qs.push({ op: '=', k: 'userHost', v: opts.host });
-				}
-			}
-			const res = await this.meilisearchNoteIndex!.search(q, {
-				sort: ['createdAt:desc'],
-				matchingStrategy: 'all',
-				attributesToRetrieve: ['id', 'createdAt'],
-				filter: compileQuery(filter),
-				limit: pagination.limit,
-			});
-			if (res.hits.length === 0) return [];
-			const [
-				userIdsWhoMeMuting,
-				userIdsWhoBlockingMe,
-			] = me ? await Promise.all([
-				this.cacheService.userMutingsCache.fetch(me.id),
-				this.cacheService.userBlockedCache.fetch(me.id),
-			]) : [new Set<string>(), new Set<string>()];
-			const notes = (await this.notesRepository.findBy({
-				id: In(res.hits.map(x => x.id)),
-			})).filter(note => {
-				if (me && isUserRelated(note, userIdsWhoBlockingMe)) return false;
-				if (me && isUserRelated(note, userIdsWhoMeMuting)) return false;
-				return true;
-			});
-			return notes.sort((a, b) => a.id > b.id ? -1 : 1);
-		} else {
+	private async searchNoteByLike(
+		q: string,
+		me: MiUser | null,
+		opts: SearchOpts,
+		pagination: SearchPagination,
+	): Promise<MiNote[]> {
 		const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), pagination.sinceId, pagination.untilId);
 
 		if (opts.userId) {
@@ -215,13 +213,18 @@ export class SearchService {
 		}
 
 		query
-				.andWhere('note.text ILIKE :q', { q: `%${ sqlLikeEscape(q) }%` })
 			.innerJoinAndSelect('note.user', 'user')
 			.leftJoinAndSelect('note.reply', 'reply')
 			.leftJoinAndSelect('note.renote', 'renote')
 			.leftJoinAndSelect('reply.user', 'replyUser')
 			.leftJoinAndSelect('renote.user', 'renoteUser');
 
+		if (this.config.fulltextSearch?.provider === 'sqlPgroonga') {
+			query.andWhere('note.text &@ :q', { q });
+		} else {
+			query.andWhere('note.text ILIKE :q', { q: `%${sqlLikeEscape(q)}%` });
+		}
+
 		if (opts.host) {
 			if (opts.host === '.') {
 				query.andWhere('user.host IS NULL');
@@ -234,7 +237,72 @@ export class SearchService {
 		if (me) this.queryService.generateMutedUserQuery(query, me);
 		if (me) this.queryService.generateBlockedUserQuery(query, me);
 
-			return await query.limit(pagination.limit).getMany();
+		return query.limit(pagination.limit).getMany();
+	}
+
+	@bindThis
+	private async searchNoteByMeiliSearch(
+		q: string,
+		me: MiUser | null,
+		opts: SearchOpts,
+		pagination: SearchPagination,
+	): Promise<MiNote[]> {
+		if (!this.meilisearch || !this.meilisearchNoteIndex) {
+			throw new Error('MeiliSearch is not available');
+		}
+
+		const filter: Q = {
+			op: 'and',
+			qs: [],
+		};
+		if (pagination.untilId) filter.qs.push({
+			op: '<',
+			k: 'createdAt',
+			v: this.idService.parse(pagination.untilId).date.getTime(),
+		});
+		if (pagination.sinceId) filter.qs.push({
+			op: '>',
+			k: 'createdAt',
+			v: this.idService.parse(pagination.sinceId).date.getTime(),
+		});
+		if (opts.userId) filter.qs.push({ op: '=', k: 'userId', v: opts.userId });
+		if (opts.channelId) filter.qs.push({ op: '=', k: 'channelId', v: opts.channelId });
+		if (opts.host) {
+			if (opts.host === '.') {
+				filter.qs.push({ op: 'is null', k: 'userHost' });
+			} else {
+				filter.qs.push({ op: '=', k: 'userHost', v: opts.host });
 			}
 		}
+
+		const res = await this.meilisearchNoteIndex.search(q, {
+			sort: ['createdAt:desc'],
+			matchingStrategy: 'all',
+			attributesToRetrieve: ['id', 'createdAt'],
+			filter: compileQuery(filter),
+			limit: pagination.limit,
+		});
+		if (res.hits.length === 0) {
+			return [];
+		}
+
+		const [
+			userIdsWhoMeMuting,
+			userIdsWhoBlockingMe,
+		] = me
+			? await Promise.all([
+				this.cacheService.userMutingsCache.fetch(me.id),
+				this.cacheService.userBlockedCache.fetch(me.id),
+			])
+			: [new Set<string>(), new Set<string>()];
+		const notes = (await this.notesRepository.findBy({
+			id: In(res.hits.map(x => x.id)),
+		})).filter(note => {
+			if (me && isUserRelated(note, userIdsWhoBlockingMe)) return false;
+			if (me && isUserRelated(note, userIdsWhoMeMuting)) return false;
+			return true;
+		});
+
+		return notes.sort((a, b) => a.id > b.id ? -1 : 1);
+	}
 }

packages/backend/src/core/WebAuthnService.ts

@@ -246,14 +246,12 @@ export class WebAuthnService {
 
 	@bindThis
 	public async verifyAuthentication(userId: MiUser['id'], response: AuthenticationResponseJSON): Promise<boolean> {
-		const challenge = await this.redisClient.get(`webauthn:challenge:${userId}`);
+		const challenge = await this.redisClient.getdel(`webauthn:challenge:${userId}`);
 
 		if (!challenge) {
 			throw new IdentifiableError('2d16e51c-007b-4edd-afd2-f7dd02c947f6', 'challenge not found');
 		}
 
-		await this.redisClient.del(`webauthn:challenge:${userId}`);
-
 		const key = await this.userSecurityKeysRepository.findOneBy({
 			id: response.id,
 			userId: userId,

packages/backend/src/core/activitypub/ApInboxService.ts

@@ -130,6 +130,7 @@ export class ApInboxService {
 		if (actor.uri) {
 			if (actor.lastFetchedAt == null || Date.now() - actor.lastFetchedAt.getTime() > 1000 * 60 * 60 * 24) {
 				setImmediate(() => {
+					// 同一ユーザーの情報を再度処理するので、使用済みのresolverを再利用してはいけない
 					this.apPersonService.updatePerson(actor.uri);
 				});
 			}

packages/backend/src/core/activitypub/models/ApPersonService.ts

@@ -163,7 +163,9 @@ export class ApPersonService implements OnModuleInit {
 		}
 
 		for (const collection of ['outbox', 'followers', 'following'] as (keyof IActor)[]) {
-			const collectionUri = getApId((x as IActor)[collection]);
+			const xCollection = (x as IActor)[collection];
+			if (xCollection != null) {
+				const collectionUri = getApId(xCollection);
 				if (typeof collectionUri === 'string' && collectionUri.length > 0) {
 					if (this.utilityService.punyHost(collectionUri) !== expectHost) {
 						throw new Error(`invalid Actor: ${collection} has different host`);
@@ -172,6 +174,7 @@ export class ApPersonService implements OnModuleInit {
 					throw new Error(`invalid Actor: wrong ${collection}`);
 				}
 			}
+		}
 
 		if (!(typeof x.preferredUsername === 'string' && x.preferredUsername.length > 0 && x.preferredUsername.length <= 128 && /^\w([\w-.]*\w)?$/.test(x.preferredUsername))) {
 			throw new Error('invalid Actor: wrong username');

packages/backend/src/server/api/endpoints/ap/show.ts

@@ -137,10 +137,11 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 			if (local != null) return local;
 		}
 
+		// 同一ユーザーの情報を再度処理するので、使用済みのresolverを再利用してはいけない
 		return await this.mergePack(
 			me,
 			isActor(object) ? await this.apPersonService.createPerson(getApId(object)) : null,
-			isPost(object) ? await this.apNoteService.createNote(getApId(object), undefined, true) : null,
+			isPost(object) ? await this.apNoteService.createNote(getApId(object), undefined, undefined, true) : null,
 		);
 	}
 

packages/backend/test-federation/.config/example.default.yml

@@ -19,7 +19,6 @@ proxyBypassHosts:
   - challenges.cloudflare.com
 proxyRemoteFiles: true
 signToActivityPubGet: true
-allowedPrivateNetworks: [
-  '127.0.0.1/32',
-  '172.20.0.0/16'
-]
+allowedPrivateNetworks:
+  - 127.0.0.1/32
+  - 172.20.0.0/16

packages/backend/test/unit/activitypub.ts

@@ -176,7 +176,7 @@ describe('ActivityPub', () => {
 			resolver.register(actor.id, actor);
 			resolver.register(post.id, post);
 
-			const note = await noteService.createNote(post.id, resolver, true);
+			const note = await noteService.createNote(post.id, undefined, resolver, true);
 
 			assert.deepStrictEqual(note?.uri, post.id);
 			assert.deepStrictEqual(note.visibility, 'public');
@@ -336,7 +336,7 @@ describe('ActivityPub', () => {
 			resolver.register(actor.featured, featured);
 			resolver.register(firstNote.id, firstNote);
 
-			const note = await noteService.createNote(firstNote.id as string, resolver);
+			const note = await noteService.createNote(firstNote.id as string, undefined, resolver);
 			assert.strictEqual(note?.uri, firstNote.id);
 		});
 	});

packages/misskey-js/package.json

@@ -1,7 +1,7 @@
 {
 	"type": "module",
 	"name": "misskey-js",
-	"version": "2024.11.0-alpha.2",
+	"version": "2024.11.0-alpha.3",
 	"description": "Misskey SDK for JavaScript",
 	"license": "MIT",
 	"main": "./built/index.js",