Merge add32485fe into 9871035597

2024-11-21 16:46:42 +09:00 · 2024-11-21 16:46:42 +09:00 · d99f3eec90
parent 9871035597 add32485fe
commit d99f3eec90
6 changed files with 236 additions and 114 deletions
--- a/.config/docker_example.yml
+++ b/.config/docker_example.yml
@ -114,9 +114,27 @@ redis:
 #  #prefix: example-prefix
 #  #db: 1

-#   ┌───────────────────────────┐
-#───┘ MeiliSearch configuration └─────────────────────────────
+#   ┌───────────────────────────────┐
+#───┘ Fulltext search configuration └─────────────────────────────

+# These are the setting items for the full-text search provider.
+fulltextSearch:
+  # You can select the ID generation method.
+  # - sqlLike (default)
+  #   Use SQL-like search.
+  #   This is a standard feature of PostgreSQL, so no special extensions are required.
+  # - sqlPgroonga
+  #   Use pgroonga.
+  #   You need to install pgroonga and configure it as a PostgreSQL extension.
+  #   In addition to the above, you need to create a pgroonga index on the text column of the note table.
+  #   see: https://pgroonga.github.io/tutorial/
+  # - meilisearch
+  #   Use Meilisearch.
+  #   You need to install Meilisearch and configure.
+  provider: sqlLike
+
+# For Meilisearch settings.
+# If you select "meilisearch" for "fulltextSearch.provider", it must be set.
 # You can set scope to local (default value) or global
 # (include notes from remote).

--- a/.config/example.yml
+++ b/.config/example.yml
@ -196,9 +196,27 @@ redis:
 #  # You can specify more ioredis options...
 #  #username: example-username

-#   ┌───────────────────────────┐
-#───┘ MeiliSearch configuration └─────────────────────────────
+#   ┌───────────────────────────────┐
+#───┘ Fulltext search configuration └─────────────────────────────

+# These are the setting items for the full-text search provider.
+fulltextSearch:
+  # You can select the ID generation method.
+  # - sqlLike (default)
+  #   Use SQL-like search.
+  #   This is a standard feature of PostgreSQL, so no special extensions are required.
+  # - sqlPgroonga
+  #   Use pgroonga.
+  #   You need to install pgroonga and configure it as a PostgreSQL extension.
+  #   In addition to the above, you need to create a pgroonga index on the text column of the note table.
+  #   see: https://pgroonga.github.io/tutorial/
+  # - meilisearch
+  #   Use Meilisearch.
+  #   You need to install Meilisearch and configure.
+  provider: sqlLike
+
+# For Meilisearch settings.
+# If you select "meilisearch" for "fulltextSearch.provider", it must be set.
 # You can set scope to local (default value) or global
 # (include notes from remote).

--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@ -1,6 +1,10 @@
 ## 2024.11.0

 ### Note
+- [重要] ノート検索プロバイダの追加に伴い、configファイル（default.ymlなど）の構成が少し変わります.
+  - 新しい設定項目"fulltextSearch.provider"が追加されました. sqlLike, sqlPgroonga, meilisearchのいずれかを設定出来ます.
+  - すでにMeilisearchをお使いの場合、 **"fulltextSearch.provider"を"meilisearch"に設定する必要** があります.
+  - 詳細は #14730 および `.config/example.yml` または `.config/docker_example.yml`の'Fulltext search configuration'をご参照願います.
 - Node.js 20.xは非推奨になりました。Node.js 22.x (LTS)の利用を推奨します。
  - なお、Node.js 23.xは対応していません。
 - DockerのNode.jsが22.11.0に更新されました
@ -51,6 +55,7 @@
  (Based on https://activitypub.software/TransFem-org/Sharkey/-/merge_requests/588)  
  (Cherry-picked from https://activitypub.software/TransFem-org/Sharkey/-/merge_requests/715)
 - Enhance: リモートユーザーの照会をオリジナルにリダイレクトするように
+- Enhance: ノート検索の選択肢としてpgroongaに対応 ( #14730 )
 - Fix: sharedInboxが無いActorに紐づくリモートユーザーを照会できない
 - Fix: Aproving request from GtS appears with some delay
 - Fix: フォロワーへのメッセージの絵文字をemojisに含めるように
--- a/packages/backend/src/GlobalModule.ts
+++ b/packages/backend/src/GlobalModule.ts
@ -7,14 +7,14 @@ import { Global, Inject, Module } from '@nestjs/common';
 import * as Redis from 'ioredis';
 import { DataSource } from 'typeorm';
 import { MeiliSearch } from 'meilisearch';
+import { MiMeta } from '@/models/Meta.js';
 import { DI } from './di-symbols.js';
 import { Config, loadConfig } from './config.js';
 import { createPostgresDataSource } from './postgres.js';
 import { RepositoryModule } from './models/RepositoryModule.js';
 import { allSettled } from './misc/promise-tracker.js';
-import type { Provider, OnApplicationShutdown } from '@nestjs/common';
-import { MiMeta } from '@/models/Meta.js';
 import { GlobalEvents } from './core/GlobalEventService.js';
+import type { Provider, OnApplicationShutdown } from '@nestjs/common';

 const $config: Provider = {
 	provide: DI.config,
@ -33,7 +33,11 @@ const $db: Provider = {
 const $meilisearch: Provider = {
 	provide: DI.meilisearch,
 	useFactory: (config: Config) => {
-		if (config.meilisearch) {
+		if (config.fulltextSearch?.provider === 'meilisearch') {
+			if (!config.meilisearch) {
+				throw new Error('MeiliSearch is enabled but no configuration is provided');
+			}
+
 			return new MeiliSearch({
 				host: `${config.meilisearch.ssl ? 'https' : 'http'}://${config.meilisearch.host}:${config.meilisearch.port}`,
 				apiKey: config.meilisearch.apiKey,
--- a/packages/backend/src/config.ts
+++ b/packages/backend/src/config.ts
@ -50,6 +50,9 @@ type Source = {
 	redisForJobQueue?: RedisOptionsSource;
 	redisForTimelines?: RedisOptionsSource;
 	redisForReactions?: RedisOptionsSource;
+	fulltextSearch?: {
+		provider?: FulltextSearchProvider;
+	};
 	meilisearch?: {
 		host: string;
 		port: string;
@ -124,6 +127,9 @@ export type Config = {
 		user: string;
 		pass: string;
 	}[] | undefined;
+	fulltextSearch?: {
+		provider?: FulltextSearchProvider;
+	};
 	meilisearch: {
 		host: string;
 		port: string;
@ -184,6 +190,8 @@ export type Config = {
 	pidFile: string;
 };

+export type FulltextSearchProvider = 'sqlLike' | 'sqlPgroonga' | 'meilisearch';
+
 const _filename = fileURLToPath(import.meta.url);
 const _dirname = dirname(_filename);

@ -252,6 +260,7 @@ export function loadConfig(): Config {
 		db: { ...config.db, db: dbDb, user: dbUser, pass: dbPass },
 		dbReplications: config.dbReplications,
 		dbSlaves: config.dbSlaves,
+		fulltextSearch: config.fulltextSearch,
 		meilisearch: config.meilisearch,
 		redis,
 		redisForPubsub: config.redisForPubsub ? convertRedisOptions(config.redisForPubsub, host) : redis,
--- a/packages/backend/src/core/SearchService.ts
+++ b/packages/backend/src/core/SearchService.ts
@ -6,16 +6,17 @@
 import { Inject, Injectable } from '@nestjs/common';
 import { In } from 'typeorm';
 import { DI } from '@/di-symbols.js';
-import type { Config } from '@/config.js';
+import { type Config, FulltextSearchProvider } from '@/config.js';
 import { bindThis } from '@/decorators.js';
 import { MiNote } from '@/models/Note.js';
-import { MiUser } from '@/models/_.js';
 import type { NotesRepository } from '@/models/_.js';
+import { MiUser } from '@/models/_.js';
 import { sqlLikeEscape } from '@/misc/sql-like-escape.js';
 import { isUserRelated } from '@/misc/is-user-related.js';
 import { CacheService } from '@/core/CacheService.js';
 import { QueryService } from '@/core/QueryService.js';
 import { IdService } from '@/core/IdService.js';
+import { LoggerService } from '@/core/LoggerService.js';
 import type { Index, MeiliSearch } from 'meilisearch';

 type K = string;
@ -27,12 +28,24 @@ type Q =
 	{ op: '<', k: K, v: number } |
 	{ op: '>=', k: K, v: number } |
 	{ op: '<=', k: K, v: number } |
-	{ op: 'is null', k: K} |
-	{ op: 'is not null', k: K} |
+	{ op: 'is null', k: K } |
+	{ op: 'is not null', k: K } |
 	{ op: 'and', qs: Q[] } |
 	{ op: 'or', qs: Q[] } |
 	{ op: 'not', q: Q };

+export type SearchOpts = {
+	userId?: MiNote['userId'] | null;
+	channelId?: MiNote['channelId'] | null;
+	host?: string | null;
+};
+
+export type SearchPagination = {
+	untilId?: MiNote['id'];
+	sinceId?: MiNote['id'];
+	limit: number;
+};
+
 function compileValue(value: V): string {
 	if (typeof value === 'string') {
 		return `'${value}'`; // TODO: escape
@ -64,7 +77,8 @@ function compileQuery(q: Q): string {
@Injectable()
 export class SearchService {
 	private readonly meilisearchIndexScope: 'local' | 'global' | string[] = 'local';
-	private meilisearchNoteIndex: Index | null = null;
+	private readonly meilisearchNoteIndex: Index | null = null;
+	private readonly provider: FulltextSearchProvider;

 	constructor(
 		@Inject(DI.config)
@ -79,6 +93,7 @@ export class SearchService {
 		private cacheService: CacheService,
 		private queryService: QueryService,
 		private idService: IdService,
+		private loggerService: LoggerService,
 	) {
 		if (meilisearch) {
 			this.meilisearchNoteIndex = meilisearch.index(`${config.meilisearch!.index}---notes`);
@ -109,132 +124,185 @@ export class SearchService {
 		if (config.meilisearch?.scope) {
 			this.meilisearchIndexScope = config.meilisearch.scope;
 		}
+
+		this.provider = config.fulltextSearch?.provider ?? 'sqlLike';
+		this.loggerService.getLogger('SearchService').info(`-- Provider: ${this.provider}`);
 	}

 	@bindThis
 	public async indexNote(note: MiNote): Promise<void> {
+		if (!this.meilisearch) return;
 		if (note.text == null && note.cw == null) return;
 		if (!['home', 'public'].includes(note.visibility)) return;

-		if (this.meilisearch) {
-			switch (this.meilisearchIndexScope) {
-				case 'global':
-					break;
+		switch (this.meilisearchIndexScope) {
+			case 'global':
+				break;

-				case 'local':
-					if (note.userHost == null) break;
-					return;
+			case 'local':
+				if (note.userHost == null) break;
+				return;

-				default: {
-					if (note.userHost == null) break;
-					if (this.meilisearchIndexScope.includes(note.userHost)) break;
-					return;
-				}
+			default: {
+				if (note.userHost == null) break;
+				if (this.meilisearchIndexScope.includes(note.userHost)) break;
+				return;
 			}
-
-			await this.meilisearchNoteIndex?.addDocuments([{
-				id: note.id,
-				createdAt: this.idService.parse(note.id).date.getTime(),
-				userId: note.userId,
-				userHost: note.userHost,
-				channelId: note.channelId,
-				cw: note.cw,
-				text: note.text,
-				tags: note.tags,
-			}], {
-				primaryKey: 'id',
-			});
 		}
+
+		await this.meilisearchNoteIndex?.addDocuments([{
+			id: note.id,
+			createdAt: this.idService.parse(note.id).date.getTime(),
+			userId: note.userId,
+			userHost: note.userHost,
+			channelId: note.channelId,
+			cw: note.cw,
+			text: note.text,
+			tags: note.tags,
+		}], {
+			primaryKey: 'id',
+		});
 	}

 	@bindThis
 	public async unindexNote(note: MiNote): Promise<void> {
+		if (!this.meilisearch) return;
 		if (!['home', 'public'].includes(note.visibility)) return;

-		if (this.meilisearch) {
-			this.meilisearchNoteIndex!.deleteDocument(note.id);
+		await this.meilisearchNoteIndex?.deleteDocument(note.id);
+	}
+
+	@bindThis
+	public async searchNote(
+		q: string,
+		me: MiUser | null,
+		opts: SearchOpts,
+		pagination: SearchPagination,
+	): Promise<MiNote[]> {
+		switch (this.provider) {
+			case 'sqlLike':
+			case 'sqlPgroonga': {
+				// ほとんど内容に差がないのでsqlLikeとsqlPgroongaを同じ処理にしている.
+				// 今後の拡張で差が出る用であれば関数を分ける.
+				return this.searchNoteByLike(q, me, opts, pagination);
+			}
+			case 'meilisearch': {
+				return this.searchNoteByMeiliSearch(q, me, opts, pagination);
+			}
+			default: {
+				// eslint-disable-next-line @typescript-eslint/no-unused-vars
+				const typeCheck: never = this.provider;
+				return [];
+			}
 		}
 	}

 	@bindThis
-	public async searchNote(q: string, me: MiUser | null, opts: {
-		userId?: MiNote['userId'] | null;
-		channelId?: MiNote['channelId'] | null;
-		host?: string | null;
-	}, pagination: {
-		untilId?: MiNote['id'];
-		sinceId?: MiNote['id'];
-		limit?: number;
-	}): Promise<MiNote[]> {
-		if (this.meilisearch) {
-			const filter: Q = {
-				op: 'and',
-				qs: [],
-			};
-			if (pagination.untilId) filter.qs.push({ op: '<', k: 'createdAt', v: this.idService.parse(pagination.untilId).date.getTime() });
-			if (pagination.sinceId) filter.qs.push({ op: '>', k: 'createdAt', v: this.idService.parse(pagination.sinceId).date.getTime() });
-			if (opts.userId) filter.qs.push({ op: '=', k: 'userId', v: opts.userId });
-			if (opts.channelId) filter.qs.push({ op: '=', k: 'channelId', v: opts.channelId });
-			if (opts.host) {
-				if (opts.host === '.') {
-					filter.qs.push({ op: 'is null', k: 'userHost' });
-				} else {
-					filter.qs.push({ op: '=', k: 'userHost', v: opts.host });
-				}
+	private async searchNoteByLike(
+		q: string,
+		me: MiUser | null,
+		opts: SearchOpts,
+		pagination: SearchPagination,
+	): Promise<MiNote[]> {
+		const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), pagination.sinceId, pagination.untilId);
+
+		if (opts.userId) {
+			query.andWhere('note.userId = :userId', { userId: opts.userId });
+		} else if (opts.channelId) {
+			query.andWhere('note.channelId = :channelId', { channelId: opts.channelId });
+		}
+
+		query
+			.innerJoinAndSelect('note.user', 'user')
+			.leftJoinAndSelect('note.reply', 'reply')
+			.leftJoinAndSelect('note.renote', 'renote')
+			.leftJoinAndSelect('reply.user', 'replyUser')
+			.leftJoinAndSelect('renote.user', 'renoteUser');
+
+		if (this.config.fulltextSearch?.provider === 'sqlPgroonga') {
+			query.andWhere('note.text &@ :q', { q });
+		} else {
+			query.andWhere('note.text ILIKE :q', { q: `%${sqlLikeEscape(q)}%` });
+		}
+
+		if (opts.host) {
+			if (opts.host === '.') {
+				query.andWhere('user.host IS NULL');
+			} else {
+				query.andWhere('user.host = :host', { host: opts.host });
 			}
-			const res = await this.meilisearchNoteIndex!.search(q, {
-				sort: ['createdAt:desc'],
-				matchingStrategy: 'all',
-				attributesToRetrieve: ['id', 'createdAt'],
-				filter: compileQuery(filter),
-				limit: pagination.limit,
-			});
-			if (res.hits.length === 0) return [];
-			const [
-				userIdsWhoMeMuting,
-				userIdsWhoBlockingMe,
-			] = me ? await Promise.all([
+		}
+
+		this.queryService.generateVisibilityQuery(query, me);
+		if (me) this.queryService.generateMutedUserQuery(query, me);
+		if (me) this.queryService.generateBlockedUserQuery(query, me);
+
+		return query.limit(pagination.limit).getMany();
+	}
+
+	@bindThis
+	private async searchNoteByMeiliSearch(
+		q: string,
+		me: MiUser | null,
+		opts: SearchOpts,
+		pagination: SearchPagination,
+	): Promise<MiNote[]> {
+		if (!this.meilisearch || !this.meilisearchNoteIndex) {
+			throw new Error('MeiliSearch is not available');
+		}
+
+		const filter: Q = {
+			op: 'and',
+			qs: [],
+		};
+		if (pagination.untilId) filter.qs.push({
+			op: '<',
+			k: 'createdAt',
+			v: this.idService.parse(pagination.untilId).date.getTime(),
+		});
+		if (pagination.sinceId) filter.qs.push({
+			op: '>',
+			k: 'createdAt',
+			v: this.idService.parse(pagination.sinceId).date.getTime(),
+		});
+		if (opts.userId) filter.qs.push({ op: '=', k: 'userId', v: opts.userId });
+		if (opts.channelId) filter.qs.push({ op: '=', k: 'channelId', v: opts.channelId });
+		if (opts.host) {
+			if (opts.host === '.') {
+				filter.qs.push({ op: 'is null', k: 'userHost' });
+			} else {
+				filter.qs.push({ op: '=', k: 'userHost', v: opts.host });
+			}
+		}
+
+		const res = await this.meilisearchNoteIndex.search(q, {
+			sort: ['createdAt:desc'],
+			matchingStrategy: 'all',
+			attributesToRetrieve: ['id', 'createdAt'],
+			filter: compileQuery(filter),
+			limit: pagination.limit,
+		});
+		if (res.hits.length === 0) {
+			return [];
+		}
+
+		const [
+			userIdsWhoMeMuting,
+			userIdsWhoBlockingMe,
+		] = me
+			? await Promise.all([
 				this.cacheService.userMutingsCache.fetch(me.id),
 				this.cacheService.userBlockedCache.fetch(me.id),
-			]) : [new Set<string>(), new Set<string>()];
-			const notes = (await this.notesRepository.findBy({
-				id: In(res.hits.map(x => x.id)),
-			})).filter(note => {
-				if (me && isUserRelated(note, userIdsWhoBlockingMe)) return false;
-				if (me && isUserRelated(note, userIdsWhoMeMuting)) return false;
-				return true;
-			});
-			return notes.sort((a, b) => a.id > b.id ? -1 : 1);
-		} else {
-			const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), pagination.sinceId, pagination.untilId);
+			])
+			: [new Set<string>(), new Set<string>()];
+		const notes = (await this.notesRepository.findBy({
+			id: In(res.hits.map(x => x.id)),
+		})).filter(note => {
+			if (me && isUserRelated(note, userIdsWhoBlockingMe)) return false;
+			if (me && isUserRelated(note, userIdsWhoMeMuting)) return false;
+			return true;
+		});

-			if (opts.userId) {
-				query.andWhere('note.userId = :userId', { userId: opts.userId });
-			} else if (opts.channelId) {
-				query.andWhere('note.channelId = :channelId', { channelId: opts.channelId });
-			}
-
-			query
-				.andWhere('note.text ILIKE :q', { q: `%${ sqlLikeEscape(q) }%` })
-				.innerJoinAndSelect('note.user', 'user')
-				.leftJoinAndSelect('note.reply', 'reply')
-				.leftJoinAndSelect('note.renote', 'renote')
-				.leftJoinAndSelect('reply.user', 'replyUser')
-				.leftJoinAndSelect('renote.user', 'renoteUser');
-
-			if (opts.host) {
-				if (opts.host === '.') {
-					query.andWhere('user.host IS NULL');
-				} else {
-					query.andWhere('user.host = :host', { host: opts.host });
-				}
-			}
-
-			this.queryService.generateVisibilityQuery(query, me);
-			if (me) this.queryService.generateMutedUserQuery(query, me);
-			if (me) this.queryService.generateBlockedUserQuery(query, me);
-
-			return await query.limit(pagination.limit).getMany();
-		}
+		return notes.sort((a, b) => a.id > b.id ? -1 : 1);
 	}
 }