Merge add32485fe into 4603ab67bb

2024-11-20 06:44:10 -06:00 · 2024-11-20 06:44:10 -06:00 · a879d11802
parent 4603ab67bb add32485fe
commit a879d11802
6 changed files with 236 additions and 114 deletions
--- a/.config/docker_example.yml
+++ b/.config/docker_example.yml
@ -114,9 +114,27 @@ redis:
 #  #prefix: example-prefix
 #  #db: 1

-#   ┌───────────────────────────┐
-#───┘ MeiliSearch configuration └─────────────────────────────
+#   ┌───────────────────────────────┐
+#───┘ Fulltext search configuration └─────────────────────────────

+# These are the setting items for the full-text search provider.
+fulltextSearch:
+  # You can select the ID generation method.
+  # - sqlLike (default)
+  #   Use SQL-like search.
+  #   This is a standard feature of PostgreSQL, so no special extensions are required.
+  # - sqlPgroonga
+  #   Use pgroonga.
+  #   You need to install pgroonga and configure it as a PostgreSQL extension.
+  #   In addition to the above, you need to create a pgroonga index on the text column of the note table.
+  #   see: https://pgroonga.github.io/tutorial/
+  # - meilisearch
+  #   Use Meilisearch.
+  #   You need to install Meilisearch and configure.
+  provider: sqlLike
+
+# For Meilisearch settings.
+# If you select "meilisearch" for "fulltextSearch.provider", it must be set.
 # You can set scope to local (default value) or global
 # (include notes from remote).

--- a/.config/example.yml
+++ b/.config/example.yml
@ -196,9 +196,27 @@ redis:
 #  # You can specify more ioredis options...
 #  #username: example-username

-#   ┌───────────────────────────┐
-#───┘ MeiliSearch configuration └─────────────────────────────
+#   ┌───────────────────────────────┐
+#───┘ Fulltext search configuration └─────────────────────────────

+# These are the setting items for the full-text search provider.
+fulltextSearch:
+  # You can select the ID generation method.
+  # - sqlLike (default)
+  #   Use SQL-like search.
+  #   This is a standard feature of PostgreSQL, so no special extensions are required.
+  # - sqlPgroonga
+  #   Use pgroonga.
+  #   You need to install pgroonga and configure it as a PostgreSQL extension.
+  #   In addition to the above, you need to create a pgroonga index on the text column of the note table.
+  #   see: https://pgroonga.github.io/tutorial/
+  # - meilisearch
+  #   Use Meilisearch.
+  #   You need to install Meilisearch and configure.
+  provider: sqlLike
+
+# For Meilisearch settings.
+# If you select "meilisearch" for "fulltextSearch.provider", it must be set.
 # You can set scope to local (default value) or global
 # (include notes from remote).

--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@ -1,6 +1,10 @@
 ## 2024.11.0

 ### Note
+- [重要] ノート検索プロバイダの追加に伴い、configファイル（default.ymlなど）の構成が少し変わります.
+  - 新しい設定項目"fulltextSearch.provider"が追加されました. sqlLike, sqlPgroonga, meilisearchのいずれかを設定出来ます.
+  - すでにMeilisearchをお使いの場合、 **"fulltextSearch.provider"を"meilisearch"に設定する必要** があります.
+  - 詳細は #14730 および `.config/example.yml` または `.config/docker_example.yml`の'Fulltext search configuration'をご参照願います.
 - Node.js 20.xは非推奨になりました。Node.js 22.x (LTS)の利用を推奨します。
  - なお、Node.js 23.xは対応していません。
 - DockerのNode.jsが22.11.0に更新されました
@ -50,6 +54,7 @@
  (Based on https://activitypub.software/TransFem-org/Sharkey/-/merge_requests/588)  
  (Cherry-picked from https://activitypub.software/TransFem-org/Sharkey/-/merge_requests/715)
 - Enhance: リモートユーザーの照会をオリジナルにリダイレクトするように
+- Enhance: ノート検索の選択肢としてpgroongaに対応 ( #14730 )
 - Fix: sharedInboxが無いActorに紐づくリモートユーザーを照会できない
 - Fix: Aproving request from GtS appears with some delay
 - Fix: フォロワーへのメッセージの絵文字をemojisに含めるように
--- a/packages/backend/src/GlobalModule.ts
+++ b/packages/backend/src/GlobalModule.ts
@ -7,14 +7,14 @@ import { Global, Inject, Module } from '@nestjs/common';
 import * as Redis from 'ioredis';
 import { DataSource } from 'typeorm';
 import { MeiliSearch } from 'meilisearch';
+import { MiMeta } from '@/models/Meta.js';
 import { DI } from './di-symbols.js';
 import { Config, loadConfig } from './config.js';
 import { createPostgresDataSource } from './postgres.js';
 import { RepositoryModule } from './models/RepositoryModule.js';
 import { allSettled } from './misc/promise-tracker.js';
-import type { Provider, OnApplicationShutdown } from '@nestjs/common';
-import { MiMeta } from '@/models/Meta.js';
 import { GlobalEvents } from './core/GlobalEventService.js';
+import type { Provider, OnApplicationShutdown } from '@nestjs/common';

 const $config: Provider = {
 	provide: DI.config,
@ -33,7 +33,11 @@ const $db: Provider = {
 const $meilisearch: Provider = {
 	provide: DI.meilisearch,
 	useFactory: (config: Config) => {
-		if (config.meilisearch) {
+		if (config.fulltextSearch?.provider === 'meilisearch') {
+			if (!config.meilisearch) {
+				throw new Error('MeiliSearch is enabled but no configuration is provided');
+			}
+
 			return new MeiliSearch({
 				host: `${config.meilisearch.ssl ? 'https' : 'http'}://${config.meilisearch.host}:${config.meilisearch.port}`,
 				apiKey: config.meilisearch.apiKey,
--- a/packages/backend/src/config.ts
+++ b/packages/backend/src/config.ts
@ -50,6 +50,9 @@ type Source = {
 	redisForJobQueue?: RedisOptionsSource;
 	redisForTimelines?: RedisOptionsSource;
 	redisForReactions?: RedisOptionsSource;
+	fulltextSearch?: {
+		provider?: FulltextSearchProvider;
+	};
 	meilisearch?: {
 		host: string;
 		port: string;
@ -124,6 +127,9 @@ export type Config = {
 		user: string;
 		pass: string;
 	}[] | undefined;
+	fulltextSearch?: {
+		provider?: FulltextSearchProvider;
+	};
 	meilisearch: {
 		host: string;
 		port: string;
@ -184,6 +190,8 @@ export type Config = {
 	pidFile: string;
 };

+export type FulltextSearchProvider = 'sqlLike' | 'sqlPgroonga' | 'meilisearch';
+
 const _filename = fileURLToPath(import.meta.url);
 const _dirname = dirname(_filename);

@ -252,6 +260,7 @@ export function loadConfig(): Config {
 		db: { ...config.db, db: dbDb, user: dbUser, pass: dbPass },
 		dbReplications: config.dbReplications,
 		dbSlaves: config.dbSlaves,
+		fulltextSearch: config.fulltextSearch,
 		meilisearch: config.meilisearch,
 		redis,
 		redisForPubsub: config.redisForPubsub ? convertRedisOptions(config.redisForPubsub, host) : redis,
--- a/packages/backend/src/core/SearchService.ts
+++ b/packages/backend/src/core/SearchService.ts
@ -6,16 +6,17 @@
 import { Inject, Injectable } from '@nestjs/common';
 import { In } from 'typeorm';
 import { DI } from '@/di-symbols.js';
-import type { Config } from '@/config.js';
+import { type Config, FulltextSearchProvider } from '@/config.js';
 import { bindThis } from '@/decorators.js';
 import { MiNote } from '@/models/Note.js';
-import { MiUser } from '@/models/_.js';
 import type { NotesRepository } from '@/models/_.js';
+import { MiUser } from '@/models/_.js';
 import { sqlLikeEscape } from '@/misc/sql-like-escape.js';
 import { isUserRelated } from '@/misc/is-user-related.js';
 import { CacheService } from '@/core/CacheService.js';
 import { QueryService } from '@/core/QueryService.js';
 import { IdService } from '@/core/IdService.js';
+import { LoggerService } from '@/core/LoggerService.js';
 import type { Index, MeiliSearch } from 'meilisearch';

 type K = string;
@ -33,6 +34,18 @@ type Q =
 	{ op: 'or', qs: Q[] } |
 	{ op: 'not', q: Q };

+export type SearchOpts = {
+	userId?: MiNote['userId'] | null;
+	channelId?: MiNote['channelId'] | null;
+	host?: string | null;
+};
+
+export type SearchPagination = {
+	untilId?: MiNote['id'];
+	sinceId?: MiNote['id'];
+	limit: number;
+};
+
 function compileValue(value: V): string {
 	if (typeof value === 'string') {
 		return `'${value}'`; // TODO: escape
@ -64,7 +77,8 @@ function compileQuery(q: Q): string {
@Injectable()
 export class SearchService {
 	private readonly meilisearchIndexScope: 'local' | 'global' | string[] = 'local';
-	private meilisearchNoteIndex: Index | null = null;
+	private readonly meilisearchNoteIndex: Index | null = null;
+	private readonly provider: FulltextSearchProvider;

 	constructor(
 		@Inject(DI.config)
@ -79,6 +93,7 @@ export class SearchService {
 		private cacheService: CacheService,
 		private queryService: QueryService,
 		private idService: IdService,
+		private loggerService: LoggerService,
 	) {
 		if (meilisearch) {
 			this.meilisearchNoteIndex = meilisearch.index(`${config.meilisearch!.index}---notes`);
@ -109,14 +124,17 @@ export class SearchService {
 		if (config.meilisearch?.scope) {
 			this.meilisearchIndexScope = config.meilisearch.scope;
 		}
+
+		this.provider = config.fulltextSearch?.provider ?? 'sqlLike';
+		this.loggerService.getLogger('SearchService').info(`-- Provider: ${this.provider}`);
 	}

 	@bindThis
 	public async indexNote(note: MiNote): Promise<void> {
+		if (!this.meilisearch) return;
 		if (note.text == null && note.cw == null) return;
 		if (!['home', 'public'].includes(note.visibility)) return;

-		if (this.meilisearch) {
 		switch (this.meilisearchIndexScope) {
 			case 'global':
 				break;
@ -145,67 +163,47 @@ export class SearchService {
 			primaryKey: 'id',
 		});
 	}
-	}

 	@bindThis
 	public async unindexNote(note: MiNote): Promise<void> {
+		if (!this.meilisearch) return;
 		if (!['home', 'public'].includes(note.visibility)) return;

-		if (this.meilisearch) {
-			this.meilisearchNoteIndex!.deleteDocument(note.id);
+		await this.meilisearchNoteIndex?.deleteDocument(note.id);
+	}
+
+	@bindThis
+	public async searchNote(
+		q: string,
+		me: MiUser | null,
+		opts: SearchOpts,
+		pagination: SearchPagination,
+	): Promise<MiNote[]> {
+		switch (this.provider) {
+			case 'sqlLike':
+			case 'sqlPgroonga': {
+				// ほとんど内容に差がないのでsqlLikeとsqlPgroongaを同じ処理にしている.
+				// 今後の拡張で差が出る用であれば関数を分ける.
+				return this.searchNoteByLike(q, me, opts, pagination);
+			}
+			case 'meilisearch': {
+				return this.searchNoteByMeiliSearch(q, me, opts, pagination);
+			}
+			default: {
+				// eslint-disable-next-line @typescript-eslint/no-unused-vars
+				const typeCheck: never = this.provider;
+				return [];
+			}
 		}
 	}

 	@bindThis
-	public async searchNote(q: string, me: MiUser | null, opts: {
-		userId?: MiNote['userId'] | null;
-		channelId?: MiNote['channelId'] | null;
-		host?: string | null;
-	}, pagination: {
-		untilId?: MiNote['id'];
-		sinceId?: MiNote['id'];
-		limit?: number;
-	}): Promise<MiNote[]> {
-		if (this.meilisearch) {
-			const filter: Q = {
-				op: 'and',
-				qs: [],
-			};
-			if (pagination.untilId) filter.qs.push({ op: '<', k: 'createdAt', v: this.idService.parse(pagination.untilId).date.getTime() });
-			if (pagination.sinceId) filter.qs.push({ op: '>', k: 'createdAt', v: this.idService.parse(pagination.sinceId).date.getTime() });
-			if (opts.userId) filter.qs.push({ op: '=', k: 'userId', v: opts.userId });
-			if (opts.channelId) filter.qs.push({ op: '=', k: 'channelId', v: opts.channelId });
-			if (opts.host) {
-				if (opts.host === '.') {
-					filter.qs.push({ op: 'is null', k: 'userHost' });
-				} else {
-					filter.qs.push({ op: '=', k: 'userHost', v: opts.host });
-				}
-			}
-			const res = await this.meilisearchNoteIndex!.search(q, {
-				sort: ['createdAt:desc'],
-				matchingStrategy: 'all',
-				attributesToRetrieve: ['id', 'createdAt'],
-				filter: compileQuery(filter),
-				limit: pagination.limit,
-			});
-			if (res.hits.length === 0) return [];
-			const [
-				userIdsWhoMeMuting,
-				userIdsWhoBlockingMe,
-			] = me ? await Promise.all([
-				this.cacheService.userMutingsCache.fetch(me.id),
-				this.cacheService.userBlockedCache.fetch(me.id),
-			]) : [new Set<string>(), new Set<string>()];
-			const notes = (await this.notesRepository.findBy({
-				id: In(res.hits.map(x => x.id)),
-			})).filter(note => {
-				if (me && isUserRelated(note, userIdsWhoBlockingMe)) return false;
-				if (me && isUserRelated(note, userIdsWhoMeMuting)) return false;
-				return true;
-			});
-			return notes.sort((a, b) => a.id > b.id ? -1 : 1);
-		} else {
+	private async searchNoteByLike(
+		q: string,
+		me: MiUser | null,
+		opts: SearchOpts,
+		pagination: SearchPagination,
+	): Promise<MiNote[]> {
 		const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), pagination.sinceId, pagination.untilId);

 		if (opts.userId) {
@ -215,13 +213,18 @@ export class SearchService {
 		}

 		query
-				.andWhere('note.text ILIKE :q', { q: `%${ sqlLikeEscape(q) }%` })
 			.innerJoinAndSelect('note.user', 'user')
 			.leftJoinAndSelect('note.reply', 'reply')
 			.leftJoinAndSelect('note.renote', 'renote')
 			.leftJoinAndSelect('reply.user', 'replyUser')
 			.leftJoinAndSelect('renote.user', 'renoteUser');

+		if (this.config.fulltextSearch?.provider === 'sqlPgroonga') {
+			query.andWhere('note.text &@ :q', { q });
+		} else {
+			query.andWhere('note.text ILIKE :q', { q: `%${sqlLikeEscape(q)}%` });
+		}
+
 		if (opts.host) {
 			if (opts.host === '.') {
 				query.andWhere('user.host IS NULL');
@ -234,7 +237,72 @@ export class SearchService {
 		if (me) this.queryService.generateMutedUserQuery(query, me);
 		if (me) this.queryService.generateBlockedUserQuery(query, me);

-			return await query.limit(pagination.limit).getMany();
+		return query.limit(pagination.limit).getMany();
+	}
+
+	@bindThis
+	private async searchNoteByMeiliSearch(
+		q: string,
+		me: MiUser | null,
+		opts: SearchOpts,
+		pagination: SearchPagination,
+	): Promise<MiNote[]> {
+		if (!this.meilisearch || !this.meilisearchNoteIndex) {
+			throw new Error('MeiliSearch is not available');
+		}
+
+		const filter: Q = {
+			op: 'and',
+			qs: [],
+		};
+		if (pagination.untilId) filter.qs.push({
+			op: '<',
+			k: 'createdAt',
+			v: this.idService.parse(pagination.untilId).date.getTime(),
+		});
+		if (pagination.sinceId) filter.qs.push({
+			op: '>',
+			k: 'createdAt',
+			v: this.idService.parse(pagination.sinceId).date.getTime(),
+		});
+		if (opts.userId) filter.qs.push({ op: '=', k: 'userId', v: opts.userId });
+		if (opts.channelId) filter.qs.push({ op: '=', k: 'channelId', v: opts.channelId });
+		if (opts.host) {
+			if (opts.host === '.') {
+				filter.qs.push({ op: 'is null', k: 'userHost' });
+			} else {
+				filter.qs.push({ op: '=', k: 'userHost', v: opts.host });
 			}
 		}
+
+		const res = await this.meilisearchNoteIndex.search(q, {
+			sort: ['createdAt:desc'],
+			matchingStrategy: 'all',
+			attributesToRetrieve: ['id', 'createdAt'],
+			filter: compileQuery(filter),
+			limit: pagination.limit,
+		});
+		if (res.hits.length === 0) {
+			return [];
+		}
+
+		const [
+			userIdsWhoMeMuting,
+			userIdsWhoBlockingMe,
+		] = me
+			? await Promise.all([
+				this.cacheService.userMutingsCache.fetch(me.id),
+				this.cacheService.userBlockedCache.fetch(me.id),
+			])
+			: [new Set<string>(), new Set<string>()];
+		const notes = (await this.notesRepository.findBy({
+			id: In(res.hits.map(x => x.id)),
+		})).filter(note => {
+			if (me && isUserRelated(note, userIdsWhoBlockingMe)) return false;
+			if (me && isUserRelated(note, userIdsWhoMeMuting)) return false;
+			return true;
+		});
+
+		return notes.sort((a, b) => a.id > b.id ? -1 : 1);
+	}
 }