From 64e6a020eb6fc1ba1b7257b614efefa310363be8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=81=8A=E3=81=95=E3=82=80=E3=81=AE=E3=81=B2=E3=81=A8?= <46447427+samunohito@users.noreply.github.com> Date: Sun, 17 Nov 2024 01:30:03 +0900 Subject: [PATCH 1/4] =?UTF-8?q?feat(backend):=20pgroonga=E3=81=AB=E5=AF=BE?= =?UTF-8?q?=E5=BF=9C=EF=BC=88config=E3=81=AE=E6=A7=8B=E6=88=90=E5=A4=89?= =?UTF-8?q?=E6=9B=B4=E3=81=82=E3=82=8A=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .config/docker_example.yml | 22 +- .config/example.yml | 22 +- CHANGELOG.md | 4 + packages/backend/src/GlobalModule.ts | 10 +- packages/backend/src/config.ts | 9 + packages/backend/src/core/SearchService.ts | 279 +++++++++++++-------- 6 files changed, 232 insertions(+), 114 deletions(-) diff --git a/.config/docker_example.yml b/.config/docker_example.yml index 3f8e5734ce..38ba6feabc 100644 --- a/.config/docker_example.yml +++ b/.config/docker_example.yml @@ -114,9 +114,27 @@ redis: # #prefix: example-prefix # #db: 1 -# ┌───────────────────────────┐ -#───┘ MeiliSearch configuration └───────────────────────────── +# ┌───────────────────────────────┐ +#───┘ Fulltext search configuration └───────────────────────────── +# These are the setting items for the full-text search provider. +fulltextSearch: + # You can select the ID generation method. + # - sqlLike (default) + # Use SQL-like search. + # This is a standard feature of PostgreSQL, so no special extensions are required. + # - sqlPgroonga + # Use pgroonga. + # You need to install pgroonga and configure it as a PostgreSQL extension. + # In addition to the above, you need to create a pgroonga index on the text column of the note table. + # see: https://pgroonga.github.io/tutorial/ + # - meilisearch + # Use Meilisearch. + # You need to install Meilisearch and configure. + provider: sqlLike + +# For Meilisearch settings. +# If you select "meilisearch" for "fulltextSearch.provider", it must be set. # You can set scope to local (default value) or global # (include notes from remote). diff --git a/.config/example.yml b/.config/example.yml index 60a6a0aa71..ce609b1d1e 100644 --- a/.config/example.yml +++ b/.config/example.yml @@ -196,9 +196,27 @@ redis: # # You can specify more ioredis options... # #username: example-username -# ┌───────────────────────────┐ -#───┘ MeiliSearch configuration └───────────────────────────── +# ┌───────────────────────────────┐ +#───┘ Fulltext search configuration └───────────────────────────── +# These are the setting items for the full-text search provider. +fulltextSearch: + # You can select the ID generation method. + # - sqlLike (default) + # Use SQL-like search. + # This is a standard feature of PostgreSQL, so no special extensions are required. + # - sqlPgroonga + # Use pgroonga. + # You need to install pgroonga and configure it as a PostgreSQL extension. + # In addition to the above, you need to create a pgroonga index on the text column of the note table. + # see: https://pgroonga.github.io/tutorial/ + # - meilisearch + # Use Meilisearch. + # You need to install Meilisearch and configure. + provider: sqlLike + +# For Meilisearch settings. +# If you select "meilisearch" for "fulltextSearch.provider", it must be set. # You can set scope to local (default value) or global # (include notes from remote). diff --git a/CHANGELOG.md b/CHANGELOG.md index 232d52d7e1..f338a233b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,9 @@ ## 2024.11.0 ### Note +- [重要] ノート検索プロバイダの追加に伴い、configファイル（default.ymlなど）の構成が少し変わります. + - 新しい設定項目"fulltextSearch.provider"が追加されました. sqlLike, sqlPgroonga, meilisearchのいずれかを設定出来ます. + - すでにMeilisearchをお使いの場合、 **"fulltextSearch.provider"を"meilisearch"に設定する必要** があります. - Node.js 20.xは非推奨になりました。Node.js 22.x (LTS)の利用を推奨します。 - DockerのNode.jsが22.11.0に更新されました @@ -44,6 +47,7 @@ (Based on https://activitypub.software/TransFem-org/Sharkey/-/merge_requests/588) (Cherry-picked from https://activitypub.software/TransFem-org/Sharkey/-/merge_requests/715) - Enhance: リモートユーザーの照会をオリジナルにリダイレクトするように +- Enhance: ノート検索の選択肢としてpgroongaに対応 ( #14730 ) - Fix: フォロワーへのメッセージの絵文字をemojisに含めるように - Fix: Nested proxy requestsを検出した際にブロックするように [ghsa-gq5q-c77c-v236](https://github.com/misskey-dev/misskey/security/advisories/ghsa-gq5q-c77c-v236) diff --git a/packages/backend/src/GlobalModule.ts b/packages/backend/src/GlobalModule.ts index 6ae8ccfbb3..ace7f7841c 100644 --- a/packages/backend/src/GlobalModule.ts +++ b/packages/backend/src/GlobalModule.ts @@ -7,14 +7,14 @@ import { Global, Inject, Module } from '@nestjs/common'; import * as Redis from 'ioredis'; import { DataSource } from 'typeorm'; import { MeiliSearch } from 'meilisearch'; +import { MiMeta } from '@/models/Meta.js'; import { DI } from './di-symbols.js'; import { Config, loadConfig } from './config.js'; import { createPostgresDataSource } from './postgres.js'; import { RepositoryModule } from './models/RepositoryModule.js'; import { allSettled } from './misc/promise-tracker.js'; -import type { Provider, OnApplicationShutdown } from '@nestjs/common'; -import { MiMeta } from '@/models/Meta.js'; import { GlobalEvents } from './core/GlobalEventService.js'; +import type { Provider, OnApplicationShutdown } from '@nestjs/common'; const $config: Provider = { provide: DI.config, @@ -33,7 +33,11 @@ const $db: Provider = { const $meilisearch: Provider = { provide: DI.meilisearch, useFactory: (config: Config) => { - if (config.meilisearch) { + if (config.fulltextSearch?.provider === 'meilisearch') { + if (!config.meilisearch) { + throw new Error('MeiliSearch is enabled but no configuration is provided'); + } + return new MeiliSearch({ host: `${config.meilisearch.ssl ? 'https' : 'http'}://${config.meilisearch.host}:${config.meilisearch.port}`, apiKey: config.meilisearch.apiKey, diff --git a/packages/backend/src/config.ts b/packages/backend/src/config.ts index 42f1033b9d..5cdb50a575 100644 --- a/packages/backend/src/config.ts +++ b/packages/backend/src/config.ts @@ -50,6 +50,9 @@ type Source = { redisForJobQueue?: RedisOptionsSource; redisForTimelines?: RedisOptionsSource; redisForReactions?: RedisOptionsSource; + fulltextSearch?: { + provider?: FulltextSearchProvider; + }; meilisearch?: { host: string; port: string; @@ -124,6 +127,9 @@ export type Config = { user: string; pass: string; }[] | undefined; + fulltextSearch?: { + provider?: FulltextSearchProvider; + }; meilisearch: { host: string; port: string; @@ -184,6 +190,8 @@ export type Config = { pidFile: string; }; +export type FulltextSearchProvider = 'sqlLike' | 'sqlPgroonga' | 'meilisearch'; + const _filename = fileURLToPath(import.meta.url); const _dirname = dirname(_filename); @@ -252,6 +260,7 @@ export function loadConfig(): Config { db: { ...config.db, db: dbDb, user: dbUser, pass: dbPass }, dbReplications: config.dbReplications, dbSlaves: config.dbSlaves, + fulltextSearch: config.fulltextSearch, meilisearch: config.meilisearch, redis, redisForPubsub: config.redisForPubsub ? convertRedisOptions(config.redisForPubsub, host) : redis, diff --git a/packages/backend/src/core/SearchService.ts b/packages/backend/src/core/SearchService.ts index edfc470375..40592962c3 100644 --- a/packages/backend/src/core/SearchService.ts +++ b/packages/backend/src/core/SearchService.ts @@ -6,11 +6,11 @@ import { Inject, Injectable } from '@nestjs/common'; import { In } from 'typeorm'; import { DI } from '@/di-symbols.js'; -import type { Config } from '@/config.js'; +import { type Config, FulltextSearchProvider } from '@/config.js'; import { bindThis } from '@/decorators.js'; import { MiNote } from '@/models/Note.js'; -import { MiUser } from '@/models/_.js'; import type { NotesRepository } from '@/models/_.js'; +import { MiUser } from '@/models/_.js'; import { sqlLikeEscape } from '@/misc/sql-like-escape.js'; import { isUserRelated } from '@/misc/is-user-related.js'; import { CacheService } from '@/core/CacheService.js'; @@ -27,12 +27,24 @@ type Q = { op: '<', k: K, v: number } | { op: '>=', k: K, v: number } | { op: '<=', k: K, v: number } | - { op: 'is null', k: K} | - { op: 'is not null', k: K} | + { op: 'is null', k: K } | + { op: 'is not null', k: K } | { op: 'and', qs: Q[] } | { op: 'or', qs: Q[] } | { op: 'not', q: Q }; +export type SearchOpts = { + userId?: MiNote['userId'] | null; + channelId?: MiNote['channelId'] | null; + host?: string | null; +}; + +export type SearchPagination = { + untilId?: MiNote['id']; + sinceId?: MiNote['id']; + limit: number; +}; + function compileValue(value: V): string { if (typeof value === 'string') { return `'${value}'`; // TODO: escape @@ -64,7 +76,8 @@ function compileQuery(q: Q): string { @Injectable() export class SearchService { private readonly meilisearchIndexScope: 'local' | 'global' | string[] = 'local'; - private meilisearchNoteIndex: Index | null = null; + private readonly meilisearchNoteIndex: Index | null = null; + private readonly provider: FulltextSearchProvider; constructor( @Inject(DI.config) @@ -109,132 +122,184 @@ export class SearchService { if (config.meilisearch?.scope) { this.meilisearchIndexScope = config.meilisearch.scope; } + + this.provider = config.fulltextSearch?.provider ?? 'sqlLike'; } @bindThis public async indexNote(note: MiNote): Promise { + if (!this.meilisearch) return; if (note.text == null && note.cw == null) return; if (!['home', 'public'].includes(note.visibility)) return; - if (this.meilisearch) { - switch (this.meilisearchIndexScope) { - case 'global': - break; + switch (this.meilisearchIndexScope) { + case 'global': + break; - case 'local': - if (note.userHost == null) break; - return; + case 'local': + if (note.userHost == null) break; + return; - default: { - if (note.userHost == null) break; - if (this.meilisearchIndexScope.includes(note.userHost)) break; - return; - } + default: { + if (note.userHost == null) break; + if (this.meilisearchIndexScope.includes(note.userHost)) break; + return; } - - await this.meilisearchNoteIndex?.addDocuments([{ - id: note.id, - createdAt: this.idService.parse(note.id).date.getTime(), - userId: note.userId, - userHost: note.userHost, - channelId: note.channelId, - cw: note.cw, - text: note.text, - tags: note.tags, - }], { - primaryKey: 'id', - }); } + + await this.meilisearchNoteIndex?.addDocuments([{ + id: note.id, + createdAt: this.idService.parse(note.id).date.getTime(), + userId: note.userId, + userHost: note.userHost, + channelId: note.channelId, + cw: note.cw, + text: note.text, + tags: note.tags, + }], { + primaryKey: 'id', + }); } @bindThis public async unindexNote(note: MiNote): Promise { + if (!this.meilisearch) return; if (!['home', 'public'].includes(note.visibility)) return; - if (this.meilisearch) { - this.meilisearchNoteIndex!.deleteDocument(note.id); + await this.meilisearchNoteIndex?.deleteDocument(note.id); + } + + @bindThis + public async searchNote( + q: string, + me: MiUser | null, + opts: SearchOpts, + pagination: SearchPagination, + ): Promise { + switch (this.provider) { + case 'sqlLike': + case 'sqlPgroonga': { + // ほとんど内容に差がないのでsqlLikeとsqlPgroongaを同じ処理にしている. + // 今後の拡張で差が出る用であれば関数を分ける. + return this.searchNoteByLike(q, me, opts, pagination); + } + case 'meilisearch': { + return this.searchNoteByMeiliSearch(q, me, opts, pagination); + } + default: { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const typeCheck: never = this.provider; + return []; + } } } @bindThis - public async searchNote(q: string, me: MiUser | null, opts: { - userId?: MiNote['userId'] | null; - channelId?: MiNote['channelId'] | null; - host?: string | null; - }, pagination: { - untilId?: MiNote['id']; - sinceId?: MiNote['id']; - limit?: number; - }): Promise { - if (this.meilisearch) { - const filter: Q = { - op: 'and', - qs: [], - }; - if (pagination.untilId) filter.qs.push({ op: '<', k: 'createdAt', v: this.idService.parse(pagination.untilId).date.getTime() }); - if (pagination.sinceId) filter.qs.push({ op: '>', k: 'createdAt', v: this.idService.parse(pagination.sinceId).date.getTime() }); - if (opts.userId) filter.qs.push({ op: '=', k: 'userId', v: opts.userId }); - if (opts.channelId) filter.qs.push({ op: '=', k: 'channelId', v: opts.channelId }); - if (opts.host) { - if (opts.host === '.') { - filter.qs.push({ op: 'is null', k: 'userHost' }); - } else { - filter.qs.push({ op: '=', k: 'userHost', v: opts.host }); - } + private async searchNoteByLike( + q: string, + me: MiUser | null, + opts: SearchOpts, + pagination: SearchPagination, + ): Promise { + const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), pagination.sinceId, pagination.untilId); + + if (opts.userId) { + query.andWhere('note.userId = :userId', { userId: opts.userId }); + } else if (opts.channelId) { + query.andWhere('note.channelId = :channelId', { channelId: opts.channelId }); + } + + query + .innerJoinAndSelect('note.user', 'user') + .leftJoinAndSelect('note.reply', 'reply') + .leftJoinAndSelect('note.renote', 'renote') + .leftJoinAndSelect('reply.user', 'replyUser') + .leftJoinAndSelect('renote.user', 'renoteUser'); + + if (this.config.fulltextSearch?.provider === 'sqlPgroonga') { + query.andWhere('note.text &@ :q', { q }); + } else { + query.andWhere('note.text ILIKE :q', { q: `%${sqlLikeEscape(q)}%` }); + } + + if (opts.host) { + if (opts.host === '.') { + query.andWhere('user.host IS NULL'); + } else { + query.andWhere('user.host = :host', { host: opts.host }); } - const res = await this.meilisearchNoteIndex!.search(q, { - sort: ['createdAt:desc'], - matchingStrategy: 'all', - attributesToRetrieve: ['id', 'createdAt'], - filter: compileQuery(filter), - limit: pagination.limit, - }); - if (res.hits.length === 0) return []; - const [ - userIdsWhoMeMuting, - userIdsWhoBlockingMe, - ] = me ? await Promise.all([ + } + + this.queryService.generateVisibilityQuery(query, me); + if (me) this.queryService.generateMutedUserQuery(query, me); + if (me) this.queryService.generateBlockedUserQuery(query, me); + + return query.limit(pagination.limit).getMany(); + } + + @bindThis + private async searchNoteByMeiliSearch( + q: string, + me: MiUser | null, + opts: SearchOpts, + pagination: SearchPagination, + ): Promise { + if (!this.meilisearch || !this.meilisearchNoteIndex) { + throw new Error('MeiliSearch is not available'); + } + + const filter: Q = { + op: 'and', + qs: [], + }; + if (pagination.untilId) filter.qs.push({ + op: '<', + k: 'createdAt', + v: this.idService.parse(pagination.untilId).date.getTime(), + }); + if (pagination.sinceId) filter.qs.push({ + op: '>', + k: 'createdAt', + v: this.idService.parse(pagination.sinceId).date.getTime(), + }); + if (opts.userId) filter.qs.push({ op: '=', k: 'userId', v: opts.userId }); + if (opts.channelId) filter.qs.push({ op: '=', k: 'channelId', v: opts.channelId }); + if (opts.host) { + if (opts.host === '.') { + filter.qs.push({ op: 'is null', k: 'userHost' }); + } else { + filter.qs.push({ op: '=', k: 'userHost', v: opts.host }); + } + } + + const res = await this.meilisearchNoteIndex.search(q, { + sort: ['createdAt:desc'], + matchingStrategy: 'all', + attributesToRetrieve: ['id', 'createdAt'], + filter: compileQuery(filter), + limit: pagination.limit, + }); + if (res.hits.length === 0) { + return []; + } + + const [ + userIdsWhoMeMuting, + userIdsWhoBlockingMe, + ] = me + ? await Promise.all([ this.cacheService.userMutingsCache.fetch(me.id), this.cacheService.userBlockedCache.fetch(me.id), - ]) : [new Set(), new Set()]; - const notes = (await this.notesRepository.findBy({ - id: In(res.hits.map(x => x.id)), - })).filter(note => { - if (me && isUserRelated(note, userIdsWhoBlockingMe)) return false; - if (me && isUserRelated(note, userIdsWhoMeMuting)) return false; - return true; - }); - return notes.sort((a, b) => a.id > b.id ? -1 : 1); - } else { - const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), pagination.sinceId, pagination.untilId); + ]) + : [new Set(), new Set()]; + const notes = (await this.notesRepository.findBy({ + id: In(res.hits.map(x => x.id)), + })).filter(note => { + if (me && isUserRelated(note, userIdsWhoBlockingMe)) return false; + if (me && isUserRelated(note, userIdsWhoMeMuting)) return false; + return true; + }); - if (opts.userId) { - query.andWhere('note.userId = :userId', { userId: opts.userId }); - } else if (opts.channelId) { - query.andWhere('note.channelId = :channelId', { channelId: opts.channelId }); - } - - query - .andWhere('note.text ILIKE :q', { q: `%${ sqlLikeEscape(q) }%` }) - .innerJoinAndSelect('note.user', 'user') - .leftJoinAndSelect('note.reply', 'reply') - .leftJoinAndSelect('note.renote', 'renote') - .leftJoinAndSelect('reply.user', 'replyUser') - .leftJoinAndSelect('renote.user', 'renoteUser'); - - if (opts.host) { - if (opts.host === '.') { - query.andWhere('user.host IS NULL'); - } else { - query.andWhere('user.host = :host', { host: opts.host }); - } - } - - this.queryService.generateVisibilityQuery(query, me); - if (me) this.queryService.generateMutedUserQuery(query, me); - if (me) this.queryService.generateBlockedUserQuery(query, me); - - return await query.limit(pagination.limit).getMany(); - } + return notes.sort((a, b) => a.id > b.id ? -1 : 1); } } From 342873a7d1e99301a35bf9ac8d35639d3942bc5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=81=8A=E3=81=95=E3=82=80=E3=81=AE=E3=81=B2=E3=81=A8?= <46447427+samunohito@users.noreply.github.com> Date: Sun, 17 Nov 2024 09:33:01 +0900 Subject: [PATCH 2/4] fix CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f338a233b3..19a0caa1db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ - [重要] ノート検索プロバイダの追加に伴い、configファイル（default.ymlなど）の構成が少し変わります. - 新しい設定項目"fulltextSearch.provider"が追加されました. sqlLike, sqlPgroonga, meilisearchのいずれかを設定出来ます. - すでにMeilisearchをお使いの場合、 **"fulltextSearch.provider"を"meilisearch"に設定する必要** があります. + - 詳細は.config/example.ymlまたは.config/docker_example.ymlの'Fulltext search configuration'をご参照願います. - Node.js 20.xは非推奨になりました。Node.js 22.x (LTS)の利用を推奨します。 - DockerのNode.jsが22.11.0に更新されました From 34aa83f682a999c33455b10e3e00a01bd5be49a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=81=8A=E3=81=95=E3=82=80=E3=81=AE=E3=81=B2=E3=81=A8?= <46447427+samunohito@users.noreply.github.com> Date: Sun, 17 Nov 2024 12:35:02 +0900 Subject: [PATCH 3/4] fix CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 19a0caa1db..fbe2dc03c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ - [重要] ノート検索プロバイダの追加に伴い、configファイル（default.ymlなど）の構成が少し変わります. - 新しい設定項目"fulltextSearch.provider"が追加されました. sqlLike, sqlPgroonga, meilisearchのいずれかを設定出来ます. - すでにMeilisearchをお使いの場合、 **"fulltextSearch.provider"を"meilisearch"に設定する必要** があります. - - 詳細は.config/example.ymlまたは.config/docker_example.ymlの'Fulltext search configuration'をご参照願います. + - 詳細は #14730 および `.config/example.yml` または `.config/docker_example.yml`の'Fulltext search configuration'をご参照願います. - Node.js 20.xは非推奨になりました。Node.js 22.x (LTS)の利用を推奨します。 - DockerのNode.jsが22.11.0に更新されました From 87888dcc016f58fc17d29010d765403fbd481db1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=81=8A=E3=81=95=E3=82=80=E3=81=AE=E3=81=B2=E3=81=A8?= <46447427+samunohito@users.noreply.github.com> Date: Sun, 17 Nov 2024 13:25:18 +0900 Subject: [PATCH 4/4] add using provider logging --- packages/backend/src/core/SearchService.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/backend/src/core/SearchService.ts b/packages/backend/src/core/SearchService.ts index 40592962c3..01d300700d 100644 --- a/packages/backend/src/core/SearchService.ts +++ b/packages/backend/src/core/SearchService.ts @@ -16,6 +16,7 @@ import { isUserRelated } from '@/misc/is-user-related.js'; import { CacheService } from '@/core/CacheService.js'; import { QueryService } from '@/core/QueryService.js'; import { IdService } from '@/core/IdService.js'; +import { LoggerService } from '@/core/LoggerService.js'; import type { Index, MeiliSearch } from 'meilisearch'; type K = string; @@ -92,6 +93,7 @@ export class SearchService { private cacheService: CacheService, private queryService: QueryService, private idService: IdService, + private loggerService: LoggerService, ) { if (meilisearch) { this.meilisearchNoteIndex = meilisearch.index(`${config.meilisearch!.index}---notes`); @@ -124,6 +126,7 @@ export class SearchService { } this.provider = config.fulltextSearch?.provider ?? 'sqlLike'; + this.loggerService.getLogger('SearchService').info(`-- Provider: ${this.provider}`); } @bindThis