From 34380dccc2c637fd3822272789ab06040f5e85e3 Mon Sep 17 00:00:00 2001 From: MomentQYC Date: Mon, 5 Feb 2024 01:01:35 +0800 Subject: [PATCH] feat: support elasticsearch --- .config/example.yml | 14 +++ packages/backend/package.json | 1 + packages/backend/src/GlobalModule.ts | 24 +++- packages/backend/src/config.ts | 17 +++ packages/backend/src/core/SearchService.ts | 124 +++++++++++++++++++++ packages/backend/src/di-symbols.ts | 1 + pnpm-lock.yaml | 27 +++++ 7 files changed, 206 insertions(+), 2 deletions(-) diff --git a/.config/example.yml b/.config/example.yml index 3c9c3bc0d7..5d3913a806 100644 --- a/.config/example.yml +++ b/.config/example.yml @@ -129,6 +129,20 @@ redis: # index: '' # scope: local +# ┌───────────────────────────┐ +#───┘ ElasticSearch configuration └───────────────────────────── + +# If MeiliSearch is also enabled, the MeiliSearch feature is used +# and ElasticSearch is disabled. + +#elasticsearch: +# host: localhost +# port: 9200 +# user: '' +# pass: '' +# ssl: true +# index: '' + # ┌───────────────┐ #───┘ ID generation └─────────────────────────────────────────── diff --git a/packages/backend/package.json b/packages/backend/package.json index 31ca56be49..f5ad5b8e48 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -71,6 +71,7 @@ "@bull-board/fastify": "5.14.0", "@bull-board/ui": "5.14.0", "@discordapp/twemoji": "15.0.2", + "@elastic/elasticsearch": "8.11.0", "@fastify/accepts": "4.3.0", "@fastify/cookie": "9.3.1", "@fastify/cors": "8.5.0", diff --git a/packages/backend/src/GlobalModule.ts b/packages/backend/src/GlobalModule.ts index c83845b94c..060d967a15 100644 --- a/packages/backend/src/GlobalModule.ts +++ b/packages/backend/src/GlobalModule.ts @@ -6,6 +6,7 @@ import { Global, Inject, Module } from '@nestjs/common'; import * as Redis from 'ioredis'; import { DataSource } from 'typeorm'; +import { Client as ElasticSearch } from '@elastic/elasticsearch'; import { MeiliSearch } from 'meilisearch'; import { DI } from './di-symbols.js'; import { Config, loadConfig } from './config.js'; @@ -43,6 +44,25 @@ const $meilisearch: Provider = { inject: [DI.config], }; +const $elasticsearch: Provider = { + provide: DI.elasticsearch, + useFactory: (config: Config) => { + if (config.elasticsearch) { + return new ElasticSearch({ + nodes: `${config.elasticsearch.ssl ? 'https' : 'http'}://${config.elasticsearch.host}:${config.elasticsearch.port}`, + auth: { + username: config.elasticsearch.user, + password: config.elasticsearch.pass, + }, + //headers: {'Content-Type': 'application/json'}, + }); + } else { + return null; + } + }, + inject: [DI.config], +}; + const $redis: Provider = { provide: DI.redis, useFactory: (config: Config) => { @@ -81,8 +101,8 @@ const $redisForTimelines: Provider = { @Global() @Module({ imports: [RepositoryModule], - providers: [$config, $db, $meilisearch, $redis, $redisForPub, $redisForSub, $redisForTimelines], - exports: [$config, $db, $meilisearch, $redis, $redisForPub, $redisForSub, $redisForTimelines, RepositoryModule], + providers: [$config, $db, $meilisearch, $elasticsearch, $redis, $redisForPub, $redisForSub, $redisForTimelines], + exports: [$config, $db, $meilisearch, $elasticsearch, $redis, $redisForPub, $redisForSub, $redisForTimelines, RepositoryModule], }) export class GlobalModule implements OnApplicationShutdown { constructor( diff --git a/packages/backend/src/config.ts b/packages/backend/src/config.ts index d433ce0eec..6be30012de 100644 --- a/packages/backend/src/config.ts +++ b/packages/backend/src/config.ts @@ -56,6 +56,14 @@ type Source = { index: string; scope?: 'local' | 'global' | string[]; }; + elasticsearch?: { + host: string; + port: string; + user: string; + pass: string; + ssl?: boolean; + index: string; + }; proxy?: string; proxySmtp?: string; @@ -124,6 +132,14 @@ export type Config = { index: string; scope?: 'local' | 'global' | string[]; } | undefined; + elasticsearch: { + host: string; + port: string; + user: string; + pass: string; + ssl?: boolean; + index: string; + } | undefined; proxy: string | undefined; proxySmtp: string | undefined; proxyBypassHosts: string[] | undefined; @@ -226,6 +242,7 @@ export function loadConfig(): Config { dbReplications: config.dbReplications, dbSlaves: config.dbSlaves, meilisearch: config.meilisearch, + elasticsearch: config.elasticsearch, redis, redisForPubsub: config.redisForPubsub ? convertRedisOptions(config.redisForPubsub, host) : redis, redisForJobQueue: config.redisForJobQueue ? convertRedisOptions(config.redisForJobQueue, host) : redis, diff --git a/packages/backend/src/core/SearchService.ts b/packages/backend/src/core/SearchService.ts index a46d68fd84..452aef068f 100644 --- a/packages/backend/src/core/SearchService.ts +++ b/packages/backend/src/core/SearchService.ts @@ -16,6 +16,7 @@ import { isUserRelated } from '@/misc/is-user-related.js'; import { CacheService } from '@/core/CacheService.js'; import { QueryService } from '@/core/QueryService.js'; import { IdService } from '@/core/IdService.js'; +import { Client as ElasticSearch } from '@elastic/elasticsearch'; import type { Index, MeiliSearch } from 'meilisearch'; type K = string; @@ -65,6 +66,7 @@ function compileQuery(q: Q): string { export class SearchService { private readonly meilisearchIndexScope: 'local' | 'global' | string[] = 'local'; private meilisearchNoteIndex: Index | null = null; + private elasticsearchNoteIndex: string | null = null; constructor( @Inject(DI.config) @@ -73,6 +75,9 @@ export class SearchService { @Inject(DI.meilisearch) private meilisearch: MeiliSearch | null, + @Inject(DI.elasticsearch) + private elasticsearch: ElasticSearch | null, + @Inject(DI.notesRepository) private notesRepository: NotesRepository, @@ -106,6 +111,46 @@ export class SearchService { }); } + if (!meilisearch && this.elasticsearch) { + const indexName = `${config.elasticsearch!.index}---notes`; + this.elasticsearchNoteIndex = indexName; + + this.elasticsearch.indices.exists({ + index: indexName, + }).then((indexExists) => { + if (!indexExists) { + this.elasticsearch?.indices.create({ + index: indexName, + body: { + mappings: { + properties: { + text: { type: 'text' }, + cw: { type: 'text' }, + createdAt: { type: 'long' }, + userId: { type: 'keyword' }, + userHost: { type: 'keyword' }, + channelId: { type: 'keyword' }, + tags: { type: 'keyword' }, + }, + }, + settings: { + //TODO: Make settings for optimization. + }, + }, + }).catch((error) => { + console.error(error); + }); + } else { + console.log(`Index ${indexName} already exists`); + } + }).catch((error) => { + console.error(error); + }); + } else { + console.error('Elasticsearch is not available'); + this.elasticsearchNoteIndex = null; + } + if (config.meilisearch?.scope) { this.meilisearchIndexScope = config.meilisearch.scope; } @@ -145,6 +190,27 @@ export class SearchService { primaryKey: 'id', }); } + + if (!this.meilisearch && this.elasticsearch) { + const body = { + createdAt: this.idService.parse(note.id).date.getTime(), + userId: note.userId, + userHost: note.userHost, + channelId: note.channelId, + cw: note.cw, + text: note.text, + tags: note.tags, + }; + + console.log(body); + console.log(this.elasticsearchNoteIndex); + + await this.elasticsearch.index({ + index: this.elasticsearchNoteIndex as string, + id: note.id, + body: body, + }); + } } @bindThis @@ -154,6 +220,13 @@ export class SearchService { if (this.meilisearch) { this.meilisearchNoteIndex!.deleteDocument(note.id); } + + if (!this.meilisearch && this.elasticsearch) { + (this.elasticsearch.delete)({ + index: this.elasticsearchNoteIndex as string, + id: note.id, + }); + } } @bindThis @@ -205,6 +278,57 @@ export class SearchService { return true; }); return notes.sort((a, b) => a.id > b.id ? -1 : 1); + } else if (this.elasticsearch) { + const esFilter: any = { + bool: { + must: [], + }, + }; + if (pagination.untilId) esFilter.bool.must.push({ range: { createdAt: { lt: this.idService.parse(pagination.untilId).date.getTime() } } }); + if (pagination.sinceId) esFilter.bool.must.push({ range: { createdAt: { gt: this.idService.parse(pagination.sinceId).date.getTime() } } }); + if (opts.userId) esFilter.bool.must.push({ term: { userId: opts.userId } }); + if (opts.channelId) esFilter.bool.must.push({ term: { channelId: opts.channelId } }); + if (opts.host) { + if (opts.host === '.') { + esFilter.bool.must.push({ bool: { must_not: [{ exists: { field: 'userHost' } }] } }); + } else { + esFilter.bool.must.push({ term: { userHost: opts.host } }); + } + } + const res = await (this.elasticsearch.search)({ + index: this.elasticsearchNoteIndex as string, + body: { + query: { + bool: { + must: [ + { wildcard: { "text": { value: `*${q}*` }, } }, + esFilter, + ] + }, + }, + }, + sort: [{ createdAt: { order: "desc" } }], + _source: ['id', 'createdAt'], + size: pagination.limit, + + }); + const noteIds = res.hits.hits.map((hit: any) => hit._id); + if (noteIds.length === 0) return []; + const [ + userIdsWhoMeMuting, + userIdsWhoBlockingMe, + ] = me ? await Promise.all([ + this.cacheService.userMutingsCache.fetch(me.id), + this.cacheService.userBlockedCache.fetch(me.id), + ]) : [new Set(), new Set()]; + const notes = (await this.notesRepository.findBy({ + id: In(noteIds), + })).filter(note => { + if (me && isUserRelated(note, userIdsWhoBlockingMe)) return false; + if (me && isUserRelated(note, userIdsWhoMeMuting)) return false; + return true; + }); + return notes.sort((a, b) => a.id > b.id ? -1 : 1); } else { const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), pagination.sinceId, pagination.untilId); diff --git a/packages/backend/src/di-symbols.ts b/packages/backend/src/di-symbols.ts index 73de01f33a..040e7c0f8a 100644 --- a/packages/backend/src/di-symbols.ts +++ b/packages/backend/src/di-symbols.ts @@ -7,6 +7,7 @@ export const DI = { config: Symbol('config'), db: Symbol('db'), meilisearch: Symbol('meilisearch'), + elasticsearch: Symbol('elasticsearch'), redis: Symbol('redis'), redisForPub: Symbol('redisForPub'), redisForSub: Symbol('redisForSub'), diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0561e8b01e..b5e8da0c6c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -77,6 +77,9 @@ importers: '@discordapp/twemoji': specifier: 15.0.2 version: 15.0.2 + '@elastic/elasticsearch': + specifier: 8.11.0 + version: 8.11.0 '@fastify/accepts': specifier: 4.3.0 version: 4.3.0 @@ -3737,6 +3740,30 @@ packages: resolution: {integrity: sha512-dBVuXR082gk3jsFp7Rd/JI4kytwGHecnCoTtXFb7DB6CNHp4rg5k1bhg0nWdLGLnOV71lmDzGQaLMy8iPLY0pw==} engines: {node: '>=10.0.0'} dev: true + + /@elastic/elasticsearch@8.11.0: + resolution: {integrity: sha512-1UEQFdGLuKdROLJnMTjegasRM3X9INm/PVADoIVgdTfuv6DeJ17UMuNwYSkCrLrC0trLjjGV4YganpbJJX/VLg==} + engines: {node: '>=18'} + dependencies: + '@elastic/transport': 8.4.0 + tslib: 2.6.2 + transitivePeerDependencies: + - supports-color + dev: false + + /@elastic/transport@8.4.0: + resolution: {integrity: sha512-Yb3fDa7yGD0ca3uMbL64M3vM1cE5h5uHmBcTjkdB4VpCasRNKSd09iDpwqX8zX1tbBtxcaKYLceKthWvPeIxTw==} + engines: {node: '>=16'} + dependencies: + debug: 4.3.4(supports-color@8.1.1) + hpagent: 1.2.0 + ms: 2.1.3 + secure-json-parse: 2.7.0 + tslib: 2.6.2 + undici: 5.28.2 + transitivePeerDependencies: + - supports-color + dev: false /@emotion/use-insertion-effect-with-fallbacks@1.0.0(react@18.2.0): resolution: {integrity: sha512-1eEgUGmkaljiBnRMTdksDV1W4kUnmwgp7X9G8B++9GYwl1lUdqSndSriIrTJ0N7LQaoauY9JJ2yhiOYK5+NI4A==}