From 7a8ab424d4a8c3c28447d01db8ce06ceaa32fcc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=81=8A=E3=81=95=E3=82=80=E3=81=AE=E3=81=B2=E3=81=A8?= <46447427+samunohito@users.noreply.github.com> Date: Fri, 28 Feb 2025 19:56:14 +0900 Subject: [PATCH] impl service --- packages/backend/src/core/AntennaService.ts | 287 ++++++++++++-------- packages/backend/src/models/Antenna.ts | 21 +- 2 files changed, 198 insertions(+), 110 deletions(-) diff --git a/packages/backend/src/core/AntennaService.ts b/packages/backend/src/core/AntennaService.ts index e827ffa68c..e8ef9e0cc2 100644 --- a/packages/backend/src/core/AntennaService.ts +++ b/packages/backend/src/core/AntennaService.ts @@ -5,24 +5,142 @@ import { Inject, Injectable } from '@nestjs/common'; import * as Redis from 'ioredis'; -import type { MiAntenna } from '@/models/Antenna.js'; +import RE2 from 're2'; +import { MiAntenna } from '@/models/Antenna.js'; import type { MiNote } from '@/models/Note.js'; import type { MiUser } from '@/models/User.js'; import { GlobalEventService } from '@/core/GlobalEventService.js'; import * as Acct from '@/misc/acct.js'; import type { Packed } from '@/misc/json-schema.js'; import { DI } from '@/di-symbols.js'; -import type { AntennasRepository, UserListMembershipsRepository } from '@/models/_.js'; +import type { AntennasRepository } from '@/models/_.js'; import { UtilityService } from '@/core/UtilityService.js'; import { bindThis } from '@/decorators.js'; import type { GlobalEvents } from '@/core/GlobalEventService.js'; import { FanoutTimelineService } from '@/core/FanoutTimelineService.js'; import type { OnApplicationShutdown } from '@nestjs/common'; +type AntennaFilter = { + antennaId: string; + src: MiAntenna; + testKeywords(target: string): boolean; +}; + +function matchAntennaKeywords(target: string, keywords: string[][]): boolean { + return keywords.some(and => and.every(keyword => target.includes(keyword))); +} + +function matchAntennaKeywordsCaseInsensitive(target: string, keywords: string[][]): boolean { + const _target = target.toLowerCase(); + return keywords.some(and => and.every(keyword => _target.includes(keyword))); +} + +function matchAntennaKeywordsRegex(target: string, patterns: RE2[]): boolean { + return patterns.every(re => re.test(target)); +} + +export const antennaFilters = { + regex: { + includeAndExclude: (target: string, keywords: RE2[], excludeKeywords: RE2[]): boolean => { + return matchAntennaKeywordsRegex(target, keywords) && !matchAntennaKeywordsRegex(target, excludeKeywords); + }, + includeOnly: (target: string, keywords: RE2[]): boolean => { + return matchAntennaKeywordsRegex(target, keywords); + }, + excludeOnly: (target: string, excludeKeywords: RE2[]): boolean => { + return !matchAntennaKeywordsRegex(target, excludeKeywords); + }, + }, + noRegex: { + caseSensitive: { + includeAndExclude: (target: string, keywords: string[][], excludeKeywords: string[][]): boolean => { + return matchAntennaKeywords(target, keywords) && !matchAntennaKeywords(target, excludeKeywords); + }, + includeOnly: (target: string, keywords: string[][]): boolean => { + return matchAntennaKeywords(target, keywords); + }, + excludeOnly: (target: string, excludeKeywords: string[][]): boolean => { + return !matchAntennaKeywords(target, excludeKeywords); + }, + }, + caseInsensitive: { + includeAndExclude: (target: string, keywords: string[][], excludeKeywords: string[][]): boolean => { + return matchAntennaKeywordsCaseInsensitive(target, keywords) && !matchAntennaKeywordsCaseInsensitive(target, excludeKeywords); + }, + includeOnly: (target: string, keywords: string[][]): boolean => { + return matchAntennaKeywordsCaseInsensitive(target, keywords); + }, + excludeOnly: (target: string, excludeKeywords: string[][]): boolean => { + return !matchAntennaKeywordsCaseInsensitive(target, excludeKeywords); + }, + }, + }, +}; + +function alwaysTrue(): boolean { + return true; +} + +function createAntennaFilter(antenna: MiAntenna): AntennaFilter { + function createTestKeywordsFunction(antenna: MiAntenna): AntennaFilter['testKeywords'] { + // Clean up + const keywords = antenna.keywords + .map(xs => xs.filter(x => x !== '')) + .filter(xs => xs.length > 0); + const excludeKeywords = antenna.excludeKeywords + .map(xs => xs.filter(x => x !== '')) + .filter(xs => xs.length > 0); + + if (antenna.useRegex) { + // 元々はAND検索を行うために2次元配列としてもっていた歴史的経緯がある. + // 正規表現の時は1行に付き1パターンとするため、[n][0]にパターンの内容すべてが格納されているものとして扱う. + const keywordsPatterns = keywords.map(line => new RE2(line[0])); + const excludeKeywordsPatterns = excludeKeywords.map(line => new RE2(line[0])); + + const regex = antennaFilters.regex; + if (keywords.length > 0 && excludeKeywords.length > 0) { + return (target: string) => regex.includeAndExclude(target, keywordsPatterns, excludeKeywordsPatterns); + } else if (keywords.length > 0) { + return (target: string) => regex.includeOnly(target, keywordsPatterns); + } else if (excludeKeywords.length > 0) { + return (target: string) => regex.excludeOnly(target, excludeKeywordsPatterns); + } else { + return alwaysTrue; + } + } else if (antenna.caseSensitive) { + if (keywords.length > 0 && excludeKeywords.length > 0) { + return (target: string) => antennaFilters.noRegex.caseSensitive.includeAndExclude(target, keywords, excludeKeywords); + } else if (keywords.length > 0) { + return (target: string) => antennaFilters.noRegex.caseSensitive.includeOnly(target, keywords); + } else if (excludeKeywords.length > 0) { + return (target: string) => antennaFilters.noRegex.caseSensitive.excludeOnly(target, excludeKeywords); + } else { + return alwaysTrue; + } + } else { + if (keywords.length > 0 && excludeKeywords.length > 0) { + return (target: string) => antennaFilters.noRegex.caseInsensitive.includeAndExclude(target, keywords, excludeKeywords); + } else if (keywords.length > 0) { + return (target: string) => antennaFilters.noRegex.caseInsensitive.includeOnly(target, keywords); + } else if (excludeKeywords.length > 0) { + return (target: string) => antennaFilters.noRegex.caseInsensitive.excludeOnly(target, excludeKeywords); + } else { + return alwaysTrue; + } + } + } + + return { + antennaId: antenna.id, + src: antenna, + testKeywords: createTestKeywordsFunction(antenna), + }; +} + @Injectable() export class AntennaService implements OnApplicationShutdown { - private antennasFetched: boolean; - private antennas: MiAntenna[]; + private filtersFetched: boolean; + private filters: AntennaFilter[]; constructor( @Inject(DI.redisForTimelines) @@ -34,15 +152,12 @@ export class AntennaService implements OnApplicationShutdown { @Inject(DI.antennasRepository) private antennasRepository: AntennasRepository, - @Inject(DI.userListMembershipsRepository) - private userListMembershipsRepository: UserListMembershipsRepository, - private utilityService: UtilityService, private globalEventService: GlobalEventService, private fanoutTimelineService: FanoutTimelineService, ) { - this.antennasFetched = false; - this.antennas = []; + this.filtersFetched = false; + this.filters = []; this.redisForSub.on('message', this.onRedisMessage); } @@ -54,37 +169,24 @@ export class AntennaService implements OnApplicationShutdown { if (obj.channel === 'internal') { const { type, body } = obj.message as GlobalEvents['internal']['payload']; switch (type) { - case 'antennaCreated': - this.antennas.push({ // TODO: このあたりのデシリアライズ処理は各modelファイル内に関数としてexportしたい - ...body, - lastUsedAt: new Date(body.lastUsedAt), - user: null, // joinなカラムは通常取ってこないので - userList: null, // joinなカラムは通常取ってこないので - }); + case 'antennaCreated': { + this.filters.push(createAntennaFilter(MiAntenna.deserialize(body))); break; + } case 'antennaUpdated': { - const idx = this.antennas.findIndex(a => a.id === body.id); + const idx = this.filters.findIndex(a => a.antennaId === body.id); if (idx >= 0) { - this.antennas[idx] = { // TODO: このあたりのデシリアライズ処理は各modelファイル内に関数としてexportしたい - ...body, - lastUsedAt: new Date(body.lastUsedAt), - user: null, // joinなカラムは通常取ってこないので - userList: null, // joinなカラムは通常取ってこないので - }; + this.filters[idx] = createAntennaFilter(MiAntenna.deserialize(body)); } else { // サーバ起動時にactiveじゃなかった場合、リストに持っていないので追加する必要あり - this.antennas.push({ // TODO: このあたりのデシリアライズ処理は各modelファイル内に関数としてexportしたい - ...body, - lastUsedAt: new Date(body.lastUsedAt), - user: null, // joinなカラムは通常取ってこないので - userList: null, // joinなカラムは通常取ってこないので - }); + this.filters.push(createAntennaFilter(MiAntenna.deserialize(body))); } + break; } + case 'antennaDeleted': { + this.filters = this.filters.filter(a => a.antennaId !== body.id); break; - case 'antennaDeleted': - this.antennas = this.antennas.filter(a => a.id !== body.id); - break; + } default: break; } @@ -93,15 +195,17 @@ export class AntennaService implements OnApplicationShutdown { @bindThis public async addNoteToAntennas(note: MiNote, noteUser: { id: MiUser['id']; username: string; host: string | null; isBot: boolean; }): Promise { - const antennas = await this.getAntennas(); - const antennasWithMatchResult = await Promise.all(antennas.map(antenna => this.checkHitAntenna(antenna, note, noteUser).then(hit => [antenna, hit] as const))); - const matchedAntennas = antennasWithMatchResult.filter(([, hit]) => hit).map(([antenna]) => antenna); + const filters = await this.getFilters(); + const checkResults = await Promise.all( + filters.map(filter => this.checkHitAntenna(filter, note, noteUser).then(hit => [filter, hit] as const)), + ); const redisPipeline = this.redisForTimelines.pipeline(); - - for (const antenna of matchedAntennas) { - this.fanoutTimelineService.push(`antennaTimeline:${antenna.id}`, note.id, 200, redisPipeline); - this.globalEventService.publishAntennaStream(antenna.id, 'note', note); + for (const [filter, hit] of checkResults) { + if (hit) { + this.fanoutTimelineService.push(`antennaTimeline:${filter.antennaId}`, note.id, 200, redisPipeline); + this.globalEventService.publishAntennaStream(filter.antennaId, 'note', note); + } } redisPipeline.exec(); @@ -110,100 +214,67 @@ export class AntennaService implements OnApplicationShutdown { // NOTE: フォローしているユーザーのノート、リストのユーザーのノート、グループのユーザーのノート指定はパフォーマンス上の理由で無効になっている @bindThis - public async checkHitAntenna(antenna: MiAntenna, note: (MiNote | Packed<'Note'>), noteUser: { id: MiUser['id']; username: string; host: string | null; isBot: boolean; }): Promise { + private async checkHitAntenna( + filter: AntennaFilter, + note: (MiNote | Packed<'Note'>), + noteUser: { id: MiUser['id']; username: string; host: string | null; isBot: boolean; }, + ): Promise { if (note.visibility === 'specified') return false; if (note.visibility === 'followers') return false; + const antenna = filter.src; if (antenna.excludeBots && noteUser.isBot) return false; - if (antenna.localOnly && noteUser.host != null) return false; - if (!antenna.withReplies && note.replyId != null) return false; if (antenna.src === 'home') { // TODO } else if (antenna.src === 'list') { - if (antenna.userListId == null) return false; - const exists = await this.userListMembershipsRepository.exists({ - where: { - userListId: antenna.userListId, - userId: note.userId, - }, - }); - if (!exists) return false; + // フロントエンドは塞がれているのでコメントアウト + // if (antenna.userListId == null) return false; + // const exists = await this.userListMembershipsRepository.exists({ + // where: { + // userListId: antenna.userListId, + // userId: note.userId, + // }, + // }); + // if (!exists) return false; } else if (antenna.src === 'users') { - const accts = antenna.users.map(x => { - const { username, host } = Acct.parse(x); - return this.utilityService.getFullApAccount(username, host).toLowerCase(); - }); - if (!accts.includes(this.utilityService.getFullApAccount(noteUser.username, noteUser.host).toLowerCase())) return false; + if (!this.isIncludeUser(antenna, noteUser)) return false; } else if (antenna.src === 'users_blacklist') { - const accts = antenna.users.map(x => { - const { username, host } = Acct.parse(x); - return this.utilityService.getFullApAccount(username, host).toLowerCase(); - }); - if (accts.includes(this.utilityService.getFullApAccount(noteUser.username, noteUser.host).toLowerCase())) return false; - } - - const keywords = antenna.keywords - // Clean up - .map(xs => xs.filter(x => x !== '')) - .filter(xs => xs.length > 0); - - if (keywords.length > 0) { - if (note.text == null && note.cw == null) return false; - - const _text = (note.text ?? '') + '\n' + (note.cw ?? ''); - - const matched = keywords.some(and => - and.every(keyword => - antenna.caseSensitive - ? _text.includes(keyword) - : _text.toLowerCase().includes(keyword.toLowerCase()), - )); - - if (!matched) return false; - } - - const excludeKeywords = antenna.excludeKeywords - // Clean up - .map(xs => xs.filter(x => x !== '')) - .filter(xs => xs.length > 0); - - if (excludeKeywords.length > 0) { - if (note.text == null && note.cw == null) return false; - - const _text = (note.text ?? '') + '\n' + (note.cw ?? ''); - - const matched = excludeKeywords.some(and => - and.every(keyword => - antenna.caseSensitive - ? _text.includes(keyword) - : _text.toLowerCase().includes(keyword.toLowerCase()), - )); - - if (matched) return false; + if (this.isIncludeUser(antenna, noteUser)) return false; } if (antenna.withFile) { if (note.fileIds && note.fileIds.length === 0) return false; } - // TODO: eval expression - - return true; + const _text = (note.text ?? '') + '\n' + (note.cw ?? ''); + return filter.testKeywords(_text); } @bindThis - public async getAntennas() { - if (!this.antennasFetched) { - this.antennas = await this.antennasRepository.findBy({ + private async getFilters() { + if (!this.filtersFetched) { + const antennas = await this.antennasRepository.findBy({ isActive: true, }); - this.antennasFetched = true; + this.filters = antennas.map(createAntennaFilter); + this.filtersFetched = true; } - return this.antennas; + return this.filters; + } + + @bindThis + private isIncludeUser(antenna: MiAntenna, noteUser: { username: string; host: string | null; }): boolean { + const antennaUserAccounts = antenna.users.map(x => { + const { username, host } = Acct.parse(x); + return this.utilityService.getFullApAccount(username, host).toLowerCase(); + }); + + const noteUserAccount = this.utilityService.getFullApAccount(noteUser.username, noteUser.host).toLowerCase(); + return antennaUserAccounts.includes(noteUserAccount); } @bindThis diff --git a/packages/backend/src/models/Antenna.ts b/packages/backend/src/models/Antenna.ts index 33e6f48189..8c165b571f 100644 --- a/packages/backend/src/models/Antenna.ts +++ b/packages/backend/src/models/Antenna.ts @@ -3,10 +3,11 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import { PrimaryColumn, Entity, Index, JoinColumn, Column, ManyToOne } from 'typeorm'; -import { id } from './util/id.js'; +import { Column, Entity, Index, JoinColumn, ManyToOne, PrimaryColumn } from 'typeorm'; +import { Serialized } from '@/types.js'; import { MiUser } from './User.js'; import { MiUserList } from './UserList.js'; +import { id } from './util/id.js'; @Entity('antenna') export class MiAntenna { @@ -100,4 +101,20 @@ export class MiAntenna { default: false, }) public localOnly: boolean; + + @Column('boolean', { + default: false, + }) + public useRegex: boolean; + + public static deserialize(data: Serialized): MiAntenna { + return { + ...data, + lastUsedAt: new Date(data.lastUsedAt), + // クエリビルダで明示的にSELECT/JOINしないかぎり設定されない値なのでnullにしておく + user: null, + // クエリビルダで明示的にSELECT/JOINしないかぎり設定されない値なのでnullにしておく + userList: null, + }; + } }