From 7a8ab424d4a8c3c28447d01db8ce06ceaa32fcc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=81=8A=E3=81=95=E3=82=80=E3=81=AE=E3=81=B2=E3=81=A8?= <46447427+samunohito@users.noreply.github.com> Date: Fri, 28 Feb 2025 19:56:14 +0900 Subject: [PATCH 1/6] impl service --- packages/backend/src/core/AntennaService.ts | 287 ++++++++++++-------- packages/backend/src/models/Antenna.ts | 21 +- 2 files changed, 198 insertions(+), 110 deletions(-) diff --git a/packages/backend/src/core/AntennaService.ts b/packages/backend/src/core/AntennaService.ts index e827ffa68c..e8ef9e0cc2 100644 --- a/packages/backend/src/core/AntennaService.ts +++ b/packages/backend/src/core/AntennaService.ts @@ -5,24 +5,142 @@ import { Inject, Injectable } from '@nestjs/common'; import * as Redis from 'ioredis'; -import type { MiAntenna } from '@/models/Antenna.js'; +import RE2 from 're2'; +import { MiAntenna } from '@/models/Antenna.js'; import type { MiNote } from '@/models/Note.js'; import type { MiUser } from '@/models/User.js'; import { GlobalEventService } from '@/core/GlobalEventService.js'; import * as Acct from '@/misc/acct.js'; import type { Packed } from '@/misc/json-schema.js'; import { DI } from '@/di-symbols.js'; -import type { AntennasRepository, UserListMembershipsRepository } from '@/models/_.js'; +import type { AntennasRepository } from '@/models/_.js'; import { UtilityService } from '@/core/UtilityService.js'; import { bindThis } from '@/decorators.js'; import type { GlobalEvents } from '@/core/GlobalEventService.js'; import { FanoutTimelineService } from '@/core/FanoutTimelineService.js'; import type { OnApplicationShutdown } from '@nestjs/common'; +type AntennaFilter = { + antennaId: string; + src: MiAntenna; + testKeywords(target: string): boolean; +}; + +function matchAntennaKeywords(target: string, keywords: string[][]): boolean { + return keywords.some(and => and.every(keyword => target.includes(keyword))); +} + +function matchAntennaKeywordsCaseInsensitive(target: string, keywords: string[][]): boolean { + const _target = target.toLowerCase(); + return keywords.some(and => and.every(keyword => _target.includes(keyword))); +} + +function matchAntennaKeywordsRegex(target: string, patterns: RE2[]): boolean { + return patterns.every(re => re.test(target)); +} + +export const antennaFilters = { + regex: { + includeAndExclude: (target: string, keywords: RE2[], excludeKeywords: RE2[]): boolean => { + return matchAntennaKeywordsRegex(target, keywords) && !matchAntennaKeywordsRegex(target, excludeKeywords); + }, + includeOnly: (target: string, keywords: RE2[]): boolean => { + return matchAntennaKeywordsRegex(target, keywords); + }, + excludeOnly: (target: string, excludeKeywords: RE2[]): boolean => { + return !matchAntennaKeywordsRegex(target, excludeKeywords); + }, + }, + noRegex: { + caseSensitive: { + includeAndExclude: (target: string, keywords: string[][], excludeKeywords: string[][]): boolean => { + return matchAntennaKeywords(target, keywords) && !matchAntennaKeywords(target, excludeKeywords); + }, + includeOnly: (target: string, keywords: string[][]): boolean => { + return matchAntennaKeywords(target, keywords); + }, + excludeOnly: (target: string, excludeKeywords: string[][]): boolean => { + return !matchAntennaKeywords(target, excludeKeywords); + }, + }, + caseInsensitive: { + includeAndExclude: (target: string, keywords: string[][], excludeKeywords: string[][]): boolean => { + return matchAntennaKeywordsCaseInsensitive(target, keywords) && !matchAntennaKeywordsCaseInsensitive(target, excludeKeywords); + }, + includeOnly: (target: string, keywords: string[][]): boolean => { + return matchAntennaKeywordsCaseInsensitive(target, keywords); + }, + excludeOnly: (target: string, excludeKeywords: string[][]): boolean => { + return !matchAntennaKeywordsCaseInsensitive(target, excludeKeywords); + }, + }, + }, +}; + +function alwaysTrue(): boolean { + return true; +} + +function createAntennaFilter(antenna: MiAntenna): AntennaFilter { + function createTestKeywordsFunction(antenna: MiAntenna): AntennaFilter['testKeywords'] { + // Clean up + const keywords = antenna.keywords + .map(xs => xs.filter(x => x !== '')) + .filter(xs => xs.length > 0); + const excludeKeywords = antenna.excludeKeywords + .map(xs => xs.filter(x => x !== '')) + .filter(xs => xs.length > 0); + + if (antenna.useRegex) { + // 元々はAND検索を行うために2次元配列としてもっていた歴史的経緯がある. + // 正規表現の時は1行に付き1パターンとするため、[n][0]にパターンの内容すべてが格納されているものとして扱う. + const keywordsPatterns = keywords.map(line => new RE2(line[0])); + const excludeKeywordsPatterns = excludeKeywords.map(line => new RE2(line[0])); + + const regex = antennaFilters.regex; + if (keywords.length > 0 && excludeKeywords.length > 0) { + return (target: string) => regex.includeAndExclude(target, keywordsPatterns, excludeKeywordsPatterns); + } else if (keywords.length > 0) { + return (target: string) => regex.includeOnly(target, keywordsPatterns); + } else if (excludeKeywords.length > 0) { + return (target: string) => regex.excludeOnly(target, excludeKeywordsPatterns); + } else { + return alwaysTrue; + } + } else if (antenna.caseSensitive) { + if (keywords.length > 0 && excludeKeywords.length > 0) { + return (target: string) => antennaFilters.noRegex.caseSensitive.includeAndExclude(target, keywords, excludeKeywords); + } else if (keywords.length > 0) { + return (target: string) => antennaFilters.noRegex.caseSensitive.includeOnly(target, keywords); + } else if (excludeKeywords.length > 0) { + return (target: string) => antennaFilters.noRegex.caseSensitive.excludeOnly(target, excludeKeywords); + } else { + return alwaysTrue; + } + } else { + if (keywords.length > 0 && excludeKeywords.length > 0) { + return (target: string) => antennaFilters.noRegex.caseInsensitive.includeAndExclude(target, keywords, excludeKeywords); + } else if (keywords.length > 0) { + return (target: string) => antennaFilters.noRegex.caseInsensitive.includeOnly(target, keywords); + } else if (excludeKeywords.length > 0) { + return (target: string) => antennaFilters.noRegex.caseInsensitive.excludeOnly(target, excludeKeywords); + } else { + return alwaysTrue; + } + } + } + + return { + antennaId: antenna.id, + src: antenna, + testKeywords: createTestKeywordsFunction(antenna), + }; +} + @Injectable() export class AntennaService implements OnApplicationShutdown { - private antennasFetched: boolean; - private antennas: MiAntenna[]; + private filtersFetched: boolean; + private filters: AntennaFilter[]; constructor( @Inject(DI.redisForTimelines) @@ -34,15 +152,12 @@ export class AntennaService implements OnApplicationShutdown { @Inject(DI.antennasRepository) private antennasRepository: AntennasRepository, - @Inject(DI.userListMembershipsRepository) - private userListMembershipsRepository: UserListMembershipsRepository, - private utilityService: UtilityService, private globalEventService: GlobalEventService, private fanoutTimelineService: FanoutTimelineService, ) { - this.antennasFetched = false; - this.antennas = []; + this.filtersFetched = false; + this.filters = []; this.redisForSub.on('message', this.onRedisMessage); } @@ -54,37 +169,24 @@ export class AntennaService implements OnApplicationShutdown { if (obj.channel === 'internal') { const { type, body } = obj.message as GlobalEvents['internal']['payload']; switch (type) { - case 'antennaCreated': - this.antennas.push({ // TODO: このあたりのデシリアライズ処理は各modelファイル内に関数としてexportしたい - ...body, - lastUsedAt: new Date(body.lastUsedAt), - user: null, // joinなカラムは通常取ってこないので - userList: null, // joinなカラムは通常取ってこないので - }); + case 'antennaCreated': { + this.filters.push(createAntennaFilter(MiAntenna.deserialize(body))); break; + } case 'antennaUpdated': { - const idx = this.antennas.findIndex(a => a.id === body.id); + const idx = this.filters.findIndex(a => a.antennaId === body.id); if (idx >= 0) { - this.antennas[idx] = { // TODO: このあたりのデシリアライズ処理は各modelファイル内に関数としてexportしたい - ...body, - lastUsedAt: new Date(body.lastUsedAt), - user: null, // joinなカラムは通常取ってこないので - userList: null, // joinなカラムは通常取ってこないので - }; + this.filters[idx] = createAntennaFilter(MiAntenna.deserialize(body)); } else { // サーバ起動時にactiveじゃなかった場合、リストに持っていないので追加する必要あり - this.antennas.push({ // TODO: このあたりのデシリアライズ処理は各modelファイル内に関数としてexportしたい - ...body, - lastUsedAt: new Date(body.lastUsedAt), - user: null, // joinなカラムは通常取ってこないので - userList: null, // joinなカラムは通常取ってこないので - }); + this.filters.push(createAntennaFilter(MiAntenna.deserialize(body))); } + break; } + case 'antennaDeleted': { + this.filters = this.filters.filter(a => a.antennaId !== body.id); break; - case 'antennaDeleted': - this.antennas = this.antennas.filter(a => a.id !== body.id); - break; + } default: break; } @@ -93,15 +195,17 @@ export class AntennaService implements OnApplicationShutdown { @bindThis public async addNoteToAntennas(note: MiNote, noteUser: { id: MiUser['id']; username: string; host: string | null; isBot: boolean; }): Promise { - const antennas = await this.getAntennas(); - const antennasWithMatchResult = await Promise.all(antennas.map(antenna => this.checkHitAntenna(antenna, note, noteUser).then(hit => [antenna, hit] as const))); - const matchedAntennas = antennasWithMatchResult.filter(([, hit]) => hit).map(([antenna]) => antenna); + const filters = await this.getFilters(); + const checkResults = await Promise.all( + filters.map(filter => this.checkHitAntenna(filter, note, noteUser).then(hit => [filter, hit] as const)), + ); const redisPipeline = this.redisForTimelines.pipeline(); - - for (const antenna of matchedAntennas) { - this.fanoutTimelineService.push(`antennaTimeline:${antenna.id}`, note.id, 200, redisPipeline); - this.globalEventService.publishAntennaStream(antenna.id, 'note', note); + for (const [filter, hit] of checkResults) { + if (hit) { + this.fanoutTimelineService.push(`antennaTimeline:${filter.antennaId}`, note.id, 200, redisPipeline); + this.globalEventService.publishAntennaStream(filter.antennaId, 'note', note); + } } redisPipeline.exec(); @@ -110,100 +214,67 @@ export class AntennaService implements OnApplicationShutdown { // NOTE: フォローしているユーザーのノート、リストのユーザーのノート、グループのユーザーのノート指定はパフォーマンス上の理由で無効になっている @bindThis - public async checkHitAntenna(antenna: MiAntenna, note: (MiNote | Packed<'Note'>), noteUser: { id: MiUser['id']; username: string; host: string | null; isBot: boolean; }): Promise { + private async checkHitAntenna( + filter: AntennaFilter, + note: (MiNote | Packed<'Note'>), + noteUser: { id: MiUser['id']; username: string; host: string | null; isBot: boolean; }, + ): Promise { if (note.visibility === 'specified') return false; if (note.visibility === 'followers') return false; + const antenna = filter.src; if (antenna.excludeBots && noteUser.isBot) return false; - if (antenna.localOnly && noteUser.host != null) return false; - if (!antenna.withReplies && note.replyId != null) return false; if (antenna.src === 'home') { // TODO } else if (antenna.src === 'list') { - if (antenna.userListId == null) return false; - const exists = await this.userListMembershipsRepository.exists({ - where: { - userListId: antenna.userListId, - userId: note.userId, - }, - }); - if (!exists) return false; + // フロントエンドは塞がれているのでコメントアウト + // if (antenna.userListId == null) return false; + // const exists = await this.userListMembershipsRepository.exists({ + // where: { + // userListId: antenna.userListId, + // userId: note.userId, + // }, + // }); + // if (!exists) return false; } else if (antenna.src === 'users') { - const accts = antenna.users.map(x => { - const { username, host } = Acct.parse(x); - return this.utilityService.getFullApAccount(username, host).toLowerCase(); - }); - if (!accts.includes(this.utilityService.getFullApAccount(noteUser.username, noteUser.host).toLowerCase())) return false; + if (!this.isIncludeUser(antenna, noteUser)) return false; } else if (antenna.src === 'users_blacklist') { - const accts = antenna.users.map(x => { - const { username, host } = Acct.parse(x); - return this.utilityService.getFullApAccount(username, host).toLowerCase(); - }); - if (accts.includes(this.utilityService.getFullApAccount(noteUser.username, noteUser.host).toLowerCase())) return false; - } - - const keywords = antenna.keywords - // Clean up - .map(xs => xs.filter(x => x !== '')) - .filter(xs => xs.length > 0); - - if (keywords.length > 0) { - if (note.text == null && note.cw == null) return false; - - const _text = (note.text ?? '') + '\n' + (note.cw ?? ''); - - const matched = keywords.some(and => - and.every(keyword => - antenna.caseSensitive - ? _text.includes(keyword) - : _text.toLowerCase().includes(keyword.toLowerCase()), - )); - - if (!matched) return false; - } - - const excludeKeywords = antenna.excludeKeywords - // Clean up - .map(xs => xs.filter(x => x !== '')) - .filter(xs => xs.length > 0); - - if (excludeKeywords.length > 0) { - if (note.text == null && note.cw == null) return false; - - const _text = (note.text ?? '') + '\n' + (note.cw ?? ''); - - const matched = excludeKeywords.some(and => - and.every(keyword => - antenna.caseSensitive - ? _text.includes(keyword) - : _text.toLowerCase().includes(keyword.toLowerCase()), - )); - - if (matched) return false; + if (this.isIncludeUser(antenna, noteUser)) return false; } if (antenna.withFile) { if (note.fileIds && note.fileIds.length === 0) return false; } - // TODO: eval expression - - return true; + const _text = (note.text ?? '') + '\n' + (note.cw ?? ''); + return filter.testKeywords(_text); } @bindThis - public async getAntennas() { - if (!this.antennasFetched) { - this.antennas = await this.antennasRepository.findBy({ + private async getFilters() { + if (!this.filtersFetched) { + const antennas = await this.antennasRepository.findBy({ isActive: true, }); - this.antennasFetched = true; + this.filters = antennas.map(createAntennaFilter); + this.filtersFetched = true; } - return this.antennas; + return this.filters; + } + + @bindThis + private isIncludeUser(antenna: MiAntenna, noteUser: { username: string; host: string | null; }): boolean { + const antennaUserAccounts = antenna.users.map(x => { + const { username, host } = Acct.parse(x); + return this.utilityService.getFullApAccount(username, host).toLowerCase(); + }); + + const noteUserAccount = this.utilityService.getFullApAccount(noteUser.username, noteUser.host).toLowerCase(); + return antennaUserAccounts.includes(noteUserAccount); } @bindThis diff --git a/packages/backend/src/models/Antenna.ts b/packages/backend/src/models/Antenna.ts index 33e6f48189..8c165b571f 100644 --- a/packages/backend/src/models/Antenna.ts +++ b/packages/backend/src/models/Antenna.ts @@ -3,10 +3,11 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import { PrimaryColumn, Entity, Index, JoinColumn, Column, ManyToOne } from 'typeorm'; -import { id } from './util/id.js'; +import { Column, Entity, Index, JoinColumn, ManyToOne, PrimaryColumn } from 'typeorm'; +import { Serialized } from '@/types.js'; import { MiUser } from './User.js'; import { MiUserList } from './UserList.js'; +import { id } from './util/id.js'; @Entity('antenna') export class MiAntenna { @@ -100,4 +101,20 @@ export class MiAntenna { default: false, }) public localOnly: boolean; + + @Column('boolean', { + default: false, + }) + public useRegex: boolean; + + public static deserialize(data: Serialized): MiAntenna { + return { + ...data, + lastUsedAt: new Date(data.lastUsedAt), + // クエリビルダで明示的にSELECT/JOINしないかぎり設定されない値なのでnullにしておく + user: null, + // クエリビルダで明示的にSELECT/JOINしないかぎり設定されない値なのでnullにしておく + userList: null, + }; + } } From 08b8bf94ef81f596e9d4269e4b623666760c6379 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=81=8A=E3=81=95=E3=82=80=E3=81=AE=E3=81=B2=E3=81=A8?= <46447427+samunohito@users.noreply.github.com> Date: Fri, 28 Feb 2025 19:59:58 +0900 Subject: [PATCH 2/6] fix --- .../1740781670204-antennaRegexSupport.js | 16 ++ packages/backend/src/core/AntennaService.ts | 260 ++++++++++++++---- .../src/core/entities/AntennaEntityService.ts | 1 + packages/backend/src/models/Antenna.ts | 13 +- .../backend/src/models/json-schema/antenna.ts | 4 + .../server/api/endpoints/antennas/create.ts | 128 ++++----- .../server/api/endpoints/antennas/update.ts | 105 +++---- packages/misskey-js/src/autogen/types.ts | 11 +- 8 files changed, 335 insertions(+), 203 deletions(-) create mode 100644 packages/backend/migration/1740781670204-antennaRegexSupport.js diff --git a/packages/backend/migration/1740781670204-antennaRegexSupport.js b/packages/backend/migration/1740781670204-antennaRegexSupport.js new file mode 100644 index 0000000000..806ca29674 --- /dev/null +++ b/packages/backend/migration/1740781670204-antennaRegexSupport.js @@ -0,0 +1,16 @@ +/* + * SPDX-FileCopyrightText: syuilo and misskey-project + * SPDX-License-Identifier: AGPL-3.0-only + */ + +export class AntennaRegexSupport1740781670204 { + name = 'AntennaRegexSupport1740781670204' + + async up(queryRunner) { + await queryRunner.query(`ALTER TABLE "antenna" ADD "useRegex" boolean NOT NULL DEFAULT false`); + } + + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "antenna" DROP COLUMN "useRegex"`); + } +} diff --git a/packages/backend/src/core/AntennaService.ts b/packages/backend/src/core/AntennaService.ts index e8ef9e0cc2..fec0c8f1b8 100644 --- a/packages/backend/src/core/AntennaService.ts +++ b/packages/backend/src/core/AntennaService.ts @@ -6,7 +6,8 @@ import { Inject, Injectable } from '@nestjs/common'; import * as Redis from 'ioredis'; import RE2 from 're2'; -import { MiAntenna } from '@/models/Antenna.js'; +import { IdService } from '@/core/IdService.js'; +import { AntennaSource, MiAntenna } from '@/models/Antenna.js'; import type { MiNote } from '@/models/Note.js'; import type { MiUser } from '@/models/User.js'; import { GlobalEventService } from '@/core/GlobalEventService.js'; @@ -18,6 +19,7 @@ import { UtilityService } from '@/core/UtilityService.js'; import { bindThis } from '@/decorators.js'; import type { GlobalEvents } from '@/core/GlobalEventService.js'; import { FanoutTimelineService } from '@/core/FanoutTimelineService.js'; +import { RoleService } from './RoleService.js'; import type { OnApplicationShutdown } from '@nestjs/common'; type AntennaFilter = { @@ -81,7 +83,7 @@ function alwaysTrue(): boolean { return true; } -function createAntennaFilter(antenna: MiAntenna): AntennaFilter { +export function createAntennaFilter(antenna: MiAntenna): AntennaFilter { function createTestKeywordsFunction(antenna: MiAntenna): AntennaFilter['testKeywords'] { // Clean up const keywords = antenna.keywords @@ -97,13 +99,12 @@ function createAntennaFilter(antenna: MiAntenna): AntennaFilter { const keywordsPatterns = keywords.map(line => new RE2(line[0])); const excludeKeywordsPatterns = excludeKeywords.map(line => new RE2(line[0])); - const regex = antennaFilters.regex; if (keywords.length > 0 && excludeKeywords.length > 0) { - return (target: string) => regex.includeAndExclude(target, keywordsPatterns, excludeKeywordsPatterns); + return (target: string) => antennaFilters.regex.includeAndExclude(target, keywordsPatterns, excludeKeywordsPatterns); } else if (keywords.length > 0) { - return (target: string) => regex.includeOnly(target, keywordsPatterns); + return (target: string) => antennaFilters.regex.includeOnly(target, keywordsPatterns); } else if (excludeKeywords.length > 0) { - return (target: string) => regex.excludeOnly(target, excludeKeywordsPatterns); + return (target: string) => antennaFilters.regex.excludeOnly(target, excludeKeywordsPatterns); } else { return alwaysTrue; } @@ -139,6 +140,19 @@ function createAntennaFilter(antenna: MiAntenna): AntennaFilter { @Injectable() export class AntennaService implements OnApplicationShutdown { + public static AntennaNotFoundError = class extends Error { + }; + public static EmptyKeyWordError = class extends Error { + }; + public static TooManyAntennasError = class extends Error { + }; + public static InvalidRegexPatternError = class extends Error { + constructor(err?: unknown) { + const msg = err instanceof Error ? err.message : undefined; + super(msg); + } + }; + private filtersFetched: boolean; private filters: AntennaFilter[]; @@ -152,9 +166,11 @@ export class AntennaService implements OnApplicationShutdown { @Inject(DI.antennasRepository) private antennasRepository: AntennasRepository, + private roleService: RoleService, private utilityService: UtilityService, private globalEventService: GlobalEventService, private fanoutTimelineService: FanoutTimelineService, + private idService: IdService, ) { this.filtersFetched = false; this.filters = []; @@ -196,13 +212,10 @@ export class AntennaService implements OnApplicationShutdown { @bindThis public async addNoteToAntennas(note: MiNote, noteUser: { id: MiUser['id']; username: string; host: string | null; isBot: boolean; }): Promise { const filters = await this.getFilters(); - const checkResults = await Promise.all( - filters.map(filter => this.checkHitAntenna(filter, note, noteUser).then(hit => [filter, hit] as const)), - ); - const redisPipeline = this.redisForTimelines.pipeline(); - for (const [filter, hit] of checkResults) { - if (hit) { + + for (const filter of filters) { + if (this.checkHitAntenna(filter, note, noteUser)) { this.fanoutTimelineService.push(`antennaTimeline:${filter.antennaId}`, note.id, 200, redisPipeline); this.globalEventService.publishAntennaStream(filter.antennaId, 'note', note); } @@ -211,59 +224,158 @@ export class AntennaService implements OnApplicationShutdown { redisPipeline.exec(); } - // NOTE: フォローしているユーザーのノート、リストのユーザーのノート、グループのユーザーのノート指定はパフォーマンス上の理由で無効になっている - @bindThis - private async checkHitAntenna( - filter: AntennaFilter, - note: (MiNote | Packed<'Note'>), - noteUser: { id: MiUser['id']; username: string; host: string | null; isBot: boolean; }, - ): Promise { - if (note.visibility === 'specified') return false; - if (note.visibility === 'followers') return false; + public async create( + ps: { + name: string; + src: AntennaSource; + keywords: string[][]; + excludeKeywords: string[][]; + users: string[]; + caseSensitive: boolean; + localOnly?: boolean; + excludeBots?: boolean; + useRegex?: boolean; + withReplies: boolean; + withFile: boolean; + }, + me: MiUser, + ): Promise { + this.validateEmptyKeyWord(ps); + this.validateRegexPattern(ps); - const antenna = filter.src; - if (antenna.excludeBots && noteUser.isBot) return false; - if (antenna.localOnly && noteUser.host != null) return false; - if (!antenna.withReplies && note.replyId != null) return false; - - if (antenna.src === 'home') { - // TODO - } else if (antenna.src === 'list') { - // フロントエンドは塞がれているのでコメントアウト - // if (antenna.userListId == null) return false; - // const exists = await this.userListMembershipsRepository.exists({ - // where: { - // userListId: antenna.userListId, - // userId: note.userId, - // }, - // }); - // if (!exists) return false; - } else if (antenna.src === 'users') { - if (!this.isIncludeUser(antenna, noteUser)) return false; - } else if (antenna.src === 'users_blacklist') { - if (this.isIncludeUser(antenna, noteUser)) return false; + const currentAntennasCount = await this.antennasRepository.countBy({ userId: me.id }); + if (currentAntennasCount >= (await this.roleService.getUserPolicies(me.id)).antennaLimit) { + throw new AntennaService.TooManyAntennasError; } - if (antenna.withFile) { - if (note.fileIds && note.fileIds.length === 0) return false; - } + const now = new Date(); + const antenna = await this.antennasRepository.insertOne({ + id: this.idService.gen(now.getTime()), + lastUsedAt: now, + userId: me.id, + name: ps.name, + src: ps.src, + keywords: ps.keywords, + excludeKeywords: ps.excludeKeywords, + users: ps.users, + caseSensitive: ps.caseSensitive, + localOnly: ps.localOnly, + excludeBots: ps.excludeBots, + withReplies: ps.withReplies, + withFile: ps.withFile, + }); - const _text = (note.text ?? '') + '\n' + (note.cw ?? ''); - return filter.testKeywords(_text); + this.globalEventService.publishInternalEvent('antennaCreated', antenna); + + return antenna; } @bindThis - private async getFilters() { - if (!this.filtersFetched) { - const antennas = await this.antennasRepository.findBy({ - isActive: true, - }); - this.filters = antennas.map(createAntennaFilter); - this.filtersFetched = true; + public async update( + ps: { + antennaId: string; + name?: string; + src?: AntennaSource; + keywords?: string[][]; + excludeKeywords?: string[][]; + users?: string[]; + caseSensitive?: boolean; + localOnly?: boolean; + excludeBots?: boolean; + useRegex?: boolean; + withReplies?: boolean; + withFile?: boolean; + }, + me: MiUser, + ): Promise { + this.validateEmptyKeyWord({ + keywords: ps.keywords ?? [], + excludeKeywords: ps.excludeKeywords ?? [], + }); + + this.validateRegexPattern({ + keywords: ps.keywords ?? [], + excludeKeywords: ps.excludeKeywords ?? [], + useRegex: ps.useRegex, + }); + + const antenna = await this.antennasRepository.findOneBy({ + id: ps.antennaId, + userId: me.id, + }); + if (antenna == null) { + throw new AntennaService.AntennaNotFoundError; } - return this.filters; + await this.antennasRepository.update(antenna.id, { + name: ps.name, + src: ps.src, + keywords: ps.keywords, + excludeKeywords: ps.excludeKeywords, + users: ps.users, + caseSensitive: ps.caseSensitive, + localOnly: ps.localOnly, + excludeBots: ps.excludeBots, + withReplies: ps.withReplies, + withFile: ps.withFile, + isActive: true, + lastUsedAt: new Date(), + }); + + this.globalEventService.publishInternalEvent('antennaUpdated', await this.antennasRepository.findOneByOrFail({ id: antenna.id })); + + return antenna; + } + + @bindThis + private validateEmptyKeyWord( + ps: { + keywords: string[][]; + excludeKeywords: string[][]; + }, + ) { + if (ps.keywords.flat().every(x => x === '') && ps.excludeKeywords.flat().every(x => x === '')) { + throw new AntennaService.EmptyKeyWordError; + } + } + + @bindThis + private validateRegexPattern( + ps: { + keywords: string[][]; + excludeKeywords: string[][]; + useRegex?: boolean; + }, + ) { + if (!ps.useRegex) { + return; + } + + // NOTE: 正規表現パターンの場合は2次元配列の2次元目0番地にパターンが格納されていることを前提にする + + if (ps.keywords.length > 0 && ps.keywords.some(x => x.length !== 1)) { + throw new AntennaService.InvalidRegexPatternError(); + } + + if (ps.excludeKeywords.length > 0 && ps.excludeKeywords.some(x => x.length !== 1)) { + throw new AntennaService.InvalidRegexPatternError(); + } + + for (const keywords of [ps.keywords, ps.excludeKeywords]) { + for (const keyword of keywords) { + const regexp = keyword[0].match(/^\/(.+)\/(.*)$/); + if (!regexp) { + throw new AntennaService.InvalidRegexPatternError(); + } + + try { + new RE2(regexp[1], regexp[2]); + } catch (err) { + throw new AntennaService.InvalidRegexPatternError(err); + } + } + } } @bindThis @@ -277,6 +389,44 @@ export class AntennaService implements OnApplicationShutdown { return antennaUserAccounts.includes(noteUserAccount); } + @bindThis + private checkHitAntenna( + filter: AntennaFilter, + note: (MiNote | Packed<'Note'>), + noteUser: { id: MiUser['id']; username: string; host: string | null; isBot: boolean; }, + ): boolean { + if (note.visibility === 'specified') return false; + if (note.visibility === 'followers') return false; + + const antenna = filter.src; + if (antenna.excludeBots && noteUser.isBot) return false; + if (antenna.localOnly && noteUser.host != null) return false; + if (!antenna.withReplies && note.replyId != null) return false; + if (antenna.withFile) { + if (note.fileIds && note.fileIds.length === 0) return false; + } + + if (antenna.src === 'users') { + if (!this.isIncludeUser(antenna, noteUser)) return false; + } else if (antenna.src === 'users_blacklist') { + if (this.isIncludeUser(antenna, noteUser)) return false; + } + + const _text = (note.text ?? '') + '\n' + (note.cw ?? ''); + return filter.testKeywords(_text); + } + + @bindThis + private async getFilters() { + if (!this.filtersFetched) { + const antennas = await this.antennasRepository.findBy({ isActive: true }); + this.filters = antennas.map(createAntennaFilter); + this.filtersFetched = true; + } + + return this.filters; + } + @bindThis public dispose(): void { this.redisForSub.off('message', this.onRedisMessage); diff --git a/packages/backend/src/core/entities/AntennaEntityService.ts b/packages/backend/src/core/entities/AntennaEntityService.ts index e770028af3..3f1c4f669f 100644 --- a/packages/backend/src/core/entities/AntennaEntityService.ts +++ b/packages/backend/src/core/entities/AntennaEntityService.ts @@ -39,6 +39,7 @@ export class AntennaEntityService { caseSensitive: antenna.caseSensitive, localOnly: antenna.localOnly, excludeBots: antenna.excludeBots, + useRegex: antenna.useRegex, withReplies: antenna.withReplies, withFile: antenna.withFile, isActive: antenna.isActive, diff --git a/packages/backend/src/models/Antenna.ts b/packages/backend/src/models/Antenna.ts index 8c165b571f..b16654cd2d 100644 --- a/packages/backend/src/models/Antenna.ts +++ b/packages/backend/src/models/Antenna.ts @@ -9,6 +9,15 @@ import { MiUser } from './User.js'; import { MiUserList } from './UserList.js'; import { id } from './util/id.js'; +export const antennaSources = [ + 'all', + 'users', + 'users_blacklist', + // 'home', // TODO + // 'list', // NOTE: フォローしているユーザーのノート、リストのユーザーのノート、グループのユーザーのノート指定はパフォーマンス上の理由で無効になっている +] as const; +export type AntennaSource = typeof antennaSources[number]; + @Entity('antenna') export class MiAntenna { @PrimaryColumn(id()) @@ -37,8 +46,8 @@ export class MiAntenna { }) public name: string; - @Column('enum', { enum: ['home', 'all', 'users', 'list', 'users_blacklist'] }) - public src: 'home' | 'all' | 'users' | 'list' | 'users_blacklist'; + @Column('enum', { enum: antennaSources }) + public src: AntennaSource; @Column({ ...id(), diff --git a/packages/backend/src/models/json-schema/antenna.ts b/packages/backend/src/models/json-schema/antenna.ts index b5b9a5b42c..a09484facd 100644 --- a/packages/backend/src/models/json-schema/antenna.ts +++ b/packages/backend/src/models/json-schema/antenna.ts @@ -72,6 +72,10 @@ export const packedAntennaSchema = { optional: false, nullable: false, default: false, }, + useRegex: { + type: 'boolean', + optional: false, nullable: false, + }, excludeBots: { type: 'boolean', optional: false, nullable: false, diff --git a/packages/backend/src/server/api/endpoints/antennas/create.ts b/packages/backend/src/server/api/endpoints/antennas/create.ts index e0c8ddcc84..a4e7be8ea8 100644 --- a/packages/backend/src/server/api/endpoints/antennas/create.ts +++ b/packages/backend/src/server/api/endpoints/antennas/create.ts @@ -3,14 +3,11 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import { Inject, Injectable } from '@nestjs/common'; -import { Endpoint } from '@/server/api/endpoint-base.js'; -import { IdService } from '@/core/IdService.js'; -import type { UserListsRepository, AntennasRepository } from '@/models/_.js'; -import { GlobalEventService } from '@/core/GlobalEventService.js'; +import { Injectable } from '@nestjs/common'; +import { AntennaService } from '@/core/AntennaService.js'; import { AntennaEntityService } from '@/core/entities/AntennaEntityService.js'; -import { DI } from '@/di-symbols.js'; -import { RoleService } from '@/core/RoleService.js'; +import { antennaSources } from '@/models/Antenna.js'; +import { Endpoint } from '@/server/api/endpoint-base.js'; import { ApiError } from '../../error.js'; export const meta = { @@ -40,6 +37,13 @@ export const meta = { code: 'EMPTY_KEYWORD', id: '53ee222e-1ddd-4f9a-92e5-9fb82ddb463a', }, + + invalidRegexPattern: { + message: 'Invalid regex pattern.', + code: 'INVALID_REGEX_PATTERN', + id: 'b06d08f4-6434-5faa-0fdd-a2aaf85e9de7', + httpStatusCode: 400, + }, }, res: { @@ -53,91 +57,65 @@ export const paramDef = { type: 'object', properties: { name: { type: 'string', minLength: 1, maxLength: 100 }, - src: { type: 'string', enum: ['home', 'all', 'users', 'list', 'users_blacklist'] }, - userListId: { type: 'string', format: 'misskey:id', nullable: true }, - keywords: { type: 'array', items: { - type: 'array', items: { - type: 'string', + src: { type: 'string', enum: antennaSources }, + keywords: { + type: 'array', + items: { + type: 'array', + items: { type: 'string' }, }, - } }, - excludeKeywords: { type: 'array', items: { - type: 'array', items: { - type: 'string', + }, + excludeKeywords: { + type: 'array', + items: { + type: 'array', + items: { type: 'string' }, }, - } }, - users: { type: 'array', items: { - type: 'string', - } }, + }, + users: { + type: 'array', + items: { type: 'string' }, + }, caseSensitive: { type: 'boolean' }, localOnly: { type: 'boolean' }, excludeBots: { type: 'boolean' }, + useRegex: { type: 'boolean' }, withReplies: { type: 'boolean' }, withFile: { type: 'boolean' }, }, - required: ['name', 'src', 'keywords', 'excludeKeywords', 'users', 'caseSensitive', 'withReplies', 'withFile'], + required: [ + 'name', + 'src', + 'keywords', + 'excludeKeywords', + 'users', + 'caseSensitive', + 'withReplies', + 'withFile', + ], } as const; @Injectable() export default class extends Endpoint { // eslint-disable-line import/no-default-export constructor( - @Inject(DI.antennasRepository) - private antennasRepository: AntennasRepository, - - @Inject(DI.userListsRepository) - private userListsRepository: UserListsRepository, - - private antennaEntityService: AntennaEntityService, - private roleService: RoleService, - private idService: IdService, - private globalEventService: GlobalEventService, + private readonly antennaEntityService: AntennaEntityService, + private readonly antennaService: AntennaService, ) { super(meta, paramDef, async (ps, me) => { - if (ps.keywords.flat().every(x => x === '') && ps.excludeKeywords.flat().every(x => x === '')) { - throw new ApiError(meta.errors.emptyKeyword); - } - - const currentAntennasCount = await this.antennasRepository.countBy({ - userId: me.id, - }); - if (currentAntennasCount >= (await this.roleService.getUserPolicies(me.id)).antennaLimit) { - throw new ApiError(meta.errors.tooManyAntennas); - } - - let userList; - - if (ps.src === 'list' && ps.userListId) { - userList = await this.userListsRepository.findOneBy({ - id: ps.userListId, - userId: me.id, - }); - - if (userList == null) { - throw new ApiError(meta.errors.noSuchUserList); + try { + const antenna = await this.antennaService.create(ps, me); + return this.antennaEntityService.pack(antenna); + } catch (e) { + if (e instanceof AntennaService.EmptyKeyWordError) { + throw new ApiError(meta.errors.emptyKeyword); + } else if (e instanceof AntennaService.TooManyAntennasError) { + throw new ApiError(meta.errors.tooManyAntennas); + } else if (e instanceof AntennaService.InvalidRegexPatternError) { + throw new ApiError(meta.errors.invalidRegexPattern); + } else { + throw e; } } - - const now = new Date(); - - const antenna = await this.antennasRepository.insertOne({ - id: this.idService.gen(now.getTime()), - lastUsedAt: now, - userId: me.id, - name: ps.name, - src: ps.src, - userListId: userList ? userList.id : null, - keywords: ps.keywords, - excludeKeywords: ps.excludeKeywords, - users: ps.users, - caseSensitive: ps.caseSensitive, - localOnly: ps.localOnly, - excludeBots: ps.excludeBots, - withReplies: ps.withReplies, - withFile: ps.withFile, - }); - - this.globalEventService.publishInternalEvent('antennaCreated', antenna); - - return await this.antennaEntityService.pack(antenna); }); } } diff --git a/packages/backend/src/server/api/endpoints/antennas/update.ts b/packages/backend/src/server/api/endpoints/antennas/update.ts index 10f26b1912..1290296857 100644 --- a/packages/backend/src/server/api/endpoints/antennas/update.ts +++ b/packages/backend/src/server/api/endpoints/antennas/update.ts @@ -4,6 +4,8 @@ */ import { Inject, Injectable } from '@nestjs/common'; +import { AntennaService } from '@/core/AntennaService.js'; +import { antennaSources } from '@/models/Antenna.js'; import { Endpoint } from '@/server/api/endpoint-base.js'; import type { AntennasRepository, UserListsRepository } from '@/models/_.js'; import { GlobalEventService } from '@/core/GlobalEventService.js'; @@ -38,6 +40,13 @@ export const meta = { code: 'EMPTY_KEYWORD', id: '721aaff6-4e1b-4d88-8de6-877fae9f68c4', }, + + invalidRegexPattern: { + message: 'Invalid regex pattern.', + code: 'INVALID_REGEX_PATTERN', + id: 'dbb44ec3-5d15-508d-e6b2-71f3794c6a41', + httpStatusCode: 400, + }, }, res: { @@ -52,24 +61,29 @@ export const paramDef = { properties: { antennaId: { type: 'string', format: 'misskey:id' }, name: { type: 'string', minLength: 1, maxLength: 100 }, - src: { type: 'string', enum: ['home', 'all', 'users', 'list', 'users_blacklist'] }, - userListId: { type: 'string', format: 'misskey:id', nullable: true }, - keywords: { type: 'array', items: { - type: 'array', items: { - type: 'string', + src: { type: 'string', enum: antennaSources }, + keywords: { + type: 'array', + items: { + type: 'array', + items: { type: 'string' }, }, - } }, - excludeKeywords: { type: 'array', items: { - type: 'array', items: { - type: 'string', + }, + excludeKeywords: { + type: 'array', + items: { + type: 'array', + items: { type: 'string' }, }, - } }, - users: { type: 'array', items: { - type: 'string', - } }, + }, + users: { + type: 'array', + items: { type: 'string' }, + }, caseSensitive: { type: 'boolean' }, localOnly: { type: 'boolean' }, excludeBots: { type: 'boolean' }, + useRegex: { type: 'boolean' }, withReplies: { type: 'boolean' }, withFile: { type: 'boolean' }, }, @@ -79,63 +93,24 @@ export const paramDef = { @Injectable() export default class extends Endpoint { // eslint-disable-line import/no-default-export constructor( - @Inject(DI.antennasRepository) - private antennasRepository: AntennasRepository, - - @Inject(DI.userListsRepository) - private userListsRepository: UserListsRepository, - - private antennaEntityService: AntennaEntityService, - private globalEventService: GlobalEventService, + private readonly antennaEntityService: AntennaEntityService, + private readonly antennaService: AntennaService, ) { super(meta, paramDef, async (ps, me) => { - if (ps.keywords && ps.excludeKeywords) { - if (ps.keywords.flat().every(x => x === '') && ps.excludeKeywords.flat().every(x => x === '')) { + try { + const antenna = await this.antennaService.update(ps, me); + return this.antennaEntityService.pack(antenna); + } catch (e) { + if (e instanceof AntennaService.EmptyKeyWordError) { throw new ApiError(meta.errors.emptyKeyword); + } else if (e instanceof AntennaService.AntennaNotFoundError) { + throw new ApiError(meta.errors.noSuchAntenna); + } else if (e instanceof AntennaService.InvalidRegexPatternError) { + throw new ApiError(meta.errors.invalidRegexPattern); + } else { + throw e; } } - // Fetch the antenna - const antenna = await this.antennasRepository.findOneBy({ - id: ps.antennaId, - userId: me.id, - }); - - if (antenna == null) { - throw new ApiError(meta.errors.noSuchAntenna); - } - - let userList; - - if ((ps.src === 'list' || antenna.src === 'list') && ps.userListId) { - userList = await this.userListsRepository.findOneBy({ - id: ps.userListId, - userId: me.id, - }); - - if (userList == null) { - throw new ApiError(meta.errors.noSuchUserList); - } - } - - await this.antennasRepository.update(antenna.id, { - name: ps.name, - src: ps.src, - userListId: ps.userListId !== undefined ? userList ? userList.id : null : undefined, - keywords: ps.keywords, - excludeKeywords: ps.excludeKeywords, - users: ps.users, - caseSensitive: ps.caseSensitive, - localOnly: ps.localOnly, - excludeBots: ps.excludeBots, - withReplies: ps.withReplies, - withFile: ps.withFile, - isActive: true, - lastUsedAt: new Date(), - }); - - this.globalEventService.publishInternalEvent('antennaUpdated', await this.antennasRepository.findOneByOrFail({ id: antenna.id })); - - return await this.antennaEntityService.pack(antenna.id); }); } } diff --git a/packages/misskey-js/src/autogen/types.ts b/packages/misskey-js/src/autogen/types.ts index ae7a8c7440..18ca9f6329 100644 --- a/packages/misskey-js/src/autogen/types.ts +++ b/packages/misskey-js/src/autogen/types.ts @@ -4651,6 +4651,7 @@ export type components = { caseSensitive: boolean; /** @default false */ localOnly: boolean; + useRegex: boolean; /** @default false */ excludeBots: boolean; /** @default false */ @@ -10918,15 +10919,14 @@ export type operations = { 'application/json': { name: string; /** @enum {string} */ - src: 'home' | 'all' | 'users' | 'list' | 'users_blacklist'; - /** Format: misskey:id */ - userListId?: string | null; + src: 'all' | 'users' | 'users_blacklist'; keywords: string[][]; excludeKeywords: string[][]; users: string[]; caseSensitive: boolean; localOnly?: boolean; excludeBots?: boolean; + useRegex?: boolean; withReplies: boolean; withFile: boolean; }; @@ -11199,15 +11199,14 @@ export type operations = { antennaId: string; name?: string; /** @enum {string} */ - src?: 'home' | 'all' | 'users' | 'list' | 'users_blacklist'; - /** Format: misskey:id */ - userListId?: string | null; + src?: 'all' | 'users' | 'users_blacklist'; keywords?: string[][]; excludeKeywords?: string[][]; users?: string[]; caseSensitive?: boolean; localOnly?: boolean; excludeBots?: boolean; + useRegex?: boolean; withReplies?: boolean; withFile?: boolean; }; From dfa66e6e2a438044b24c3e941eabc6beccccf628 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=81=8A=E3=81=95=E3=82=80=E3=81=AE=E3=81=B2=E3=81=A8?= <46447427+samunohito@users.noreply.github.com> Date: Sat, 1 Mar 2025 09:00:35 +0900 Subject: [PATCH 3/6] fix --- locales/index.d.ts | 8 +++ locales/ja-JP.yml | 2 + .../backend/src/models/json-schema/antenna.ts | 4 +- .../src/components/MkAntennaEditor.vue | 67 ++++++++++++++----- packages/misskey-js/src/autogen/types.ts | 2 +- 5 files changed, 63 insertions(+), 20 deletions(-) diff --git a/locales/index.d.ts b/locales/index.d.ts index 74e3cdeceb..06db0a9d7a 100644 --- a/locales/index.d.ts +++ b/locales/index.d.ts @@ -1706,6 +1706,14 @@ export interface Locale extends ILocale { * Botアカウントを除外 */ "antennaExcludeBots": string; + /** + * 正規表現を使用する + */ + "antennaUseRegex": string; + /** + * {src}の{line}行目にエラーがあります。 + */ + "antennaUseRegexError": ParameterizedString<"src" | "line">; /** * スペースで区切るとAND指定になり、改行で区切るとOR指定になります */ diff --git a/locales/ja-JP.yml b/locales/ja-JP.yml index 270b5fc265..266cf720fb 100644 --- a/locales/ja-JP.yml +++ b/locales/ja-JP.yml @@ -422,6 +422,8 @@ antennaSource: "受信ソース" antennaKeywords: "受信キーワード" antennaExcludeKeywords: "除外キーワード" antennaExcludeBots: "Botアカウントを除外" +antennaUseRegex: "正規表現を使用する" +antennaUseRegexError: "{src}の{line}行目にエラーがあります。" antennaKeywordsDescription: "スペースで区切るとAND指定になり、改行で区切るとOR指定になります" notifyAntenna: "新しいノートを通知する" withFileAntenna: "ファイルが添付されたノートのみ" diff --git a/packages/backend/src/models/json-schema/antenna.ts b/packages/backend/src/models/json-schema/antenna.ts index a09484facd..72623cbd8b 100644 --- a/packages/backend/src/models/json-schema/antenna.ts +++ b/packages/backend/src/models/json-schema/antenna.ts @@ -3,6 +3,8 @@ * SPDX-License-Identifier: AGPL-3.0-only */ +import { antennaSources } from '@/models/Antenna.js'; + export const packedAntennaSchema = { type: 'object', properties: { @@ -47,7 +49,7 @@ export const packedAntennaSchema = { src: { type: 'string', optional: false, nullable: false, - enum: ['home', 'all', 'users', 'list', 'users_blacklist'], + enum: antennaSources, }, userListId: { type: 'string', diff --git a/packages/frontend/src/components/MkAntennaEditor.vue b/packages/frontend/src/components/MkAntennaEditor.vue index e622d57f1e..1e59f21636 100644 --- a/packages/frontend/src/components/MkAntennaEditor.vue +++ b/packages/frontend/src/components/MkAntennaEditor.vue @@ -13,21 +13,16 @@ SPDX-License-Identifier: AGPL-3.0-only - - - - - - - + {{ i18n.ts.antennaExcludeBots }} {{ i18n.ts.withReplies }} + {{ i18n.ts.antennaUseRegex }} @@ -53,6 +48,7 @@ SPDX-License-Identifier: AGPL-3.0-only