This commit is contained in:
おさむのひと 2025-02-28 19:59:58 +09:00
parent 7a8ab424d4
commit 08b8bf94ef
8 changed files with 335 additions and 203 deletions

View File

@ -0,0 +1,16 @@
/*
* SPDX-FileCopyrightText: syuilo and misskey-project
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class AntennaRegexSupport1740781670204 {
name = 'AntennaRegexSupport1740781670204'
async up(queryRunner) {
await queryRunner.query(`ALTER TABLE "antenna" ADD "useRegex" boolean NOT NULL DEFAULT false`);
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "antenna" DROP COLUMN "useRegex"`);
}
}

View File

@ -6,7 +6,8 @@
import { Inject, Injectable } from '@nestjs/common';
import * as Redis from 'ioredis';
import RE2 from 're2';
import { MiAntenna } from '@/models/Antenna.js';
import { IdService } from '@/core/IdService.js';
import { AntennaSource, MiAntenna } from '@/models/Antenna.js';
import type { MiNote } from '@/models/Note.js';
import type { MiUser } from '@/models/User.js';
import { GlobalEventService } from '@/core/GlobalEventService.js';
@ -18,6 +19,7 @@ import { UtilityService } from '@/core/UtilityService.js';
import { bindThis } from '@/decorators.js';
import type { GlobalEvents } from '@/core/GlobalEventService.js';
import { FanoutTimelineService } from '@/core/FanoutTimelineService.js';
import { RoleService } from './RoleService.js';
import type { OnApplicationShutdown } from '@nestjs/common';
type AntennaFilter = {
@ -81,7 +83,7 @@ function alwaysTrue(): boolean {
return true;
}
function createAntennaFilter(antenna: MiAntenna): AntennaFilter {
export function createAntennaFilter(antenna: MiAntenna): AntennaFilter {
function createTestKeywordsFunction(antenna: MiAntenna): AntennaFilter['testKeywords'] {
// Clean up
const keywords = antenna.keywords
@ -97,13 +99,12 @@ function createAntennaFilter(antenna: MiAntenna): AntennaFilter {
const keywordsPatterns = keywords.map(line => new RE2(line[0]));
const excludeKeywordsPatterns = excludeKeywords.map(line => new RE2(line[0]));
const regex = antennaFilters.regex;
if (keywords.length > 0 && excludeKeywords.length > 0) {
return (target: string) => regex.includeAndExclude(target, keywordsPatterns, excludeKeywordsPatterns);
return (target: string) => antennaFilters.regex.includeAndExclude(target, keywordsPatterns, excludeKeywordsPatterns);
} else if (keywords.length > 0) {
return (target: string) => regex.includeOnly(target, keywordsPatterns);
return (target: string) => antennaFilters.regex.includeOnly(target, keywordsPatterns);
} else if (excludeKeywords.length > 0) {
return (target: string) => regex.excludeOnly(target, excludeKeywordsPatterns);
return (target: string) => antennaFilters.regex.excludeOnly(target, excludeKeywordsPatterns);
} else {
return alwaysTrue;
}
@ -139,6 +140,19 @@ function createAntennaFilter(antenna: MiAntenna): AntennaFilter {
@Injectable()
export class AntennaService implements OnApplicationShutdown {
public static AntennaNotFoundError = class extends Error {
};
public static EmptyKeyWordError = class extends Error {
};
public static TooManyAntennasError = class extends Error {
};
public static InvalidRegexPatternError = class extends Error {
constructor(err?: unknown) {
const msg = err instanceof Error ? err.message : undefined;
super(msg);
}
};
private filtersFetched: boolean;
private filters: AntennaFilter[];
@ -152,9 +166,11 @@ export class AntennaService implements OnApplicationShutdown {
@Inject(DI.antennasRepository)
private antennasRepository: AntennasRepository,
private roleService: RoleService,
private utilityService: UtilityService,
private globalEventService: GlobalEventService,
private fanoutTimelineService: FanoutTimelineService,
private idService: IdService,
) {
this.filtersFetched = false;
this.filters = [];
@ -196,13 +212,10 @@ export class AntennaService implements OnApplicationShutdown {
@bindThis
public async addNoteToAntennas(note: MiNote, noteUser: { id: MiUser['id']; username: string; host: string | null; isBot: boolean; }): Promise<void> {
const filters = await this.getFilters();
const checkResults = await Promise.all(
filters.map(filter => this.checkHitAntenna(filter, note, noteUser).then(hit => [filter, hit] as const)),
);
const redisPipeline = this.redisForTimelines.pipeline();
for (const [filter, hit] of checkResults) {
if (hit) {
for (const filter of filters) {
if (this.checkHitAntenna(filter, note, noteUser)) {
this.fanoutTimelineService.push(`antennaTimeline:${filter.antennaId}`, note.id, 200, redisPipeline);
this.globalEventService.publishAntennaStream(filter.antennaId, 'note', note);
}
@ -211,59 +224,158 @@ export class AntennaService implements OnApplicationShutdown {
redisPipeline.exec();
}
// NOTE: フォローしているユーザーのノート、リストのユーザーのノート、グループのユーザーのノート指定はパフォーマンス上の理由で無効になっている
@bindThis
private async checkHitAntenna(
filter: AntennaFilter,
note: (MiNote | Packed<'Note'>),
noteUser: { id: MiUser['id']; username: string; host: string | null; isBot: boolean; },
): Promise<boolean> {
if (note.visibility === 'specified') return false;
if (note.visibility === 'followers') return false;
public async create(
ps: {
name: string;
src: AntennaSource;
keywords: string[][];
excludeKeywords: string[][];
users: string[];
caseSensitive: boolean;
localOnly?: boolean;
excludeBots?: boolean;
useRegex?: boolean;
withReplies: boolean;
withFile: boolean;
},
me: MiUser,
): Promise<MiAntenna> {
this.validateEmptyKeyWord(ps);
this.validateRegexPattern(ps);
const antenna = filter.src;
if (antenna.excludeBots && noteUser.isBot) return false;
if (antenna.localOnly && noteUser.host != null) return false;
if (!antenna.withReplies && note.replyId != null) return false;
if (antenna.src === 'home') {
// TODO
} else if (antenna.src === 'list') {
// フロントエンドは塞がれているのでコメントアウト
// if (antenna.userListId == null) return false;
// const exists = await this.userListMembershipsRepository.exists({
// where: {
// userListId: antenna.userListId,
// userId: note.userId,
// },
// });
// if (!exists) return false;
} else if (antenna.src === 'users') {
if (!this.isIncludeUser(antenna, noteUser)) return false;
} else if (antenna.src === 'users_blacklist') {
if (this.isIncludeUser(antenna, noteUser)) return false;
const currentAntennasCount = await this.antennasRepository.countBy({ userId: me.id });
if (currentAntennasCount >= (await this.roleService.getUserPolicies(me.id)).antennaLimit) {
throw new AntennaService.TooManyAntennasError;
}
if (antenna.withFile) {
if (note.fileIds && note.fileIds.length === 0) return false;
}
const _text = (note.text ?? '') + '\n' + (note.cw ?? '');
return filter.testKeywords(_text);
}
@bindThis
private async getFilters() {
if (!this.filtersFetched) {
const antennas = await this.antennasRepository.findBy({
isActive: true,
const now = new Date();
const antenna = await this.antennasRepository.insertOne({
id: this.idService.gen(now.getTime()),
lastUsedAt: now,
userId: me.id,
name: ps.name,
src: ps.src,
keywords: ps.keywords,
excludeKeywords: ps.excludeKeywords,
users: ps.users,
caseSensitive: ps.caseSensitive,
localOnly: ps.localOnly,
excludeBots: ps.excludeBots,
withReplies: ps.withReplies,
withFile: ps.withFile,
});
this.filters = antennas.map(createAntennaFilter);
this.filtersFetched = true;
this.globalEventService.publishInternalEvent('antennaCreated', antenna);
return antenna;
}
return this.filters;
@bindThis
public async update(
ps: {
antennaId: string;
name?: string;
src?: AntennaSource;
keywords?: string[][];
excludeKeywords?: string[][];
users?: string[];
caseSensitive?: boolean;
localOnly?: boolean;
excludeBots?: boolean;
useRegex?: boolean;
withReplies?: boolean;
withFile?: boolean;
},
me: MiUser,
): Promise<MiAntenna> {
this.validateEmptyKeyWord({
keywords: ps.keywords ?? [],
excludeKeywords: ps.excludeKeywords ?? [],
});
this.validateRegexPattern({
keywords: ps.keywords ?? [],
excludeKeywords: ps.excludeKeywords ?? [],
useRegex: ps.useRegex,
});
const antenna = await this.antennasRepository.findOneBy({
id: ps.antennaId,
userId: me.id,
});
if (antenna == null) {
throw new AntennaService.AntennaNotFoundError;
}
await this.antennasRepository.update(antenna.id, {
name: ps.name,
src: ps.src,
keywords: ps.keywords,
excludeKeywords: ps.excludeKeywords,
users: ps.users,
caseSensitive: ps.caseSensitive,
localOnly: ps.localOnly,
excludeBots: ps.excludeBots,
withReplies: ps.withReplies,
withFile: ps.withFile,
isActive: true,
lastUsedAt: new Date(),
});
this.globalEventService.publishInternalEvent('antennaUpdated', await this.antennasRepository.findOneByOrFail({ id: antenna.id }));
return antenna;
}
@bindThis
private validateEmptyKeyWord(
ps: {
keywords: string[][];
excludeKeywords: string[][];
},
) {
if (ps.keywords.flat().every(x => x === '') && ps.excludeKeywords.flat().every(x => x === '')) {
throw new AntennaService.EmptyKeyWordError;
}
}
@bindThis
private validateRegexPattern(
ps: {
keywords: string[][];
excludeKeywords: string[][];
useRegex?: boolean;
},
) {
if (!ps.useRegex) {
return;
}
// NOTE: 正規表現パターンの場合は2次元配列の2次元目0番地にパターンが格納されていることを前提にする
if (ps.keywords.length > 0 && ps.keywords.some(x => x.length !== 1)) {
throw new AntennaService.InvalidRegexPatternError();
}
if (ps.excludeKeywords.length > 0 && ps.excludeKeywords.some(x => x.length !== 1)) {
throw new AntennaService.InvalidRegexPatternError();
}
for (const keywords of [ps.keywords, ps.excludeKeywords]) {
for (const keyword of keywords) {
const regexp = keyword[0].match(/^\/(.+)\/(.*)$/);
if (!regexp) {
throw new AntennaService.InvalidRegexPatternError();
}
try {
new RE2(regexp[1], regexp[2]);
} catch (err) {
throw new AntennaService.InvalidRegexPatternError(err);
}
}
}
}
@bindThis
@ -277,6 +389,44 @@ export class AntennaService implements OnApplicationShutdown {
return antennaUserAccounts.includes(noteUserAccount);
}
@bindThis
private checkHitAntenna(
filter: AntennaFilter,
note: (MiNote | Packed<'Note'>),
noteUser: { id: MiUser['id']; username: string; host: string | null; isBot: boolean; },
): boolean {
if (note.visibility === 'specified') return false;
if (note.visibility === 'followers') return false;
const antenna = filter.src;
if (antenna.excludeBots && noteUser.isBot) return false;
if (antenna.localOnly && noteUser.host != null) return false;
if (!antenna.withReplies && note.replyId != null) return false;
if (antenna.withFile) {
if (note.fileIds && note.fileIds.length === 0) return false;
}
if (antenna.src === 'users') {
if (!this.isIncludeUser(antenna, noteUser)) return false;
} else if (antenna.src === 'users_blacklist') {
if (this.isIncludeUser(antenna, noteUser)) return false;
}
const _text = (note.text ?? '') + '\n' + (note.cw ?? '');
return filter.testKeywords(_text);
}
@bindThis
private async getFilters() {
if (!this.filtersFetched) {
const antennas = await this.antennasRepository.findBy({ isActive: true });
this.filters = antennas.map(createAntennaFilter);
this.filtersFetched = true;
}
return this.filters;
}
@bindThis
public dispose(): void {
this.redisForSub.off('message', this.onRedisMessage);

View File

@ -39,6 +39,7 @@ export class AntennaEntityService {
caseSensitive: antenna.caseSensitive,
localOnly: antenna.localOnly,
excludeBots: antenna.excludeBots,
useRegex: antenna.useRegex,
withReplies: antenna.withReplies,
withFile: antenna.withFile,
isActive: antenna.isActive,

View File

@ -9,6 +9,15 @@ import { MiUser } from './User.js';
import { MiUserList } from './UserList.js';
import { id } from './util/id.js';
export const antennaSources = [
'all',
'users',
'users_blacklist',
// 'home', // TODO
// 'list', // NOTE: フォローしているユーザーのノート、リストのユーザーのノート、グループのユーザーのノート指定はパフォーマンス上の理由で無効になっている
] as const;
export type AntennaSource = typeof antennaSources[number];
@Entity('antenna')
export class MiAntenna {
@PrimaryColumn(id())
@ -37,8 +46,8 @@ export class MiAntenna {
})
public name: string;
@Column('enum', { enum: ['home', 'all', 'users', 'list', 'users_blacklist'] })
public src: 'home' | 'all' | 'users' | 'list' | 'users_blacklist';
@Column('enum', { enum: antennaSources })
public src: AntennaSource;
@Column({
...id(),

View File

@ -72,6 +72,10 @@ export const packedAntennaSchema = {
optional: false, nullable: false,
default: false,
},
useRegex: {
type: 'boolean',
optional: false, nullable: false,
},
excludeBots: {
type: 'boolean',
optional: false, nullable: false,

View File

@ -3,14 +3,11 @@
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { Inject, Injectable } from '@nestjs/common';
import { Endpoint } from '@/server/api/endpoint-base.js';
import { IdService } from '@/core/IdService.js';
import type { UserListsRepository, AntennasRepository } from '@/models/_.js';
import { GlobalEventService } from '@/core/GlobalEventService.js';
import { Injectable } from '@nestjs/common';
import { AntennaService } from '@/core/AntennaService.js';
import { AntennaEntityService } from '@/core/entities/AntennaEntityService.js';
import { DI } from '@/di-symbols.js';
import { RoleService } from '@/core/RoleService.js';
import { antennaSources } from '@/models/Antenna.js';
import { Endpoint } from '@/server/api/endpoint-base.js';
import { ApiError } from '../../error.js';
export const meta = {
@ -40,6 +37,13 @@ export const meta = {
code: 'EMPTY_KEYWORD',
id: '53ee222e-1ddd-4f9a-92e5-9fb82ddb463a',
},
invalidRegexPattern: {
message: 'Invalid regex pattern.',
code: 'INVALID_REGEX_PATTERN',
id: 'b06d08f4-6434-5faa-0fdd-a2aaf85e9de7',
httpStatusCode: 400,
},
},
res: {
@ -53,91 +57,65 @@ export const paramDef = {
type: 'object',
properties: {
name: { type: 'string', minLength: 1, maxLength: 100 },
src: { type: 'string', enum: ['home', 'all', 'users', 'list', 'users_blacklist'] },
userListId: { type: 'string', format: 'misskey:id', nullable: true },
keywords: { type: 'array', items: {
type: 'array', items: {
type: 'string',
src: { type: 'string', enum: antennaSources },
keywords: {
type: 'array',
items: {
type: 'array',
items: { type: 'string' },
},
} },
excludeKeywords: { type: 'array', items: {
type: 'array', items: {
type: 'string',
},
} },
users: { type: 'array', items: {
type: 'string',
} },
excludeKeywords: {
type: 'array',
items: {
type: 'array',
items: { type: 'string' },
},
},
users: {
type: 'array',
items: { type: 'string' },
},
caseSensitive: { type: 'boolean' },
localOnly: { type: 'boolean' },
excludeBots: { type: 'boolean' },
useRegex: { type: 'boolean' },
withReplies: { type: 'boolean' },
withFile: { type: 'boolean' },
},
required: ['name', 'src', 'keywords', 'excludeKeywords', 'users', 'caseSensitive', 'withReplies', 'withFile'],
required: [
'name',
'src',
'keywords',
'excludeKeywords',
'users',
'caseSensitive',
'withReplies',
'withFile',
],
} as const;
@Injectable()
export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-disable-line import/no-default-export
constructor(
@Inject(DI.antennasRepository)
private antennasRepository: AntennasRepository,
@Inject(DI.userListsRepository)
private userListsRepository: UserListsRepository,
private antennaEntityService: AntennaEntityService,
private roleService: RoleService,
private idService: IdService,
private globalEventService: GlobalEventService,
private readonly antennaEntityService: AntennaEntityService,
private readonly antennaService: AntennaService,
) {
super(meta, paramDef, async (ps, me) => {
if (ps.keywords.flat().every(x => x === '') && ps.excludeKeywords.flat().every(x => x === '')) {
try {
const antenna = await this.antennaService.create(ps, me);
return this.antennaEntityService.pack(antenna);
} catch (e) {
if (e instanceof AntennaService.EmptyKeyWordError) {
throw new ApiError(meta.errors.emptyKeyword);
}
const currentAntennasCount = await this.antennasRepository.countBy({
userId: me.id,
});
if (currentAntennasCount >= (await this.roleService.getUserPolicies(me.id)).antennaLimit) {
} else if (e instanceof AntennaService.TooManyAntennasError) {
throw new ApiError(meta.errors.tooManyAntennas);
}
let userList;
if (ps.src === 'list' && ps.userListId) {
userList = await this.userListsRepository.findOneBy({
id: ps.userListId,
userId: me.id,
});
if (userList == null) {
throw new ApiError(meta.errors.noSuchUserList);
} else if (e instanceof AntennaService.InvalidRegexPatternError) {
throw new ApiError(meta.errors.invalidRegexPattern);
} else {
throw e;
}
}
const now = new Date();
const antenna = await this.antennasRepository.insertOne({
id: this.idService.gen(now.getTime()),
lastUsedAt: now,
userId: me.id,
name: ps.name,
src: ps.src,
userListId: userList ? userList.id : null,
keywords: ps.keywords,
excludeKeywords: ps.excludeKeywords,
users: ps.users,
caseSensitive: ps.caseSensitive,
localOnly: ps.localOnly,
excludeBots: ps.excludeBots,
withReplies: ps.withReplies,
withFile: ps.withFile,
});
this.globalEventService.publishInternalEvent('antennaCreated', antenna);
return await this.antennaEntityService.pack(antenna);
});
}
}

View File

@ -4,6 +4,8 @@
*/
import { Inject, Injectable } from '@nestjs/common';
import { AntennaService } from '@/core/AntennaService.js';
import { antennaSources } from '@/models/Antenna.js';
import { Endpoint } from '@/server/api/endpoint-base.js';
import type { AntennasRepository, UserListsRepository } from '@/models/_.js';
import { GlobalEventService } from '@/core/GlobalEventService.js';
@ -38,6 +40,13 @@ export const meta = {
code: 'EMPTY_KEYWORD',
id: '721aaff6-4e1b-4d88-8de6-877fae9f68c4',
},
invalidRegexPattern: {
message: 'Invalid regex pattern.',
code: 'INVALID_REGEX_PATTERN',
id: 'dbb44ec3-5d15-508d-e6b2-71f3794c6a41',
httpStatusCode: 400,
},
},
res: {
@ -52,24 +61,29 @@ export const paramDef = {
properties: {
antennaId: { type: 'string', format: 'misskey:id' },
name: { type: 'string', minLength: 1, maxLength: 100 },
src: { type: 'string', enum: ['home', 'all', 'users', 'list', 'users_blacklist'] },
userListId: { type: 'string', format: 'misskey:id', nullable: true },
keywords: { type: 'array', items: {
type: 'array', items: {
type: 'string',
src: { type: 'string', enum: antennaSources },
keywords: {
type: 'array',
items: {
type: 'array',
items: { type: 'string' },
},
} },
excludeKeywords: { type: 'array', items: {
type: 'array', items: {
type: 'string',
},
} },
users: { type: 'array', items: {
type: 'string',
} },
excludeKeywords: {
type: 'array',
items: {
type: 'array',
items: { type: 'string' },
},
},
users: {
type: 'array',
items: { type: 'string' },
},
caseSensitive: { type: 'boolean' },
localOnly: { type: 'boolean' },
excludeBots: { type: 'boolean' },
useRegex: { type: 'boolean' },
withReplies: { type: 'boolean' },
withFile: { type: 'boolean' },
},
@ -79,63 +93,24 @@ export const paramDef = {
@Injectable()
export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-disable-line import/no-default-export
constructor(
@Inject(DI.antennasRepository)
private antennasRepository: AntennasRepository,
@Inject(DI.userListsRepository)
private userListsRepository: UserListsRepository,
private antennaEntityService: AntennaEntityService,
private globalEventService: GlobalEventService,
private readonly antennaEntityService: AntennaEntityService,
private readonly antennaService: AntennaService,
) {
super(meta, paramDef, async (ps, me) => {
if (ps.keywords && ps.excludeKeywords) {
if (ps.keywords.flat().every(x => x === '') && ps.excludeKeywords.flat().every(x => x === '')) {
try {
const antenna = await this.antennaService.update(ps, me);
return this.antennaEntityService.pack(antenna);
} catch (e) {
if (e instanceof AntennaService.EmptyKeyWordError) {
throw new ApiError(meta.errors.emptyKeyword);
}
}
// Fetch the antenna
const antenna = await this.antennasRepository.findOneBy({
id: ps.antennaId,
userId: me.id,
});
if (antenna == null) {
} else if (e instanceof AntennaService.AntennaNotFoundError) {
throw new ApiError(meta.errors.noSuchAntenna);
}
let userList;
if ((ps.src === 'list' || antenna.src === 'list') && ps.userListId) {
userList = await this.userListsRepository.findOneBy({
id: ps.userListId,
userId: me.id,
});
if (userList == null) {
throw new ApiError(meta.errors.noSuchUserList);
} else if (e instanceof AntennaService.InvalidRegexPatternError) {
throw new ApiError(meta.errors.invalidRegexPattern);
} else {
throw e;
}
}
await this.antennasRepository.update(antenna.id, {
name: ps.name,
src: ps.src,
userListId: ps.userListId !== undefined ? userList ? userList.id : null : undefined,
keywords: ps.keywords,
excludeKeywords: ps.excludeKeywords,
users: ps.users,
caseSensitive: ps.caseSensitive,
localOnly: ps.localOnly,
excludeBots: ps.excludeBots,
withReplies: ps.withReplies,
withFile: ps.withFile,
isActive: true,
lastUsedAt: new Date(),
});
this.globalEventService.publishInternalEvent('antennaUpdated', await this.antennasRepository.findOneByOrFail({ id: antenna.id }));
return await this.antennaEntityService.pack(antenna.id);
});
}
}

View File

@ -4651,6 +4651,7 @@ export type components = {
caseSensitive: boolean;
/** @default false */
localOnly: boolean;
useRegex: boolean;
/** @default false */
excludeBots: boolean;
/** @default false */
@ -10918,15 +10919,14 @@ export type operations = {
'application/json': {
name: string;
/** @enum {string} */
src: 'home' | 'all' | 'users' | 'list' | 'users_blacklist';
/** Format: misskey:id */
userListId?: string | null;
src: 'all' | 'users' | 'users_blacklist';
keywords: string[][];
excludeKeywords: string[][];
users: string[];
caseSensitive: boolean;
localOnly?: boolean;
excludeBots?: boolean;
useRegex?: boolean;
withReplies: boolean;
withFile: boolean;
};
@ -11199,15 +11199,14 @@ export type operations = {
antennaId: string;
name?: string;
/** @enum {string} */
src?: 'home' | 'all' | 'users' | 'list' | 'users_blacklist';
/** Format: misskey:id */
userListId?: string | null;
src?: 'all' | 'users' | 'users_blacklist';
keywords?: string[][];
excludeKeywords?: string[][];
users?: string[];
caseSensitive?: boolean;
localOnly?: boolean;
excludeBots?: boolean;
useRegex?: boolean;
withReplies?: boolean;
withFile?: boolean;
};