/* * SPDX-FileCopyrightText: syuilo and misskey-project * SPDX-License-Identifier: AGPL-3.0-only */ import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common'; import { Brackets, In, IsNull, ObjectLiteral, SelectQueryBuilder, WhereExpressionBuilder } from 'typeorm'; import * as Redis from 'ioredis'; import { DI } from '@/di-symbols.js'; import { IdService } from '@/core/IdService.js'; import { EmojiEntityService } from '@/core/entities/EmojiEntityService.js'; import { GlobalEventService } from '@/core/GlobalEventService.js'; import type { MiEmoji } from '@/models/Emoji.js'; import type { EmojisRepository, MiRole, MiUser } from '@/models/_.js'; import { bindThis } from '@/decorators.js'; import { MemoryKVCache, RedisSingleCache } from '@/misc/cache.js'; import { UtilityService } from '@/core/UtilityService.js'; import type { Serialized } from '@/types.js'; import { ModerationLogService } from '@/core/ModerationLogService.js'; const parseEmojiStrRegexp = /^([-\w]+)(?:@([\w.-]+))?$/; export const fetchEmojisHostTypes = [ 'local', 'remote', 'all', ] as const; export type FetchEmojisHostTypes = typeof fetchEmojisHostTypes[number]; export const fetchEmojisSortKeys = [ 'id', 'updatedAt', 'name', 'host', 'uri', 'publicUrl', 'type', 'aliases', 'category', 'license', 'isSensitive', 'localOnly', 'roleIdsThatCanBeUsedThisEmojiAsReaction', ] as const; export type FetchEmojisSortKeys = typeof fetchEmojisSortKeys[number]; export type FetchEmojisParams = { query?: { updatedAtFrom?: string; updatedAtTo?: string; name?: string; host?: string; uri?: string; publicUrl?: string; type?: string; aliases?: string; category?: string; license?: string; isSensitive?: boolean; localOnly?: boolean; hostType?: FetchEmojisHostTypes; roleIds?: string[]; }, sinceId?: string; untilId?: string; limit?: number; page?: number; sort?: { key: FetchEmojisSortKeys; direction: 'ASC' | 'DESC'; }[] } @Injectable() export class CustomEmojiService implements OnApplicationShutdown { private cache: MemoryKVCache<MiEmoji | null>; public localEmojisCache: RedisSingleCache<Map<string, MiEmoji>>; constructor( @Inject(DI.redis) private redisClient: Redis.Redis, @Inject(DI.emojisRepository) private emojisRepository: EmojisRepository, private utilityService: UtilityService, private idService: IdService, private emojiEntityService: EmojiEntityService, private moderationLogService: ModerationLogService, private globalEventService: GlobalEventService, ) { this.cache = new MemoryKVCache<MiEmoji | null>(1000 * 60 * 60 * 12); this.localEmojisCache = new RedisSingleCache<Map<string, MiEmoji>>(this.redisClient, 'localEmojis', { lifetime: 1000 * 60 * 30, // 30m memoryCacheLifetime: 1000 * 60 * 3, // 3m fetcher: () => this.emojisRepository.find({ where: { host: IsNull() } }).then(emojis => new Map(emojis.map(emoji => [emoji.name, emoji]))), toRedisConverter: (value) => JSON.stringify(Array.from(value.values())), fromRedisConverter: (value) => { return new Map(JSON.parse(value).map((x: Serialized<MiEmoji>) => [x.name, { ...x, updatedAt: x.updatedAt ? new Date(x.updatedAt) : null, }])); }, }); } @bindThis public async add(data: { originalUrl: string; publicUrl: string; fileType: string; name: string; category: string | null; aliases: string[]; host: string | null; license: string | null; isSensitive: boolean; localOnly: boolean; roleIdsThatCanBeUsedThisEmojiAsReaction: MiRole['id'][]; }, moderator?: MiUser): Promise<MiEmoji> { const emoji = await this.emojisRepository.insertOne({ id: this.idService.gen(), updatedAt: new Date(), name: data.name, category: data.category, host: data.host, aliases: data.aliases, originalUrl: data.originalUrl, publicUrl: data.publicUrl, type: data.fileType, license: data.license, isSensitive: data.isSensitive, localOnly: data.localOnly, roleIdsThatCanBeUsedThisEmojiAsReaction: data.roleIdsThatCanBeUsedThisEmojiAsReaction, }); if (data.host == null) { this.localEmojisCache.refresh(); this.globalEventService.publishBroadcastStream('emojiAdded', { emoji: await this.emojiEntityService.packDetailed(emoji.id), }); if (moderator) { this.moderationLogService.log(moderator, 'addCustomEmoji', { emojiId: emoji.id, emoji: emoji, }); } } return emoji; } @bindThis public async update(id: MiEmoji['id'], data: { originalUrl?: string; publicUrl?: string; fileType?: string; name?: string; category?: string | null; aliases?: string[]; license?: string | null; isSensitive?: boolean; localOnly?: boolean; roleIdsThatCanBeUsedThisEmojiAsReaction?: MiRole['id'][]; }, moderator?: MiUser): Promise<void> { const emoji = await this.emojisRepository.findOneByOrFail({ id: id }); const sameNameEmoji = await this.emojisRepository.findOneBy({ name: data.name, host: IsNull() }); if (sameNameEmoji != null && sameNameEmoji.id !== id) throw new Error('name already exists'); await this.emojisRepository.update(emoji.id, { updatedAt: new Date(), name: data.name, category: data.category, aliases: data.aliases, license: data.license, isSensitive: data.isSensitive, localOnly: data.localOnly, originalUrl: data.originalUrl, publicUrl: data.publicUrl, type: data.fileType, roleIdsThatCanBeUsedThisEmojiAsReaction: data.roleIdsThatCanBeUsedThisEmojiAsReaction ?? undefined, }); this.localEmojisCache.refresh(); const packed = await this.emojiEntityService.packDetailed(emoji.id); if (emoji.name === data.name) { this.globalEventService.publishBroadcastStream('emojiUpdated', { emojis: [packed], }); } else { this.globalEventService.publishBroadcastStream('emojiDeleted', { emojis: [await this.emojiEntityService.packDetailed(emoji)], }); this.globalEventService.publishBroadcastStream('emojiAdded', { emoji: packed, }); } if (moderator) { const updated = await this.emojisRepository.findOneByOrFail({ id: id }); this.moderationLogService.log(moderator, 'updateCustomEmoji', { emojiId: emoji.id, before: emoji, after: updated, }); } } @bindThis public async addAliasesBulk(ids: MiEmoji['id'][], aliases: string[]) { const emojis = await this.emojisRepository.findBy({ id: In(ids), }); for (const emoji of emojis) { await this.emojisRepository.update(emoji.id, { updatedAt: new Date(), aliases: [...new Set(emoji.aliases.concat(aliases))], }); } this.localEmojisCache.refresh(); this.globalEventService.publishBroadcastStream('emojiUpdated', { emojis: await this.emojiEntityService.packDetailedMany(ids), }); } @bindThis public async setAliasesBulk(ids: MiEmoji['id'][], aliases: string[]) { await this.emojisRepository.update({ id: In(ids), }, { updatedAt: new Date(), aliases: aliases, }); this.localEmojisCache.refresh(); this.globalEventService.publishBroadcastStream('emojiUpdated', { emojis: await this.emojiEntityService.packDetailedMany(ids), }); } @bindThis public async removeAliasesBulk(ids: MiEmoji['id'][], aliases: string[]) { const emojis = await this.emojisRepository.findBy({ id: In(ids), }); for (const emoji of emojis) { await this.emojisRepository.update(emoji.id, { updatedAt: new Date(), aliases: emoji.aliases.filter(x => !aliases.includes(x)), }); } this.localEmojisCache.refresh(); this.globalEventService.publishBroadcastStream('emojiUpdated', { emojis: await this.emojiEntityService.packDetailedMany(ids), }); } @bindThis public async setCategoryBulk(ids: MiEmoji['id'][], category: string | null) { await this.emojisRepository.update({ id: In(ids), }, { updatedAt: new Date(), category: category, }); this.localEmojisCache.refresh(); this.globalEventService.publishBroadcastStream('emojiUpdated', { emojis: await this.emojiEntityService.packDetailedMany(ids), }); } @bindThis public async setLicenseBulk(ids: MiEmoji['id'][], license: string | null) { await this.emojisRepository.update({ id: In(ids), }, { updatedAt: new Date(), license: license, }); this.localEmojisCache.refresh(); this.globalEventService.publishBroadcastStream('emojiUpdated', { emojis: await this.emojiEntityService.packDetailedMany(ids), }); } @bindThis public async delete(id: MiEmoji['id'], moderator?: MiUser) { const emoji = await this.emojisRepository.findOneByOrFail({ id: id }); await this.emojisRepository.delete(emoji.id); this.localEmojisCache.refresh(); this.globalEventService.publishBroadcastStream('emojiDeleted', { emojis: [await this.emojiEntityService.packDetailed(emoji)], }); if (moderator) { this.moderationLogService.log(moderator, 'deleteCustomEmoji', { emojiId: emoji.id, emoji: emoji, }); } } @bindThis public async deleteBulk(ids: MiEmoji['id'][], moderator?: MiUser) { const emojis = await this.emojisRepository.findBy({ id: In(ids), }); for (const emoji of emojis) { await this.emojisRepository.delete(emoji.id); if (moderator) { this.moderationLogService.log(moderator, 'deleteCustomEmoji', { emojiId: emoji.id, emoji: emoji, }); } } this.localEmojisCache.refresh(); this.globalEventService.publishBroadcastStream('emojiDeleted', { emojis: await this.emojiEntityService.packDetailedMany(emojis), }); } @bindThis private normalizeHost(src: string | undefined, noteUserHost: string | null): string | null { // クエリに使うホスト let host = src === '.' ? null // .はローカルホスト (ここがマッチするのはリアクションのみ) : src === undefined ? noteUserHost // ノートなどでホスト省略表記の場合はローカルホスト (ここがリアクションにマッチすることはない) : this.utilityService.isSelfHost(src) ? null // 自ホスト指定 : (src || noteUserHost); // 指定されたホスト || ノートなどの所有者のホスト (こっちがリアクションにマッチすることはない) host = this.utilityService.toPunyNullable(host); return host; } @bindThis public parseEmojiStr(emojiName: string, noteUserHost: string | null) { const match = emojiName.match(parseEmojiStrRegexp); if (!match) return { name: null, host: null }; const name = match[1]; // ホスト正規化 const host = this.utilityService.toPunyNullable(this.normalizeHost(match[2], noteUserHost)); return { name, host }; } /** * 添付用(リモート)カスタム絵文字URLを解決する * @param emojiName ノートやユーザープロフィールに添付された、またはリアクションのカスタム絵文字名 (:は含めない, リアクションでローカルホストの場合は@.を付ける (これはdecodeReactionで可能)) * @param noteUserHost ノートやユーザープロフィールの所有者のホスト * @returns URL, nullは未マッチを意味する */ @bindThis public async populateEmoji(emojiName: string, noteUserHost: string | null): Promise<string | null> { const { name, host } = this.parseEmojiStr(emojiName, noteUserHost); if (name == null) return null; if (host == null) return null; const queryOrNull = async () => (await this.emojisRepository.findOneBy({ name, host, })) ?? null; const emoji = await this.cache.fetch(`${name} ${host}`, queryOrNull); if (emoji == null) return null; return emoji.publicUrl || emoji.originalUrl; // || emoji.originalUrl してるのは後方互換性のため(publicUrlはstringなので??はだめ) } /** * 複数の添付用(リモート)カスタム絵文字URLを解決する (キャシュ付き, 存在しないものは結果から除外される) */ @bindThis public async populateEmojis(emojiNames: string[], noteUserHost: string | null): Promise<Record<string, string>> { const emojis = await Promise.all(emojiNames.map(x => this.populateEmoji(x, noteUserHost))); const res = {} as Record<string, string>; for (let i = 0; i < emojiNames.length; i++) { const resolvedEmoji = emojis[i]; if (resolvedEmoji != null) { res[emojiNames[i]] = resolvedEmoji; } } return res; } /** * 与えられた絵文字のリストをデータベースから取得し、キャッシュに追加します */ @bindThis public async prefetchEmojis(emojis: { name: string; host: string | null; }[]): Promise<void> { const notCachedEmojis = emojis.filter(emoji => this.cache.get(`${emoji.name} ${emoji.host}`) == null); const emojisQuery: any[] = []; const hosts = new Set(notCachedEmojis.map(e => e.host)); for (const host of hosts) { if (host == null) continue; emojisQuery.push({ name: In(notCachedEmojis.filter(e => e.host === host).map(e => e.name)), host: host, }); } const _emojis = emojisQuery.length > 0 ? await this.emojisRepository.find({ where: emojisQuery, select: ['name', 'host', 'originalUrl', 'publicUrl'], }) : []; for (const emoji of _emojis) { this.cache.set(`${emoji.name} ${emoji.host}`, emoji); } } /** * ローカル内の絵文字に重複がないかチェックします * @param name 絵文字名 */ @bindThis public checkDuplicate(name: string): Promise<boolean> { return this.emojisRepository.exists({ where: { name, host: IsNull() } }); } @bindThis public getEmojiById(id: string): Promise<MiEmoji | null> { return this.emojisRepository.findOneBy({ id }); } @bindThis public getEmojiByName(name: string): Promise<MiEmoji | null> { return this.emojisRepository.findOneBy({ name, host: IsNull() }); } @bindThis public async fetchEmojis(params?: FetchEmojisParams) { function multipleWordsToQuery<T extends ObjectLiteral>( query: string, builder: SelectQueryBuilder<T>, action: (qb: WhereExpressionBuilder, idx: number, word: string) => void, ) { const words = query.split(/\s/); builder.andWhere(new Brackets((qb => { for (const [idx, word] of words.entries()) { action(qb, idx, word); } }))); } const builder = this.emojisRepository.createQueryBuilder('emoji'); if (params?.query) { const q = params.query; if (q.updatedAtFrom) { // noIndexScan builder.andWhere('CAST(emoji.updatedAt AS DATE) >= :updateAtFrom', { updateAtFrom: q.updatedAtFrom }); } if (q.updatedAtTo) { // noIndexScan builder.andWhere('CAST(emoji.updatedAt AS DATE) <= :updateAtTo', { updateAtTo: q.updatedAtTo }); } if (q.name) { multipleWordsToQuery(q.name, builder, (qb, idx, word) => { qb.orWhere(`emoji.name LIKE :name${idx}`, Object.fromEntries([[`name${idx}`, `%${word}%`]])); }); } switch (true) { case q.hostType === 'local': { builder.andWhere('emoji.host IS NULL'); break; } case q.hostType === 'remote': { if (q.host) { // noIndexScan multipleWordsToQuery(q.host, builder, (qb, idx, word) => { qb.orWhere(`emoji.host LIKE :host${idx}`, Object.fromEntries([[`host${idx}`, `%${word}%`]])); }); } else { builder.andWhere('emoji.host IS NOT NULL'); } break; } } if (q.uri) { // noIndexScan multipleWordsToQuery(q.uri, builder, (qb, idx, word) => { qb.orWhere(`emoji.uri LIKE :uri${idx}`, Object.fromEntries([[`uri${idx}`, `%${word}%`]])); }); } if (q.publicUrl) { // noIndexScan multipleWordsToQuery(q.publicUrl, builder, (qb, idx, word) => { qb.orWhere(`emoji.publicUrl LIKE :publicUrl${idx}`, Object.fromEntries([[`publicUrl${idx}`, `%${word}%`]])); }); } if (q.type) { // noIndexScan multipleWordsToQuery(q.type, builder, (qb, idx, word) => { qb.orWhere(`emoji.type LIKE :type${idx}`, Object.fromEntries([[`type${idx}`, `%${word}%`]])); }); } if (q.aliases) { // noIndexScan const subQueryBuilder = builder.subQuery() .select('COUNT(0)', 'count') .from( sq2 => sq2 .select('unnest(subEmoji.aliases)', 'alias') .addSelect('subEmoji.id', 'id') .from('emoji', 'subEmoji'), 'aliasTable', ) .where('"emoji"."id" = "aliasTable"."id"'); multipleWordsToQuery(q.aliases, subQueryBuilder, (qb, idx, word) => { qb.orWhere(`"aliasTable"."alias" LIKE :aliases${idx}`, Object.fromEntries([[`aliases${idx}`, `%${word}%`]])); }); builder.andWhere(`(${subQueryBuilder.getQuery()}) > 0`); } if (q.category) { multipleWordsToQuery(q.category, builder, (qb, idx, word) => { qb.orWhere(`emoji.category LIKE :category${idx}`, Object.fromEntries([[`category${idx}`, `%${word}%`]])); }); } if (q.license) { // noIndexScan multipleWordsToQuery(q.license, builder, (qb, idx, word) => { qb.orWhere(`emoji.license LIKE :license${idx}`, Object.fromEntries([[`license${idx}`, `%${word}%`]])); }); } if (q.isSensitive != null) { // noIndexScan builder.andWhere('emoji.isSensitive = :isSensitive', { isSensitive: q.isSensitive }); } if (q.localOnly != null) { // noIndexScan builder.andWhere('emoji.localOnly = :localOnly', { localOnly: q.localOnly }); } if (q.roleIds && q.roleIds.length > 0) { builder.andWhere('emoji.roleIdsThatCanBeUsedThisEmojiAsReaction @> :roleIds', { roleIds: q.roleIds }); } } if (params?.sinceId) { builder.andWhere('emoji.id > :sinceId', { sinceId: params.sinceId }); } if (params?.untilId) { builder.andWhere('emoji.id < :untilId', { untilId: params.untilId }); } if (params?.sort && params.sort.length > 0) { for (const sort of params.sort) { builder.addOrderBy(`emoji.${sort.key}`, sort.direction); } } else { builder.addOrderBy('emoji.id', 'DESC'); } const limit = params?.limit ?? 10; if (params?.page) { builder.skip((params.page - 1) * limit); } builder.take(limit); const [emojis, count] = await builder.getManyAndCount(); return { emojis, count: (count > limit ? emojis.length : count), allCount: count, allPages: Math.ceil(count / limit), }; } @bindThis public dispose(): void { this.cache.dispose(); } @bindThis public onApplicationShutdown(signal?: string | undefined): void { this.dispose(); } }