usbharu
/
misskey
mirror of https://github.com/misskey-dev/misskey.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'develop' into followed-message

This commit is contained in:
syuilo 2024-08-18 17:25:27 +09:00
parent 42cf8b26d2 6c5593d456
commit 54a6866613
16 changed files with 179 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
CHANGELOG.md
View File

@ -1,3 +1,15 @@
## Unreleased
### General
-
### Client
-
### Server
-
## 2024.8.0
### General
@ -33,6 +45,8 @@
- Fix: 無制限にストリーミングのチャンネルに接続できる問題を修正
- Fix: ベースロールのポリシーを変更した際にモデログに記録されないのを修正
(Cherry-picked from https://github.com/MisskeyIO/misskey/pull/700)
- Fix: Prevent memory leak from memory caches (#14310)
- Fix: More reliable memory cache eviction (#14311)
## 2024.7.0

4
locales/en-US.yml
View File

@ -2316,6 +2316,7 @@ _pages:
eyeCatchingImageSet: "Set thumbnail"
eyeCatchingImageRemove: "Delete thumbnail"
chooseBlock: "Add a block"
enterSectionTitle: "Enter a section title"
selectType: "Select a type"
contentBlocks: "Content"
inputBlocks: "Input"
@ -2499,7 +2500,10 @@ _moderationLogTypes:
createAbuseReportNotificationRecipient: "Create a recipient for abuse reports"
updateAbuseReportNotificationRecipient: "Update recipients for abuse reports"
deleteAbuseReportNotificationRecipient: "Delete a recipient for abuse reports"
deleteAccount: "Delete the account"
deletePage: "Delete the page"
deleteFlash: "Delete Play"
deleteGalleryPost: "Delete the gallery post"
_fileViewer:
title: "File details"
type: "File type"

2
package.json
View File

@ -1,6 +1,6 @@
{
"name": "misskey",
"version": "2024.8.0-rc.3",
"version": "2024.8.0",
"codename": "nasubi",
"repository": {
"type": "git",

2
packages/backend/src/core/AvatarDecorationService.ts
View File

@ -29,7 +29,7 @@ export class AvatarDecorationService implements OnApplicationShutdown {
private moderationLogService: ModerationLogService,
private globalEventService: GlobalEventService,
) {
this.cache = new MemorySingleCache<MiAvatarDecoration[]>(1000 * 60 * 30);
this.cache = new MemorySingleCache<MiAvatarDecoration[]>(1000 * 60 * 30); // 30s
this.redisForSub.on('message', this.onMessage);
}

12
packages/backend/src/core/CacheService.ts
View File

@ -56,10 +56,10 @@ export class CacheService implements OnApplicationShutdown {
) {
//this.onMessage = this.onMessage.bind(this);
this.userByIdCache = new MemoryKVCache<MiUser>(Infinity);
this.localUserByNativeTokenCache = new MemoryKVCache<MiLocalUser | null>(Infinity);
this.localUserByIdCache = new MemoryKVCache<MiLocalUser>(Infinity);
this.uriPersonCache = new MemoryKVCache<MiUser | null>(Infinity);
this.userByIdCache = new MemoryKVCache<MiUser>(1000 * 60 * 5); // 5m
this.localUserByNativeTokenCache = new MemoryKVCache<MiLocalUser | null>(1000 * 60 * 5); // 5m
this.localUserByIdCache = new MemoryKVCache<MiLocalUser>(1000 * 60 * 5); // 5m
this.uriPersonCache = new MemoryKVCache<MiUser | null>(1000 * 60 * 5); // 5m
this.userProfileCache = new RedisKVCache<MiUserProfile>(this.redisClient, 'userProfile', {
lifetime: 1000 * 60 * 30, // 30m
@ -135,14 +135,14 @@ export class CacheService implements OnApplicationShutdown {
if (user == null) {
this.userByIdCache.delete(body.id);
this.localUserByIdCache.delete(body.id);
for (const [k, v] of this.uriPersonCache.cache.entries()) {
for (const [k, v] of this.uriPersonCache.entries) {
if (v.value?.id === body.id) {
this.uriPersonCache.delete(k);
}
}
} else {
this.userByIdCache.set(user.id, user);
for (const [k, v] of this.uriPersonCache.cache.entries()) {
for (const [k, v] of this.uriPersonCache.entries) {
if (v.value?.id === user.id) {
this.uriPersonCache.set(k, user);
}

12
packages/backend/src/core/CustomEmojiService.ts
View File

@ -24,7 +24,7 @@ const parseEmojiStrRegexp = /^([-\w]+)(?:@([\w.-]+))?$/;
@Injectable()
export class CustomEmojiService implements OnApplicationShutdown {
private cache: MemoryKVCache<MiEmoji | null>;
private emojisCache: MemoryKVCache<MiEmoji | null>;
public localEmojisCache: RedisSingleCache<Map<string, MiEmoji>>;
constructor(
@ -40,7 +40,7 @@ export class CustomEmojiService implements OnApplicationShutdown {
private moderationLogService: ModerationLogService,
private globalEventService: GlobalEventService,
) {
this.cache = new MemoryKVCache<MiEmoji | null>(1000 * 60 * 60 * 12);
this.emojisCache = new MemoryKVCache<MiEmoji | null>(1000 * 60 * 60 * 12); // 12h
this.localEmojisCache = new RedisSingleCache<Map<string, MiEmoji>>(this.redisClient, 'localEmojis', {
lifetime: 1000 * 60 * 30, // 30m
@ -334,7 +334,7 @@ export class CustomEmojiService implements OnApplicationShutdown {
host,
})) ?? null;
const emoji = await this.cache.fetch(`${name} ${host}`, queryOrNull);
const emoji = await this.emojisCache.fetch(`${name} ${host}`, queryOrNull);
if (emoji == null) return null;
return emoji.publicUrl || emoji.originalUrl; // || emoji.originalUrl してるのは後方互換性のため（publicUrlはstringなので??はだめ）
@ -361,7 +361,7 @@ export class CustomEmojiService implements OnApplicationShutdown {
*/
@bindThis
public async prefetchEmojis(emojis: { name: string; host: string | null; }[]): Promise<void> {
const notCachedEmojis = emojis.filter(emoji => this.cache.get(`${emoji.name} ${emoji.host}`) == null);
const notCachedEmojis = emojis.filter(emoji => this.emojisCache.get(`${emoji.name} ${emoji.host}`) == null);
const emojisQuery: any[] = [];
const hosts = new Set(notCachedEmojis.map(e => e.host));
for (const host of hosts) {
@ -376,7 +376,7 @@ export class CustomEmojiService implements OnApplicationShutdown {
select: ['name', 'host', 'originalUrl', 'publicUrl'],
}) : [];
for (const emoji of _emojis) {
this.cache.set(`${emoji.name} ${emoji.host}`, emoji);
this.emojisCache.set(`${emoji.name} ${emoji.host}`, emoji);
}
}
@ -401,7 +401,7 @@ export class CustomEmojiService implements OnApplicationShutdown {
@bindThis
public dispose(): void {
this.cache.dispose();
this.emojisCache.dispose();
}
@bindThis

2
packages/backend/src/core/RelayService.ts
View File

@ -35,7 +35,7 @@ export class RelayService {
private createSystemUserService: CreateSystemUserService,
private apRendererService: ApRendererService,
) {
this.relaysCache = new MemorySingleCache<MiRelay[]>(1000 * 60 * 10);
this.relaysCache = new MemorySingleCache<MiRelay[]>(1000 * 60 * 10); // 10m
}
@bindThis

6
packages/backend/src/core/RoleService.ts
View File

@ -127,10 +127,8 @@ export class RoleService implements OnApplicationShutdown, OnModuleInit {
private moderationLogService: ModerationLogService,
private fanoutTimelineService: FanoutTimelineService,
) {
//this.onMessage = this.onMessage.bind(this);
this.rolesCache = new MemorySingleCache<MiRole[]>(1000 * 60 * 60 * 1);
this.roleAssignmentByUserIdCache = new MemoryKVCache<MiRoleAssignment[]>(1000 * 60 * 60 * 1);
this.rolesCache = new MemorySingleCache<MiRole[]>(1000 * 60 * 60); // 1h
this.roleAssignmentByUserIdCache = new MemoryKVCache<MiRoleAssignment[]>(1000 * 60 * 5); // 5m
this.redisForSub.on('message', this.onMessage);
}

2
packages/backend/src/core/UserKeypairService.ts
View File

@ -25,7 +25,7 @@ export class UserKeypairService implements OnApplicationShutdown {
) {
this.cache = new RedisKVCache<MiUserKeypair>(this.redisClient, 'userKeypair', {
lifetime: 1000 * 60 * 60 * 24, // 24h
memoryCacheLifetime: Infinity,
memoryCacheLifetime: 1000 * 60 * 60, // 1h
fetcher: (key) => this.userKeypairsRepository.findOneByOrFail({ userId: key }),
toRedisConverter: (value) => JSON.stringify(value),
fromRedisConverter: (value) => JSON.parse(value),

4
packages/backend/src/core/activitypub/ApDbResolverService.ts
View File

@ -54,8 +54,8 @@ export class ApDbResolverService implements OnApplicationShutdown {
private cacheService: CacheService,
private apPersonService: ApPersonService,
) {
this.publicKeyCache = new MemoryKVCache<MiUserPublickey | null>(Infinity);
this.publicKeyByUserIdCache = new MemoryKVCache<MiUserPublickey | null>(Infinity);
this.publicKeyCache = new MemoryKVCache<MiUserPublickey | null>(1000 * 60 * 60 * 12); // 12h
this.publicKeyByUserIdCache = new MemoryKVCache<MiUserPublickey | null>(1000 * 60 * 60 * 12); // 12h
}
@bindThis

137
packages/backend/src/misc/cache.ts
View File

@ -7,23 +7,23 @@ import * as Redis from 'ioredis';
import { bindThis } from '@/decorators.js';
export class RedisKVCache<T> {
private redisClient: Redis.Redis;
private name: string;
private lifetime: number;
private memoryCache: MemoryKVCache<T>;
private fetcher: (key: string) => Promise<T>;
private toRedisConverter: (value: T) => string;
private fromRedisConverter: (value: string) => T | undefined;
private readonly lifetime: number;
private readonly memoryCache: MemoryKVCache<T>;
private readonly fetcher: (key: string) => Promise<T>;
private readonly toRedisConverter: (value: T) => string;
private readonly fromRedisConverter: (value: string) => T | undefined;
constructor(redisClient: RedisKVCache<T>['redisClient'], name: RedisKVCache<T>['name'], opts: {
lifetime: RedisKVCache<T>['lifetime'];
memoryCacheLifetime: number;
fetcher: RedisKVCache<T>['fetcher'];
toRedisConverter: RedisKVCache<T>['toRedisConverter'];
fromRedisConverter: RedisKVCache<T>['fromRedisConverter'];
}) {
this.redisClient = redisClient;
this.name = name;
constructor(
private redisClient: Redis.Redis,
private name: string,
opts: {
lifetime: RedisKVCache<T>['lifetime'];
memoryCacheLifetime: number;
fetcher: RedisKVCache<T>['fetcher'];
toRedisConverter: RedisKVCache<T>['toRedisConverter'];
fromRedisConverter: RedisKVCache<T>['fromRedisConverter'];
},
) {
this.lifetime = opts.lifetime;
this.memoryCache = new MemoryKVCache(opts.memoryCacheLifetime);
this.fetcher = opts.fetcher;
@ -55,7 +55,13 @@ export class RedisKVCache<T> {
const cached = await this.redisClient.get(`kvcache:${this.name}:${key}`);
if (cached == null) return undefined;
return this.fromRedisConverter(cached);
const value = this.fromRedisConverter(cached);
if (value !== undefined) {
this.memoryCache.set(key, value);
}
return value;
}
@bindThis
@ -66,6 +72,10 @@ export class RedisKVCache<T> {
/**
* キャッシュがあればそれを返し、無ければfetcherを呼び出して結果をキャッシュ&返します
* This awaits the call to Redis to ensure that the write succeeded, which is important for a few reasons:
* * Other code uses this to synchronize changes between worker processes. A failed write can internally de-sync the cluster.
* * Without an `await`, consecutive calls could race. An unlucky race could result in the older write overwriting the newer value.
* * Not awaiting here makes the entire cache non-consistent. The prevents many possible uses.
*/
@bindThis
public async fetch(key: string): Promise<T> {
@ -77,14 +87,14 @@ export class RedisKVCache<T> {
// Cache MISS
const value = await this.fetcher(key);
this.set(key, value);
await this.set(key, value);
return value;
}
@bindThis
public async refresh(key: string) {
const value = await this.fetcher(key);
this.set(key, value);
await this.set(key, value);
// TODO: イベント発行して他プロセスのメモリキャッシュも更新できるようにする
}
@ -101,23 +111,23 @@ export class RedisKVCache<T> {
}
export class RedisSingleCache<T> {
private redisClient: Redis.Redis;
private name: string;
private lifetime: number;
private memoryCache: MemorySingleCache<T>;
private fetcher: () => Promise<T>;
private toRedisConverter: (value: T) => string;
private fromRedisConverter: (value: string) => T | undefined;
private readonly lifetime: number;
private readonly memoryCache: MemorySingleCache<T>;
private readonly fetcher: () => Promise<T>;
private readonly toRedisConverter: (value: T) => string;
private readonly fromRedisConverter: (value: string) => T | undefined;
constructor(redisClient: RedisSingleCache<T>['redisClient'], name: RedisSingleCache<T>['name'], opts: {
lifetime: RedisSingleCache<T>['lifetime'];
memoryCacheLifetime: number;
fetcher: RedisSingleCache<T>['fetcher'];
toRedisConverter: RedisSingleCache<T>['toRedisConverter'];
fromRedisConverter: RedisSingleCache<T>['fromRedisConverter'];
}) {
this.redisClient = redisClient;
this.name = name;
constructor(
private redisClient: Redis.Redis,
private name: string,
opts: {
lifetime: number;
memoryCacheLifetime: number;
fetcher: RedisSingleCache<T>['fetcher'];
toRedisConverter: RedisSingleCache<T>['toRedisConverter'];
fromRedisConverter: RedisSingleCache<T>['fromRedisConverter'];
},
) {
this.lifetime = opts.lifetime;
this.memoryCache = new MemorySingleCache(opts.memoryCacheLifetime);
this.fetcher = opts.fetcher;
@ -149,7 +159,13 @@ export class RedisSingleCache<T> {
const cached = await this.redisClient.get(`singlecache:${this.name}`);
if (cached == null) return undefined;
return this.fromRedisConverter(cached);
const value = this.fromRedisConverter(cached);
if (value !== undefined) {
this.memoryCache.set(value);
}
return value;
}
@bindThis
@ -160,6 +176,10 @@ export class RedisSingleCache<T> {
/**
* キャッシュがあればそれを返し、無ければfetcherを呼び出して結果をキャッシュ&返します
* This awaits the call to Redis to ensure that the write succeeded, which is important for a few reasons:
* * Other code uses this to synchronize changes between worker processes. A failed write can internally de-sync the cluster.
* * Without an `await`, consecutive calls could race. An unlucky race could result in the older write overwriting the newer value.
* * Not awaiting here makes the entire cache non-consistent. The prevents many possible uses.
*/
@bindThis
public async fetch(): Promise<T> {
@ -171,14 +191,14 @@ export class RedisSingleCache<T> {
// Cache MISS
const value = await this.fetcher();
this.set(value);
await this.set(value);
return value;
}
@bindThis
public async refresh() {
const value = await this.fetcher();
this.set(value);
await this.set(value);
// TODO: イベント発行して他プロセスのメモリキャッシュも更新できるようにする
}
@ -187,22 +207,12 @@ export class RedisSingleCache<T> {
// TODO: メモリ節約のためあまり参照されないキーを定期的に削除できるようにする？
export class MemoryKVCache<T> {
/**
* データを持つマップ
* @deprecated これを直接操作するべきではない
*/
public cache: Map<string, { date: number; value: T; }>;
private lifetime: number;
private gcIntervalHandle: NodeJS.Timeout;
private readonly cache = new Map<string, { date: number; value: T; }>();
private readonly gcIntervalHandle = setInterval(() => this.gc(), 1000 * 60 * 3); // 3m
constructor(lifetime: MemoryKVCache<never>['lifetime']) {
this.cache = new Map();
this.lifetime = lifetime;
this.gcIntervalHandle = setInterval(() => {
this.gc();
}, 1000 * 60 * 3);
}
constructor(
private readonly lifetime: number,
) {}
@bindThis
/**
@ -287,10 +297,14 @@ export class MemoryKVCache<T> {
@bindThis
public gc(): void {
const now = Date.now();
for (const [key, { date }] of this.cache.entries()) {
if ((now - date) > this.lifetime) {
this.cache.delete(key);
}
// The map is ordered from oldest to youngest.
// We can stop once we find an entry that's still active, because all following entries must *also* be active.
const age = now - date;
if (age < this.lifetime) break;
this.cache.delete(key);
}
}
@ -298,16 +312,19 @@ export class MemoryKVCache<T> {
public dispose(): void {
clearInterval(this.gcIntervalHandle);
}
public get entries() {
return this.cache.entries();
}
}
export class MemorySingleCache<T> {
private cachedAt: number | null = null;
private value: T | undefined;
private lifetime: number;
constructor(lifetime: MemorySingleCache<never>['lifetime']) {
this.lifetime = lifetime;
}
constructor(
private lifetime: number,
) {}
@bindThis
public set(value: T): void {

2
packages/backend/src/queue/processors/DeliverProcessorService.ts
View File

@ -45,7 +45,7 @@ export class DeliverProcessorService {
private queueLoggerService: QueueLoggerService,
) {
this.logger = this.queueLoggerService.logger.createSubLogger('deliver');
this.suspendedHostsCache = new MemorySingleCache<MiInstance[]>(1000 * 60 * 60);
this.suspendedHostsCache = new MemorySingleCache<MiInstance[]>(1000 * 60 * 60); // 1h
}
@bindThis

2
packages/backend/src/server/NodeinfoServerService.ts
View File

@ -134,7 +134,7 @@ export class NodeinfoServerService {
return document;
};
const cache = new MemorySingleCache<Awaited<ReturnType<typeof nodeinfo2>>>(1000 * 60 * 10);
const cache = new MemorySingleCache<Awaited<ReturnType<typeof nodeinfo2>>>(1000 * 60 * 10); // 10m
fastify.get(nodeinfo2_1path, async (request, reply) => {
const base = await cache.fetch(() => nodeinfo2(21));

2
packages/backend/src/server/api/AuthenticateService.ts
View File

@ -37,7 +37,7 @@ export class AuthenticateService implements OnApplicationShutdown {
private cacheService: CacheService,
) {
this.appCache = new MemoryKVCache<MiApp>(Infinity);
this.appCache = new MemoryKVCache<MiApp>(1000 * 60 * 60 * 24 * 7); // 1w
}
@bindThis

64
packages/frontend/src/scripts/focus-trap.ts
View File

@ -16,21 +16,57 @@ function containsFocusTrappedElements(el: HTMLElement): boolean {
});
}
function getZIndex(el: HTMLElement): number {
const zIndex = parseInt(window.getComputedStyle(el).zIndex || '0', 10);
if (isNaN(zIndex)) {
return 0;
}
return zIndex;
}
function getHighestZIndexElement(): { el: HTMLElement; zIndex: number; } | null {
let highestZIndexElement: HTMLElement | null = null;
let highestZIndex = -Infinity;
focusTrapElements.forEach((el) => {
const zIndex = getZIndex(el);
if (zIndex > highestZIndex) {
highestZIndex = zIndex;
highestZIndexElement = el;
}
});
return highestZIndexElement == null ? null : {
el: highestZIndexElement,
zIndex: highestZIndex,
};
}
function releaseFocusTrap(el: HTMLElement): void {
focusTrapElements.delete(el);
if (el.inert === true) {
el.inert = false;
}
const highestZIndexElement = getHighestZIndexElement();
if (el.parentElement != null && el !== document.body) {
el.parentElement.childNodes.forEach((siblingNode) => {
const siblingEl = getHTMLElementOrNull(siblingNode);
if (!siblingEl) return;
if (siblingEl !== el && (focusTrapElements.has(siblingEl) || containsFocusTrappedElements(siblingEl) || focusTrapElements.size === 0)) {
if (
siblingEl !== el &&
(
highestZIndexElement == null ||
siblingEl === highestZIndexElement.el ||
siblingEl.contains(highestZIndexElement.el)
)
) {
siblingEl.inert = false;
} else if (
focusTrapElements.size > 0 &&
!containsFocusTrappedElements(siblingEl) &&
!focusTrapElements.has(siblingEl) &&
highestZIndexElement != null &&
siblingEl !== highestZIndexElement.el &&
!siblingEl.contains(highestZIndexElement.el) &&
!ignoreElements.includes(siblingEl.tagName.toLowerCase())
) {
siblingEl.inert = true;
@ -45,9 +81,29 @@ function releaseFocusTrap(el: HTMLElement): void {
export function focusTrap(el: HTMLElement, hasInteractionWithOtherFocusTrappedEls: boolean, parent: true): void;
export function focusTrap(el: HTMLElement, hasInteractionWithOtherFocusTrappedEls?: boolean, parent?: false): { release: () => void; };
export function focusTrap(el: HTMLElement, hasInteractionWithOtherFocusTrappedEls = false, parent = false): { release: () => void; } | void {
const highestZIndexElement = getHighestZIndexElement();
const highestZIndex = highestZIndexElement == null ? -Infinity : highestZIndexElement.zIndex;
const zIndex = getZIndex(el);
// If the element has a lower z-index than the highest z-index element, focus trap the highest z-index element instead
// Focus trapping for this element will be done in the release function
if (!parent && zIndex < highestZIndex) {
focusTrapElements.add(el);
if (highestZIndexElement) {
focusTrap(highestZIndexElement.el, hasInteractionWithOtherFocusTrappedEls);
}
return {
release: () => {
releaseFocusTrap(el);
},
};
}
if (el.inert === true) {
el.inert = false;
}
if (el.parentElement != null && el !== document.body) {
el.parentElement.childNodes.forEach((siblingNode) => {
const siblingEl = getHTMLElementOrNull(siblingNode);

2
packages/misskey-js/package.json
View File

@ -1,7 +1,7 @@
{
"type": "module",
"name": "misskey-js",
"version": "2024.8.0-rc.3",
"version": "2024.8.0",
"description": "Misskey SDK for JavaScript",
"license": "MIT",
"main": "./built/index.js",