usbharu
/
misskey
mirror of https://github.com/misskey-dev/misskey.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

wip

This commit is contained in:
syuilo 2024-09-20 13:08:04 +09:00
parent e896b08722
commit 480ec9803a
13 changed files with 93 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
.config/cypress-devcontainer.yml
View File

@ -103,6 +103,14 @@ redis:
# #prefix: example-prefix
# #db: 1
#redisForReactions:
# host: redis
# port: 6379
# #family: 0 # 0=Both, 4=IPv4, 6=IPv6
# #pass: example-pass
# #prefix: example-prefix
# #db: 1
# ┌───────────────────────────┐
#───┘ MeiliSearch configuration └─────────────────────────────

8
.config/docker_example.yml
View File

@ -106,6 +106,14 @@ redis:
# #prefix: example-prefix
# #db: 1
#redisForReactions:
# host: redis
# port: 6379
# #family: 0 # 0=Both, 4=IPv4, 6=IPv6
# #pass: example-pass
# #prefix: example-prefix
# #db: 1
# ┌───────────────────────────┐
#───┘ MeiliSearch configuration └─────────────────────────────

10
.config/example.yml
View File

@ -172,6 +172,16 @@ redis:
# # You can specify more ioredis options...
# #username: example-username
#redisForReactions:
# host: localhost
# port: 6379
# #family: 0 # 0=Both, 4=IPv4, 6=IPv6
# #pass: example-pass
# #prefix: example-prefix
# #db: 1
# # You can specify more ioredis options...
# #username: example-username
# ┌───────────────────────────┐
#───┘ MeiliSearch configuration └─────────────────────────────

8
.devcontainer/devcontainer.yml
View File

@ -103,6 +103,14 @@ redis:
# #prefix: example-prefix
# #db: 1
#redisForReactions:
# host: redis
# port: 6379
# #family: 0 # 0=Both, 4=IPv4, 6=IPv6
# #pass: example-pass
# #prefix: example-prefix
# #db: 1
# ┌───────────────────────────┐
#───┘ MeiliSearch configuration └─────────────────────────────

8
chart/files/default.yml
View File

@ -124,6 +124,14 @@ redis:
# #prefix: example-prefix
# #db: 1
#redisForReactions:
# host: redis
# port: 6379
# #family: 0 # 0=Both, 4=IPv4, 6=IPv6
# #pass: example-pass
# #prefix: example-prefix
# #db: 1
# ┌───────────────────────────┐
#───┘ MeiliSearch configuration └─────────────────────────────

14
packages/backend/src/GlobalModule.ts
View File

@ -78,11 +78,19 @@ const $redisForTimelines: Provider = {
inject: [DI.config],
};
const $redisForReactions: Provider = {
provide: DI.redisForReactions,
useFactory: (config: Config) => {
return new Redis.Redis(config.redisForReactions);
},
inject: [DI.config],
};
@Global()
@Module({
imports: [RepositoryModule],
providers: [$config, $db, $meilisearch, $redis, $redisForPub, $redisForSub, $redisForTimelines],
exports: [$config, $db, $meilisearch, $redis, $redisForPub, $redisForSub, $redisForTimelines, RepositoryModule],
providers: [$config, $db, $meilisearch, $redis, $redisForPub, $redisForSub, $redisForTimelines, $redisForReactions],
exports: [$config, $db, $meilisearch, $redis, $redisForPub, $redisForSub, $redisForTimelines, $redisForReactions, RepositoryModule],
})
export class GlobalModule implements OnApplicationShutdown {
constructor(
@ -91,6 +99,7 @@ export class GlobalModule implements OnApplicationShutdown {
@Inject(DI.redisForPub) private redisForPub: Redis.Redis,
@Inject(DI.redisForSub) private redisForSub: Redis.Redis,
@Inject(DI.redisForTimelines) private redisForTimelines: Redis.Redis,
@Inject(DI.redisForReactions) private redisForReactions: Redis.Redis,
) { }
public async dispose(): Promise<void> {
@ -103,6 +112,7 @@ export class GlobalModule implements OnApplicationShutdown {
this.redisForPub.disconnect(),
this.redisForSub.disconnect(),
this.redisForTimelines.disconnect(),
this.redisForReactions.disconnect(),
]);
}

3
packages/backend/src/config.ts
View File

@ -49,6 +49,7 @@ type Source = {
redisForPubsub?: RedisOptionsSource;
redisForJobQueue?: RedisOptionsSource;
redisForTimelines?: RedisOptionsSource;
redisForReactions?: RedisOptionsSource;
meilisearch?: {
host: string;
port: string;
@ -171,6 +172,7 @@ export type Config = {
redisForPubsub: RedisOptions & RedisOptionsSource;
redisForJobQueue: RedisOptions & RedisOptionsSource;
redisForTimelines: RedisOptions & RedisOptionsSource;
redisForReactions: RedisOptions & RedisOptionsSource;
sentryForBackend: { options: Partial<Sentry.NodeOptions>; enableNodeProfiling: boolean; } | undefined;
sentryForFrontend: { options: Partial<Sentry.NodeOptions> } | undefined;
perChannelMaxNoteCacheCount: number;
@ -251,6 +253,7 @@ export function loadConfig(): Config {
redisForPubsub: config.redisForPubsub ? convertRedisOptions(config.redisForPubsub, host) : redis,
redisForJobQueue: config.redisForJobQueue ? convertRedisOptions(config.redisForJobQueue, host) : redis,
redisForTimelines: config.redisForTimelines ? convertRedisOptions(config.redisForTimelines, host) : redis,
redisForReactions: config.redisForReactions ? convertRedisOptions(config.redisForReactions, host) : redis,
sentryForBackend: config.sentryForBackend,
sentryForFrontend: config.sentryForFrontend,
id: config.id,

8
packages/backend/src/core/ReactionService.ts
View File

@ -4,7 +4,6 @@
*/
import { Inject, Injectable } from '@nestjs/common';
import * as Redis from 'ioredis';
import { DI } from '@/di-symbols.js';
import type { EmojisRepository, NoteReactionsRepository, UsersRepository, NotesRepository } from '@/models/_.js';
import { IdentifiableError } from '@/misc/identifiable-error.js';
@ -72,9 +71,6 @@ const decodeCustomEmojiRegexp = /^:([\w+-]+)(?:@([\w.-]+))?:$/;
@Injectable()
export class ReactionService {
constructor(
@Inject(DI.redis)
private redisClient: Redis.Redis, // TODO: 専用のRedisインスタンスにする
@Inject(DI.usersRepository)
private usersRepository: UsersRepository,
@ -198,10 +194,8 @@ export class ReactionService {
}
}
const rbt = true;
// Increment reactions count
if (rbt) {
if (meta.enableReactionsBuffering) {
this.reactionsBufferingService.create(note, reaction);
// for debugging

23
packages/backend/src/core/ReactionsBufferingService.ts
View File

@ -19,8 +19,8 @@ export class ReactionsBufferingService {
@Inject(DI.config)
private config: Config,
@Inject(DI.redis)
private redisClient: Redis.Redis, // TODO: 専用のRedisインスタンスにする
@Inject(DI.redisForReactions)
private redisForReactions: Redis.Redis, // TODO: 専用のRedisインスタンスにする
@Inject(DI.notesRepository)
private notesRepository: NotesRepository,
@ -29,17 +29,17 @@ export class ReactionsBufferingService {
@bindThis
public async create(note: MiNote, reaction: string) {
this.redisClient.hincrby(`${REDIS_PREFIX}:${note.id}`, reaction, 1);
this.redisForReactions.hincrby(`${REDIS_PREFIX}:${note.id}`, reaction, 1);
}
@bindThis
public async delete(note: MiNote, reaction: string) {
this.redisClient.hincrby(`${REDIS_PREFIX}:${note.id}`, reaction, -1);
this.redisForReactions.hincrby(`${REDIS_PREFIX}:${note.id}`, reaction, -1);
}
@bindThis
public async get(noteId: MiNote['id']): Promise<Record<string, number>> {
const result = await this.redisClient.hgetall(`${REDIS_PREFIX}:${noteId}`);
const result = await this.redisForReactions.hgetall(`${REDIS_PREFIX}:${noteId}`);
const delta = {} as Record<string, number>;
for (const [name, count] of Object.entries(result)) {
delta[name] = parseInt(count);
@ -51,7 +51,7 @@ export class ReactionsBufferingService {
public async getMany(noteIds: MiNote['id'][]): Promise<Map<MiNote['id'], Record<string, number>>> {
const deltas = new Map<MiNote['id'], Record<string, number>>();
const pipeline = this.redisClient.pipeline();
const pipeline = this.redisForReactions.pipeline();
for (const noteId of noteIds) {
pipeline.hgetall(`${REDIS_PREFIX}:${noteId}`);
}
@ -77,8 +77,13 @@ export class ReactionsBufferingService {
let cursor = '0';
do {
// https://github.com/redis/ioredis#transparent-key-prefixing
const result = await this.redisClient.scan(cursor, 'MATCH', `${this.config.redis.prefix}:${REDIS_PREFIX}:*`, 'COUNT', '1000');
console.log(result);
const result = await this.redisForReactions.scan(
cursor,
'MATCH',
`${this.config.redis.prefix}:${REDIS_PREFIX}:*`,
'COUNT',
'1000');
cursor = result[0];
bufferedNoteIds.push(...result[1].map(x => x.replace(`${this.config.redis.prefix}:${REDIS_PREFIX}:`, '')));
} while (cursor !== '0');
@ -86,7 +91,7 @@ export class ReactionsBufferingService {
const deltas = await this.getMany(bufferedNoteIds);
// clear
const pipeline = this.redisClient.pipeline();
const pipeline = this.redisForReactions.pipeline();
for (const noteId of bufferedNoteIds) {
pipeline.del(`${REDIS_PREFIX}:${noteId}`);
}

10
packages/backend/src/core/entities/NoteEntityService.ts
View File

@ -16,6 +16,7 @@ import { bindThis } from '@/decorators.js';
import { DebounceLoader } from '@/misc/loader.js';
import { IdService } from '@/core/IdService.js';
import { ReactionsBufferingService } from '@/core/ReactionsBufferingService.js';
import { MetaService } from '@/core/MetaService.js';
import type { OnModuleInit } from '@nestjs/common';
import type { CustomEmojiService } from '../CustomEmojiService.js';
import type { ReactionService } from '../ReactionService.js';
@ -42,6 +43,7 @@ export class NoteEntityService implements OnModuleInit {
private reactionService: ReactionService;
private reactionsBufferingService: ReactionsBufferingService;
private idService: IdService;
private metaService: MetaService;
private noteLoader = new DebounceLoader(this.findNoteOrFail);
constructor(
@ -73,6 +75,8 @@ export class NoteEntityService implements OnModuleInit {
//private customEmojiService: CustomEmojiService,
//private reactionService: ReactionService,
//private reactionsBufferingService: ReactionsBufferingService,
//private idService: IdService,
//private metaService: MetaService,
) {
}
@ -83,6 +87,7 @@ export class NoteEntityService implements OnModuleInit {
this.reactionService = this.moduleRef.get('ReactionService');
this.reactionsBufferingService = this.moduleRef.get('ReactionsBufferingService');
this.idService = this.moduleRef.get('IdService');
this.metaService = this.moduleRef.get('MetaService');
}
@bindThis
@ -417,8 +422,9 @@ export class NoteEntityService implements OnModuleInit {
) {
if (notes.length === 0) return [];
const rbt = true;
const reactionsDeltas = rbt ? await this.reactionsBufferingService.getMany(notes.map(x => x.id)) : new Map();
const meta = await this.metaService.fetch();
const reactionsDeltas = meta.enableReactionsBuffering ? await this.reactionsBufferingService.getMany(notes.map(x => x.id)) : new Map();
const meId = me ? me.id : null;
const myReactionsMap = new Map<MiNote['id'], string | null>();

1
packages/backend/src/di-symbols.ts
View File

@ -11,6 +11,7 @@ export const DI = {
redisForPub: Symbol('redisForPub'),
redisForSub: Symbol('redisForSub'),
redisForTimelines: Symbol('redisForTimelines'),
redisForReactions: Symbol('redisForReactions'),
//#region Repositories
usersRepository: Symbol('usersRepository'),

8
packages/backend/src/queue/processors/BakeBufferedReactionsProcessorService.ts
View File

@ -7,6 +7,7 @@ import { Inject, Injectable } from '@nestjs/common';
import type Logger from '@/logger.js';
import { bindThis } from '@/decorators.js';
import { ReactionsBufferingService } from '@/core/ReactionsBufferingService.js';
import { MetaService } from '@/core/MetaService.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type * as Bull from 'bullmq';
@ -16,6 +17,7 @@ export class BakeBufferedReactionsProcessorService {
constructor(
private reactionsBufferingService: ReactionsBufferingService,
private metaService: MetaService,
private queueLoggerService: QueueLoggerService,
) {
this.logger = this.queueLoggerService.logger.createSubLogger('bake-buffered-reactions');
@ -23,6 +25,12 @@ export class BakeBufferedReactionsProcessorService {
@bindThis
public async process(): Promise<void> {
const meta = await this.metaService.fetch();
if (!meta.enableReactionsBuffering) {
this.logger.info('Reactions buffering is disabled. Skipping...');
return;
}
this.logger.info('Baking buffered reactions...');
await this.reactionsBufferingService.bake();

4
packages/backend/src/server/HealthServerService.ts
View File

@ -27,6 +27,9 @@ export class HealthServerService {
@Inject(DI.redisForTimelines)
private redisForTimelines: Redis.Redis,
@Inject(DI.redisForReactions)
private redisForReactions: Redis.Redis,
@Inject(DI.db)
private db: DataSource,
@ -43,6 +46,7 @@ export class HealthServerService {
this.redisForPub.ping(),
this.redisForSub.ping(),
this.redisForTimelines.ping(),
this.redisForReactions.ping(),
this.db.query('SELECT 1'),
...(this.meilisearch ? [this.meilisearch.health()] : []),
]).then(() => 200, () => 503));