usbharu
/
misskey
mirror of https://github.com/misskey-dev/misskey.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'misskey-original/develop' into develop 

# Conflicts:
#	package.json
#	packages/backend/src/server/api/endpoints/users/notes.ts
This commit is contained in:
mattyatea 2023-10-08 15:44:52 +09:00
parent 271d3483b2 9240db35f3
commit 38b9bc3a4d
17 changed files with 238 additions and 134 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
CHANGELOG.md
View File

@ -36,14 +36,18 @@
### Client
- Enhance: 二要素認証のバックアップコード一覧をテキストファイルでダウンロード可能に
- Enhance: 動画再生時のデフォルトボリュームを30%に
- Fix: リアクションしたユーザ一覧のUIが稀に左上に残ってしまう不具合を修正
### Server
- Enhance: タイムライン取得時のパフォーマンスを大幅に向上
- Enhance: ハイライト取得時のパフォーマンスを大幅に向上
- Enhance: トレンドハッシュタグ取得時のパフォーマンスを大幅に向上
- Enhance: WebSocket接続が多い場合のパフォーマンスを向上
- Enhance: 不要なPostgreSQLのインデックスを削除しパフォーマンスを向上
- Fix: 連合なしアンケートに投票をするとUpdateがリモートに配信されてしまうのを修正
- Fix: nodeinfoにおいてCORS用のヘッダーが設定されていないのを修正
- Fix: 同じ種類のTLのストリーミングを複数接続できない問題を修正
## 2023.9.3
### General

3
package.json
View File

@ -1,6 +1,6 @@
{
"name": "misskey",
"version": "2023.10.0-beta.7-prismisskey.1",
"version": "2023.10.0-beta.8",
"codename": "nasubi",
"repository": {
"type": "git",
@ -18,7 +18,6 @@
"build-assets": "node ./scripts/build-assets.mjs",
"build": "pnpm build-pre && pnpm -r build && pnpm build-assets",
"build-storybook": "pnpm --filter frontend build-storybook",
"build-and-start": "pnpm i && pnpm build && pnpm start",
"start": "pnpm check:connect && cd packages/backend && node ./built/boot/entry.js",
"start:test": "cd packages/backend && cross-env NODE_ENV=test node ./built/boot/entry.js",
"init": "pnpm migrate",

9
packages/backend/src/core/HashtagService.ts
View File

@ -174,16 +174,15 @@ export class HashtagService {
const redisPipeline = this.redisClient.pipeline();
// TODO: これらの Set は Bloom Filter を使うようにしても良さそう
// チャート用
redisPipeline.sadd(`hashtagUsers:${hashtag}:${window}`, userId);
redisPipeline.pfadd(`hashtagUsers:${hashtag}:${window}`, userId);
redisPipeline.expire(`hashtagUsers:${hashtag}:${window}`,
60 * 60 * 24 * 3, // 3日間
'NX', // "NX -- Set expiry only when the key has no expiry" = 有効期限がないときだけ設定
);
// ユニークカウント用
// TODO: Bloom Filter を使うようにしても良さそう
redisPipeline.sadd(`hashtagUsers:${hashtag}`, userId);
redisPipeline.expire(`hashtagUsers:${hashtag}`,
60 * 60, // 1時間
@ -202,7 +201,7 @@ export class HashtagService {
for (let i = 0; i < range; i++) {
const window = `${now.getUTCFullYear()}${(now.getUTCMonth() + 1).toString().padStart(2, '0')}${now.getUTCDate().toString().padStart(2, '0')}${now.getUTCHours().toString().padStart(2, '0')}${now.getUTCMinutes().toString().padStart(2, '0')}`;
redisPipeline.scard(`hashtagUsers:${hashtag}:${window}`);
redisPipeline.pfcount(`hashtagUsers:${hashtag}:${window}`);
now.setMinutes(now.getMinutes() - (i * 10), 0, 0);
}
@ -223,7 +222,7 @@ export class HashtagService {
for (let i = 0; i < range; i++) {
const window = `${now.getUTCFullYear()}${(now.getUTCMonth() + 1).toString().padStart(2, '0')}${now.getUTCDate().toString().padStart(2, '0')}${now.getUTCHours().toString().padStart(2, '0')}${now.getUTCMinutes().toString().padStart(2, '0')}`;
for (const hashtag of hashtags) {
redisPipeline.scard(`hashtagUsers:${hashtag}:${window}`);
redisPipeline.pfcount(`hashtagUsers:${hashtag}:${window}`);
}
now.setMinutes(now.getMinutes() - (i * 10), 0, 0);
}

12
packages/backend/src/core/entities/NoteEntityService.ts
View File

@ -17,6 +17,7 @@ import type { MiNoteReaction } from '@/models/NoteReaction.js';
import type { UsersRepository, NotesRepository, FollowingsRepository, PollsRepository, PollVotesRepository, NoteReactionsRepository, ChannelsRepository } from '@/models/_.js';
import { bindThis } from '@/decorators.js';
import { isNotNull } from '@/misc/is-not-null.js';
import { DebounceLoader } from '@/misc/loader.js';
import type { OnModuleInit } from '@nestjs/common';
import type { CustomEmojiService } from '../CustomEmojiService.js';
import type { ReactionService } from '../ReactionService.js';
@ -29,6 +30,7 @@ export class NoteEntityService implements OnModuleInit {
private driveFileEntityService: DriveFileEntityService;
private customEmojiService: CustomEmojiService;
private reactionService: ReactionService;
private noteLoader = new DebounceLoader(this.findNoteOrFail);
constructor(
private moduleRef: ModuleRef,
@ -285,7 +287,7 @@ export class NoteEntityService implements OnModuleInit {
}, options);
const meId = me ? me.id : null;
const note = typeof src === 'object' ? src : await this.notesRepository.findOneOrFail({ where: { id: src }, relations: ['user'] });
const note = typeof src === 'object' ? src : await this.noteLoader.load(src);
const host = note.userHost;
let text = note.text;
@ -451,4 +453,12 @@ export class NoteEntityService implements OnModuleInit {
}
return emojis.filter(x => x.name != null && x.host != null) as { name: string; host: string; }[];
}
@bindThis
private findNoteOrFail(id: string): Promise<MiNote> {
return this.notesRepository.findOneOrFail({
where: { id },
relations: ['user'],
});
}
}

52
packages/backend/src/misc/loader.ts Normal file
View File

@ -0,0 +1,52 @@
export type FetchFunction<K, V> = (key: K) => Promise<V>;
type ResolveReject<V> = Parameters<ConstructorParameters<typeof Promise<V>>[0]>;
type ResolverPair<V> = {
resolve: ResolveReject<V>[0];
reject: ResolveReject<V>[1];
};
export class DebounceLoader<K, V> {
private resolverMap = new Map<K, ResolverPair<V>>();
private promiseMap = new Map<K, Promise<V>>();
private resolvedPromise = Promise.resolve();
constructor(private loadFn: FetchFunction<K, V>) {}
public load(key: K): Promise<V> {
const promise = this.promiseMap.get(key);
if (typeof promise !== 'undefined') {
return promise;
}
const isFirst = this.promiseMap.size === 0;
const newPromise = new Promise<V>((resolve, reject) => {
this.resolverMap.set(key, { resolve, reject });
});
this.promiseMap.set(key, newPromise);
if (isFirst) {
this.enqueueDebouncedLoadJob();
}
return newPromise;
}
private runDebouncedLoad(): void {
const resolvers = [...this.resolverMap];
this.resolverMap.clear();
this.promiseMap.clear();
for (const [key, { resolve, reject }] of resolvers) {
this.loadFn(key).then(resolve, reject);
}
}
private enqueueDebouncedLoadJob(): void {
this.resolvedPromise.then(() => {
process.nextTick(() => {
this.runDebouncedLoad();
});
});
}
}

12
packages/backend/src/server/NodeinfoServerService.ts
View File

@ -135,7 +135,11 @@ export class NodeinfoServerService {
.type(
'application/json; profile="http://nodeinfo.diaspora.software/ns/schema/2.1#"',
)
.header('Cache-Control', 'public, max-age=600');
.header('Cache-Control', 'public, max-age=600')
.header('Access-Control-Allow-Headers', 'Accept')
.header('Access-Control-Allow-Methods', 'GET, OPTIONS')
.header('Access-Control-Allow-Origin', '*')
.header('Access-Control-Expose-Headers', 'Vary');
return { version: '2.1', ...base };
});
@ -148,7 +152,11 @@ export class NodeinfoServerService {
.type(
'application/json; profile="http://nodeinfo.diaspora.software/ns/schema/2.0#"',
)
.header('Cache-Control', 'public, max-age=600');
.header('Cache-Control', 'public, max-age=600')
.header('Access-Control-Allow-Headers', 'Accept')
.header('Access-Control-Allow-Methods', 'GET, OPTIONS')
.header('Access-Control-Allow-Origin', '*')
.header('Access-Control-Expose-Headers', 'Vary');
return { version: '2.0', ...base };
});

6
packages/backend/src/server/ServerService.ts
View File

@ -199,10 +199,10 @@ export class ServerService implements OnApplicationShutdown {
includeSecrets: true,
}));
reply.code(200);
return 'Verify succeeded!';
reply.code(200).send('Verification succeeded! メールアドレスの認証に成功しました。');
return;
} else {
reply.code(404);
reply.code(404).send('Verification failed. Please try again. メールアドレスの認証に失敗しました。もう一度お試しください');
return;
}
});

15
packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts
View File

@ -98,20 +98,23 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
const limit = ps.limit + (ps.untilId ? 1 : 0) + (ps.sinceId ? 1 : 0); // untilIdに指定したものも含まれるため+1
const [htlNoteIds, ltlNoteIds] = await Promise.all([
this.redisForTimelines.xrevrange(
const redisPipeline = this.redisForTimelines.pipeline();
redisPipeline.xrevrange(
ps.withFiles ? `homeTimelineWithFiles:${me.id}` : `homeTimeline:${me.id}`,
ps.untilId ? this.idService.parse(ps.untilId).date.getTime() : ps.untilDate ?? '+',
ps.sinceId ? this.idService.parse(ps.sinceId).date.getTime() : ps.sinceDate ?? '-',
'COUNT', limit,
).then(res => res.map(x => x[1][1]).filter(x => x !== ps.untilId && x !== ps.sinceId)),
this.redisForTimelines.xrevrange(
);
redisPipeline.xrevrange(
ps.withFiles ? 'localTimelineWithFiles' : 'localTimeline',
ps.untilId ? this.idService.parse(ps.untilId).date.getTime() : ps.untilDate ?? '+',
ps.sinceId ? this.idService.parse(ps.sinceId).date.getTime() : ps.sinceDate ?? '-',
'COUNT', limit,
).then(res => res.map(x => x[1][1]).filter(x => x !== ps.untilId && x !== ps.sinceId)),
]);
);
const [htlNoteIds, ltlNoteIds] = await redisPipeline.exec().then(res => res ? [
(res[0][1] as string[][]).map(x => x[1][1]).filter(x => x !== ps.untilId && x !== ps.sinceId),
(res[1][1] as string[][]).map(x => x[1][1]).filter(x => x !== ps.untilId && x !== ps.sinceId),
] : []);
let noteIds = Array.from(new Set([...htlNoteIds, ...ltlNoteIds]));
noteIds.sort((a, b) => a > b ? -1 : 1);

83
packages/backend/src/server/api/endpoints/users/notes.ts
View File

@ -15,7 +15,6 @@ import { IdService } from '@/core/IdService.js';
import { isUserRelated } from '@/misc/is-user-related.js';
import { QueryService } from '@/core/QueryService.js';
import { ApiError } from '../../error.js';
import { GetterService } from '@/server/api/GetterService.js'
export const meta = {
tags: ['users', 'notes'],
@ -67,9 +66,8 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
@Inject(DI.notesRepository)
private notesRepository: NotesRepository,
private getterService: GetterService,
private queryService: QueryService,
private noteEntityService: NoteEntityService,
private queryService: QueryService,
private cacheService: CacheService,
private idService: IdService,
) {
@ -80,8 +78,6 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
this.cacheService.userMutingsCache.fetch(me.id),
]) : [new Set<string>()];
let timeline: MiNote[] = [];
const limit = ps.limit + (ps.untilId ? 1 : 0) + (ps.sinceId ? 1 : 0); // untilIdに指定したものも含まれるため+1
const [noteIdsRes, repliesNoteIdsRes, channelNoteIdsRes] = await Promise.all([
@ -117,65 +113,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
noteIds.sort((a, b) => a > b ? -1 : 1);
noteIds = noteIds.slice(0, ps.limit);
if (noteIds.length < limit) {
//#region Construct query
const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), ps.sinceId, ps.untilId, ps.sinceDate, ps.untilDate)
.andWhere('note.userId = :userId', { userId: ps.userId })
.innerJoinAndSelect('note.user', 'user')
.leftJoinAndSelect('note.reply', 'reply')
.leftJoinAndSelect('note.renote', 'renote')
.leftJoinAndSelect('note.channel', 'channel')
.leftJoinAndSelect('reply.user', 'replyUser')
.leftJoinAndSelect('renote.user', 'renoteUser');
query.andWhere(new Brackets(qb => {
qb.orWhere('note.channelId IS NULL');
qb.orWhere('channel.isSensitive = false');
}));
this.queryService.generateVisibilityQuery(query, me);
if (me) {
this.queryService.generateMutedUserQuery(query, me);
this.queryService.generateBlockedUserQuery(query, me);
}
if (ps.withFiles) {
query.andWhere('note.fileIds != \'{}\'');
}
if (!ps.withReplies) {
query.andWhere('note.replyId IS NULL');
}
if (ps.withRenotes === false) {
query.andWhere(new Brackets(qb => {
qb.orWhere('note.renoteId IS NULL');
qb.orWhere(new Brackets(qb => {
qb.orWhere('note.text IS NOT NULL');
qb.orWhere('note.fileIds != \'{}\'');
}));
}));
}
if (ps.includeMyRenotes === false) {
query.andWhere(new Brackets(qb => {
qb.orWhere('note.userId != :userId', { userId: ps.userId });
qb.orWhere('note.renoteId IS NULL');
qb.orWhere('note.text IS NOT NULL');
qb.orWhere('note.fileIds != \'{}\'');
qb.orWhere('0 < (SELECT COUNT(*) FROM poll WHERE poll."noteId" = note.id)');
}));
}
const ids = await query.limit(limit - noteIds.length).getMany();
noteIds = noteIds.concat(ids.map(note => note.id));
}
if (noteIds.length === 0) {
return [];
}
if (noteIds.length > 0) {
const isFollowing = me ? Object.hasOwn(await this.cacheService.userFollowingsCache.fetch(me.id), ps.userId) : false;
const query = this.notesRepository.createQueryBuilder('note')
@ -187,7 +125,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
.leftJoinAndSelect('renote.user', 'renoteUser')
.leftJoinAndSelect('note.channel', 'channel');
timeline = await query.getMany();
let timeline = await query.getMany();
timeline = timeline.filter(note => {
if (me && isUserRelated(note, userIdsWhoMeMuting, true)) return false;
@ -205,8 +143,10 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
timeline.sort((a, b) => a.id > b.id ? -1 : 1);
return await this.noteEntityService.packMany(timeline, me);
} else {
// fallback to database
if (timeline.length === 0) {
//#region Construct query
const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), ps.sinceId, ps.untilId, ps.sinceDate, ps.untilDate)
.andWhere('note.userId = :userId', { userId: ps.userId })
@ -217,10 +157,9 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
.leftJoinAndSelect('reply.user', 'replyUser')
.leftJoinAndSelect('renote.user', 'renoteUser');
query.andWhere(new Brackets(qb => {
qb.orWhere('note.channelId IS NULL');
qb.orWhere('channel.isSensitive = false');
}));
if (!ps.withChannelNotes) {
query.andWhere('note.channelId IS NULL');
}
this.queryService.generateVisibilityQuery(query, me);
@ -239,10 +178,10 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
}
//#endregion
timeline = await query.limit(ps.limit).getMany();
}
const timeline = await query.limit(ps.limit).getMany();
return await this.noteEntityService.packMany(timeline, me);
}
});
}
}

2
packages/backend/src/server/api/stream/channels/global-timeline.ts
View File

@ -16,7 +16,7 @@ import Channel from '../channel.js';
class GlobalTimelineChannel extends Channel {
public readonly chName = 'globalTimeline';
public static shouldShare = true;
public static shouldShare = false;
public static requireCredential = false;
private withReplies: boolean;
private withFiles: boolean;

2
packages/backend/src/server/api/stream/channels/home-timeline.ts
View File

@ -14,7 +14,7 @@ import Channel from '../channel.js';
class HomeTimelineChannel extends Channel {
public readonly chName = 'homeTimeline';
public static shouldShare = true;
public static shouldShare = false;
public static requireCredential = true;
private withReplies: boolean;
private withFiles: boolean;

2
packages/backend/src/server/api/stream/channels/hybrid-timeline.ts
View File

@ -16,7 +16,7 @@ import Channel from '../channel.js';
class HybridTimelineChannel extends Channel {
public readonly chName = 'hybridTimeline';
public static shouldShare = true;
public static shouldShare = false;
public static requireCredential = true;
private withReplies: boolean;
private withFiles: boolean;

2
packages/backend/src/server/api/stream/channels/local-timeline.ts
View File

@ -15,7 +15,7 @@ import Channel from '../channel.js';
class LocalTimelineChannel extends Channel {
public readonly chName = 'localTimeline';
public static shouldShare = true;
public static shouldShare = false;
public static requireCredential = false;
private withReplies: boolean;
private withFiles: boolean;

88
packages/backend/test/unit/misc/loader.ts Normal file
View File

@ -0,0 +1,88 @@
import { DebounceLoader } from '@/misc/loader.js';
class Mock {
loadCountByKey = new Map<number, number>();
load = async (key: number): Promise<number> => {
const count = this.loadCountByKey.get(key);
if (typeof count === 'undefined') {
this.loadCountByKey.set(key, 1);
} else {
this.loadCountByKey.set(key, count + 1);
}
return key * 2;
};
reset() {
this.loadCountByKey.clear();
}
}
describe(DebounceLoader, () => {
describe('single request', () => {
it('loads once', async () => {
const mock = new Mock();
const loader = new DebounceLoader(mock.load);
expect(await loader.load(7)).toBe(14);
expect(mock.loadCountByKey.size).toBe(1);
expect(mock.loadCountByKey.get(7)).toBe(1);
});
});
describe('two duplicated requests at same time', () => {
it('loads once', async () => {
const mock = new Mock();
const loader = new DebounceLoader(mock.load);
const [v1, v2] = await Promise.all([
loader.load(7),
loader.load(7),
]);
expect(v1).toBe(14);
expect(v2).toBe(14);
expect(mock.loadCountByKey.size).toBe(1);
expect(mock.loadCountByKey.get(7)).toBe(1);
});
});
describe('two different requests at same time', () => {
it('loads twice', async () => {
const mock = new Mock();
const loader = new DebounceLoader(mock.load);
const [v1, v2] = await Promise.all([
loader.load(7),
loader.load(13),
]);
expect(v1).toBe(14);
expect(v2).toBe(26);
expect(mock.loadCountByKey.size).toBe(2);
expect(mock.loadCountByKey.get(7)).toBe(1);
expect(mock.loadCountByKey.get(13)).toBe(1);
});
});
describe('non-continuous same two requests', () => {
it('loads twice', async () => {
const mock = new Mock();
const loader = new DebounceLoader(mock.load);
expect(await loader.load(7)).toBe(14);
expect(mock.loadCountByKey.size).toBe(1);
expect(mock.loadCountByKey.get(7)).toBe(1);
mock.reset();
expect(await loader.load(7)).toBe(14);
expect(mock.loadCountByKey.size).toBe(1);
expect(mock.loadCountByKey.get(7)).toBe(1);
});
});
describe('non-continuous different two requests', () => {
it('loads twice', async () => {
const mock = new Mock();
const loader = new DebounceLoader(mock.load);
expect(await loader.load(7)).toBe(14);
expect(mock.loadCountByKey.size).toBe(1);
expect(mock.loadCountByKey.get(7)).toBe(1);
mock.reset();
expect(await loader.load(13)).toBe(26);
expect(mock.loadCountByKey.size).toBe(1);
expect(mock.loadCountByKey.get(13)).toBe(1);
});
});
});

15
packages/frontend/src/components/MkMediaBanner.vue
View File

@ -17,7 +17,6 @@ SPDX-License-Identifier: AGPL-3.0-only
:title="media.name"
controls
preload="metadata"
@volumechange="volumechange"
/>
</div>
<a
@ -33,7 +32,7 @@ SPDX-License-Identifier: AGPL-3.0-only
</template>
<script lang="ts" setup>
import { onMounted } from 'vue';
import { onMounted, shallowRef, watch } from 'vue';
import * as Misskey from 'misskey-js';
import { soundConfigStore } from '@/scripts/sound.js';
import { i18n } from '@/i18n.js';
@ -43,15 +42,13 @@ const props = withDefaults(defineProps<{
}>(), {
});
const audioEl = $shallowRef<HTMLAudioElement | null>();
const audioEl = shallowRef<HTMLAudioElement>();
let hide = $ref(true);
function volumechange() {
if (audioEl) soundConfigStore.set('mediaVolume', audioEl.volume);
}
onMounted(() => {
if (audioEl) audioEl.volume = soundConfigStore.state.mediaVolume;
watch(audioEl, () => {
if (audioEl.value) {
audioEl.value.volume = 0.3;
}
});
</script>

11
packages/frontend/src/components/MkMediaVideo.vue
View File

@ -14,6 +14,7 @@ SPDX-License-Identifier: AGPL-3.0-only
</div>
<div v-else :class="[$style.visible, (video.isSensitive && defaultStore.state.highlightSensitiveMedia) && $style.sensitiveContainer]">
<video
ref="videoEl"
:class="$style.video"
:poster="video.thumbnailUrl"
:title="video.comment"
@ -31,7 +32,7 @@ SPDX-License-Identifier: AGPL-3.0-only
</template>
<script lang="ts" setup>
import { ref } from 'vue';
import { ref, shallowRef, watch } from 'vue';
import * as Misskey from 'misskey-js';
import bytes from '@/filters/bytes.js';
import { defaultStore } from '@/store.js';
@ -42,6 +43,14 @@ const props = defineProps<{
}>();
const hide = ref((defaultStore.state.nsfw === 'force' || defaultStore.state.enableDataSaverMode) ? true : (props.video.isSensitive && defaultStore.state.nsfw !== 'ignore'));
const videoEl = shallowRef<HTMLVideoElement>();
watch(videoEl, () => {
if (videoEl.value) {
videoEl.value.volume = 0.3;
}
});
</script>
<style lang="scss" module>

4
packages/frontend/src/scripts/sound.ts
View File

@ -7,10 +7,6 @@ import { markRaw } from 'vue';
import { Storage } from '@/pizzax.js';
export const soundConfigStore = markRaw(new Storage('sound', {
mediaVolume: {
where: 'device',
default: 0.5,
},
sound_masterVolume: {
where: 'device',
default: 0.3,