From f906ad6ca7044e4c509a5fe01f398f841a44027a Mon Sep 17 00:00:00 2001 From: zawa-ch Date: Tue, 27 Feb 2024 18:45:46 +0900 Subject: [PATCH 01/21] =?UTF-8?q?Enhance:=20=E3=82=B3=E3=83=B3=E3=83=87?= =?UTF-8?q?=E3=82=A3=E3=82=B7=E3=83=A7=E3=83=8A=E3=83=AB=E3=83=AD=E3=83=BC?= =?UTF-8?q?=E3=83=AB=E3=81=AE=E6=9D=A1=E4=BB=B6=E3=81=AB=E3=80=8C=E3=83=9E?= =?UTF-8?q?=E3=83=8B=E3=83=A5=E3=82=A2=E3=83=AB=E3=83=AD=E3=83=BC=E3=83=AB?= =?UTF-8?q?=E3=81=B8=E3=81=AE=E3=82=A2=E3=82=B5=E3=82=A4=E3=83=B3=E3=80=8D?= =?UTF-8?q?=E3=82=92=E8=BF=BD=E5=8A=A0=20(#13463)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * コンディショナルロールの条件に「マニュアルロールへのアサイン」を追加 * コメント修正 --- CHANGELOG.md | 1 + locales/index.d.ts | 4 +++ locales/ja-JP.yml | 1 + packages/backend/src/core/RoleService.ts | 19 +++++++------ packages/backend/src/misc/json-schema.ts | 2 ++ packages/backend/src/models/Role.ts | 6 ++++ .../backend/src/models/json-schema/role.ts | 20 +++++++++++++ packages/backend/test/unit/RoleService.ts | 28 +++++++++++++++++++ .../src/pages/admin/RolesEditorFormula.vue | 9 ++++++ packages/misskey-js/etc/misskey-js.api.md | 4 +++ packages/misskey-js/src/autogen/models.ts | 1 + packages/misskey-js/src/autogen/types.ts | 11 +++++++- 12 files changed, 97 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ebbe22f2f6..513338e667 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ ### General - Enhance: サーバーごとにモデレーションノートを残せるように +- Enhance: コンディショナルロールの条件に「マニュアルロールへのアサイン」を追加 ### Client - Enhance: ノート作成画面のファイル添付メニューの区切り線の位置を調整 diff --git a/locales/index.d.ts b/locales/index.d.ts index 1a2565b06a..7d5f8ce732 100644 --- a/locales/index.d.ts +++ b/locales/index.d.ts @@ -6528,6 +6528,10 @@ export interface Locale extends ILocale { "avatarDecorationLimit": string; }; "_condition": { + /** + * マニュアルロールにアサイン済み + */ + "roleAssignedTo": string; /** * ローカルユーザー */ diff --git a/locales/ja-JP.yml b/locales/ja-JP.yml index 61c61b8f96..1bb56738c6 100644 --- a/locales/ja-JP.yml +++ b/locales/ja-JP.yml @@ -1687,6 +1687,7 @@ _role: canUseTranslator: "翻訳機能の利用" avatarDecorationLimit: "アイコンデコレーションの最大取付個数" _condition: + roleAssignedTo: "マニュアルロールにアサイン済み" isLocal: "ローカルユーザー" isRemote: "リモートユーザー" createdLessThan: "アカウント作成から～以内" diff --git a/packages/backend/src/core/RoleService.ts b/packages/backend/src/core/RoleService.ts index c5baaf3fff..8312489a78 100644 --- a/packages/backend/src/core/RoleService.ts +++ b/packages/backend/src/core/RoleService.ts @@ -200,17 +200,20 @@ export class RoleService implements OnApplicationShutdown, OnModuleInit { } @bindThis - private evalCond(user: MiUser, value: RoleCondFormulaValue): boolean { + private evalCond(user: MiUser, roles: MiRole[], value: RoleCondFormulaValue): boolean { try { switch (value.type) { case 'and': { - return value.values.every(v => this.evalCond(user, v)); + return value.values.every(v => this.evalCond(user, roles, v)); } case 'or': { - return value.values.some(v => this.evalCond(user, v)); + return value.values.some(v => this.evalCond(user, roles, v)); } case 'not': { - return !this.evalCond(user, value.value); + return !this.evalCond(user, roles, value.value); + } + case 'roleAssignedTo': { + return roles.some(r => r.id === value.roleId); } case 'isLocal': { return this.userEntityService.isLocalUser(user); @@ -272,7 +275,7 @@ export class RoleService implements OnApplicationShutdown, OnModuleInit { const assigns = await this.getUserAssigns(userId); const assignedRoles = roles.filter(r => assigns.map(x => x.roleId).includes(r.id)); const user = roles.some(r => r.target === 'conditional') ? await this.cacheService.findUserById(userId) : null; - const matchedCondRoles = roles.filter(r => r.target === 'conditional' && this.evalCond(user!, r.condFormula)); + const matchedCondRoles = roles.filter(r => r.target === 'conditional' && this.evalCond(user!, assignedRoles, r.condFormula)); return [...assignedRoles, ...matchedCondRoles]; } @@ -285,13 +288,13 @@ export class RoleService implements OnApplicationShutdown, OnModuleInit { let assigns = await this.roleAssignmentByUserIdCache.fetch(userId, () => this.roleAssignmentsRepository.findBy({ userId })); // 期限切れのロールを除外 assigns = assigns.filter(a => a.expiresAt == null || (a.expiresAt.getTime() > now)); - const assignedRoleIds = assigns.map(x => x.roleId); const roles = await this.rolesCache.fetch(() => this.rolesRepository.findBy({})); - const assignedBadgeRoles = roles.filter(r => r.asBadge && assignedRoleIds.includes(r.id)); + const assignedRoles = roles.filter(r => assigns.map(x => x.roleId).includes(r.id)); + const assignedBadgeRoles = assignedRoles.filter(r => r.asBadge); const badgeCondRoles = roles.filter(r => r.asBadge && (r.target === 'conditional')); if (badgeCondRoles.length > 0) { const user = roles.some(r => r.target === 'conditional') ? await this.cacheService.findUserById(userId) : null; - const matchedBadgeCondRoles = badgeCondRoles.filter(r => this.evalCond(user!, r.condFormula)); + const matchedBadgeCondRoles = badgeCondRoles.filter(r => this.evalCond(user!, assignedRoles, r.condFormula)); return [...assignedBadgeRoles, ...matchedBadgeCondRoles]; } else { return assignedBadgeRoles; diff --git a/packages/backend/src/misc/json-schema.ts b/packages/backend/src/misc/json-schema.ts index 8449e5ff07..46b0bb2fab 100644 --- a/packages/backend/src/misc/json-schema.ts +++ b/packages/backend/src/misc/json-schema.ts @@ -44,6 +44,7 @@ import { packedRoleCondFormulaLogicsSchema, packedRoleCondFormulaValueNot, packedRoleCondFormulaValueIsLocalOrRemoteSchema, + packedRoleCondFormulaValueAssignedRoleSchema, packedRoleCondFormulaValueCreatedSchema, packedRoleCondFormulaFollowersOrFollowingOrNotesSchema, packedRoleCondFormulaValueSchema, @@ -96,6 +97,7 @@ export const refs = { RoleCondFormulaLogics: packedRoleCondFormulaLogicsSchema, RoleCondFormulaValueNot: packedRoleCondFormulaValueNot, RoleCondFormulaValueIsLocalOrRemote: packedRoleCondFormulaValueIsLocalOrRemoteSchema, + RoleCondFormulaValueAssignedRole: packedRoleCondFormulaValueAssignedRoleSchema, RoleCondFormulaValueCreated: packedRoleCondFormulaValueCreatedSchema, RoleCondFormulaFollowersOrFollowingOrNotes: packedRoleCondFormulaFollowersOrFollowingOrNotesSchema, RoleCondFormulaValue: packedRoleCondFormulaValueSchema, diff --git a/packages/backend/src/models/Role.ts b/packages/backend/src/models/Role.ts index fa05ea8637..058abe3118 100644 --- a/packages/backend/src/models/Role.ts +++ b/packages/backend/src/models/Role.ts @@ -29,6 +29,11 @@ type CondFormulaValueIsRemote = { type: 'isRemote'; }; +type CondFormulaValueRoleAssignedTo = { + type: 'roleAssignedTo'; + roleId: string; +}; + type CondFormulaValueCreatedLessThan = { type: 'createdLessThan'; sec: number; @@ -75,6 +80,7 @@ export type RoleCondFormulaValue = { id: string } & ( CondFormulaValueNot | CondFormulaValueIsLocal | CondFormulaValueIsRemote | + CondFormulaValueRoleAssignedTo | CondFormulaValueCreatedLessThan | CondFormulaValueCreatedMoreThan | CondFormulaValueFollowersLessThanOrEq | diff --git a/packages/backend/src/models/json-schema/role.ts b/packages/backend/src/models/json-schema/role.ts index ef6b279bee..9f2b5b17ed 100644 --- a/packages/backend/src/models/json-schema/role.ts +++ b/packages/backend/src/models/json-schema/role.ts @@ -57,6 +57,23 @@ export const packedRoleCondFormulaValueIsLocalOrRemoteSchema = { }, } as const; +export const packedRoleCondFormulaValueAssignedRoleSchema = { + type: 'object', + properties: { + type: { + type: 'string', + nullable: false, optional: false, + enum: ['roleAssignedTo'], + }, + roleId: { + type: 'string', + nullable: false, optional: false, + format: 'id', + example: 'xxxxxxxxxx', + }, + }, +} as const; + export const packedRoleCondFormulaValueCreatedSchema = { type: 'object', properties: { @@ -115,6 +132,9 @@ export const packedRoleCondFormulaValueSchema = { { ref: 'RoleCondFormulaValueIsLocalOrRemote', }, + { + ref: 'RoleCondFormulaValueAssignedRole', + }, { ref: 'RoleCondFormulaValueCreated', }, diff --git a/packages/backend/test/unit/RoleService.ts b/packages/backend/test/unit/RoleService.ts index 5222745b7f..fe5ad31597 100644 --- a/packages/backend/test/unit/RoleService.ts +++ b/packages/backend/test/unit/RoleService.ts @@ -251,6 +251,34 @@ describe('RoleService', () => { expect(user2Policies.canManageCustomEmojis).toBe(true); }); + test('コンディショナルロール: マニュアルロールにアサイン済み', async () => { + const [user1, user2, role1] = await Promise.all([ + createUser(), + createUser(), + createRole({ + name: 'manual role', + }), + ]); + const role2 = await createRole({ + name: 'conditional role', + target: 'conditional', + condFormula: { + // idはバックエンドのロジックに必要ない？ + id: 'bdc612bd-9d54-4675-ae83-0499c82ea670', + type: 'roleAssignedTo', + roleId: role1.id, + }, + }); + await roleService.assign(user2.id, role1.id); + + const [u1role, u2role] = await Promise.all([ + roleService.getUserRoles(user1.id), + roleService.getUserRoles(user2.id), + ]); + expect(u1role.some(r => r.id === role2.id)).toBe(false); + expect(u2role.some(r => r.id === role2.id)).toBe(true); + }); + test('expired role', async () => { const user = await createUser(); const role = await createRole({ diff --git a/packages/frontend/src/pages/admin/RolesEditorFormula.vue b/packages/frontend/src/pages/admin/RolesEditorFormula.vue index f4a8f44955..2f5b4c47d8 100644 --- a/packages/frontend/src/pages/admin/RolesEditorFormula.vue +++ b/packages/frontend/src/pages/admin/RolesEditorFormula.vue @@ -9,6 +9,7 @@ SPDX-License-Identifier: AGPL-3.0-only + @@ -51,6 +52,10 @@ SPDX-License-Identifier: AGPL-3.0-only + + + + @@ -62,6 +67,7 @@ import MkSelect from '@/components/MkSelect.vue'; import MkButton from '@/components/MkButton.vue'; import { i18n } from '@/i18n.js'; import { deepClone } from '@/scripts/clone.js'; +import { rolesCache } from '@/cache.js'; const Sortable = defineAsyncComponent(() => import('vuedraggable').then(x => x.default)); @@ -77,6 +83,8 @@ const props = defineProps<{ const v = ref(deepClone(props.modelValue)); +const roles = await rolesCache.fetch(); + watch(() => props.modelValue, () => { if (JSON.stringify(props.modelValue) === JSON.stringify(v.value)) return; v.value = deepClone(props.modelValue); @@ -92,6 +100,7 @@ const type = computed({ if (t === 'and') v.value.values = []; if (t === 'or') v.value.values = []; if (t === 'not') v.value.value = { id: uuid(), type: 'isRemote' }; + if (t === 'roleAssignedTo') v.value.roleId = ''; if (t === 'createdLessThan') v.value.sec = 86400; if (t === 'createdMoreThan') v.value.sec = 86400; if (t === 'followersLessThanOrEq') v.value.value = 10; diff --git a/packages/misskey-js/etc/misskey-js.api.md b/packages/misskey-js/etc/misskey-js.api.md index b5e7ec7548..0e990ffd5a 100644 --- a/packages/misskey-js/etc/misskey-js.api.md +++ b/packages/misskey-js/etc/misskey-js.api.md @@ -1712,6 +1712,7 @@ declare namespace entities { RoleCondFormulaLogics, RoleCondFormulaValueNot, RoleCondFormulaValueIsLocalOrRemote, + RoleCondFormulaValueAssignedRole, RoleCondFormulaValueCreated, RoleCondFormulaFollowersOrFollowingOrNotes, RoleCondFormulaValue, @@ -2731,6 +2732,9 @@ type RoleCondFormulaLogics = components['schemas']['RoleCondFormulaLogics']; // @public (undocumented) type RoleCondFormulaValue = components['schemas']['RoleCondFormulaValue']; +// @public (undocumented) +type RoleCondFormulaValueAssignedRole = components['schemas']['RoleCondFormulaValueAssignedRole']; + // @public (undocumented) type RoleCondFormulaValueCreated = components['schemas']['RoleCondFormulaValueCreated']; diff --git a/packages/misskey-js/src/autogen/models.ts b/packages/misskey-js/src/autogen/models.ts index ab49f9478a..6f61458600 100644 --- a/packages/misskey-js/src/autogen/models.ts +++ b/packages/misskey-js/src/autogen/models.ts @@ -38,6 +38,7 @@ export type Signin = components['schemas']['Signin']; export type RoleCondFormulaLogics = components['schemas']['RoleCondFormulaLogics']; export type RoleCondFormulaValueNot = components['schemas']['RoleCondFormulaValueNot']; export type RoleCondFormulaValueIsLocalOrRemote = components['schemas']['RoleCondFormulaValueIsLocalOrRemote']; +export type RoleCondFormulaValueAssignedRole = components['schemas']['RoleCondFormulaValueAssignedRole']; export type RoleCondFormulaValueCreated = components['schemas']['RoleCondFormulaValueCreated']; export type RoleCondFormulaFollowersOrFollowingOrNotes = components['schemas']['RoleCondFormulaFollowersOrFollowingOrNotes']; export type RoleCondFormulaValue = components['schemas']['RoleCondFormulaValue']; diff --git a/packages/misskey-js/src/autogen/types.ts b/packages/misskey-js/src/autogen/types.ts index 733670d704..8d700fb828 100644 --- a/packages/misskey-js/src/autogen/types.ts +++ b/packages/misskey-js/src/autogen/types.ts @@ -4573,6 +4573,15 @@ export type components = { /** @enum {string} */ type: 'isLocal' | 'isRemote'; }; + RoleCondFormulaValueAssignedRole: { + /** @enum {string} */ + type: 'roleAssignedTo'; + /** + * Format: id + * @example xxxxxxxxxx + */ + roleId: string; + }; RoleCondFormulaValueCreated: { id: string; /** @enum {string} */ @@ -4585,7 +4594,7 @@ export type components = { type: 'followersLessThanOrEq' | 'followersMoreThanOrEq' | 'followingLessThanOrEq' | 'followingMoreThanOrEq' | 'notesLessThanOrEq' | 'notesMoreThanOrEq'; value: number; }; - RoleCondFormulaValue: components['schemas']['RoleCondFormulaLogics'] | components['schemas']['RoleCondFormulaValueNot'] | components['schemas']['RoleCondFormulaValueIsLocalOrRemote'] | components['schemas']['RoleCondFormulaValueCreated'] | components['schemas']['RoleCondFormulaFollowersOrFollowingOrNotes']; + RoleCondFormulaValue: components['schemas']['RoleCondFormulaLogics'] | components['schemas']['RoleCondFormulaValueNot'] | components['schemas']['RoleCondFormulaValueIsLocalOrRemote'] | components['schemas']['RoleCondFormulaValueAssignedRole'] | components['schemas']['RoleCondFormulaValueCreated'] | components['schemas']['RoleCondFormulaFollowersOrFollowingOrNotes']; RoleLite: { /** * Format: id From 0d47877db1e1012aaba78a2926b165cf9e039d3d Mon Sep 17 00:00:00 2001 From: zyoshoka <107108195+zyoshoka@users.noreply.github.com> Date: Wed, 28 Feb 2024 09:49:34 +0900 Subject: [PATCH 02/21] =?UTF-8?q?enhance(backend):=20=E3=83=95=E3=82=A9?= =?UTF-8?q?=E3=83=AD=E3=83=BC=E3=83=BB=E3=83=95=E3=82=A9=E3=83=AD=E3=83=AF?= =?UTF-8?q?=E3=83=BC=E9=96=A2=E9=80=A3=E3=81=AE=E9=80=9A=E7=9F=A5=E3=81=AE?= =?UTF-8?q?=E5=8F=97=E4=BF=A1=E8=A8=AD=E5=AE=9A=E3=81=AE=E5=BC=B7=E5=8C=96?= =?UTF-8?q?=20(#13468)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * enhance(backend): 通知の受信設定に「フォロー中またはフォロワー」を追加 * fix(backend): 通知の受信設定で「相互フォロー」が正しく動作しない問題を修正 * Update CHANGELOG.md --- CHANGELOG.md | 2 + locales/index.d.ts | 4 + locales/ja-JP.yml | 1 + .../backend/src/core/NotificationService.ts | 8 ++ packages/backend/src/models/UserProfile.ts | 2 + .../backend/src/models/json-schema/user.ts | 2 +- .../notifications.notification-config.vue | 1 + .../src/pages/settings/notifications.vue | 1 + packages/misskey-js/src/autogen/types.ts | 84 +++++++++---------- 9 files changed, 62 insertions(+), 43 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 513338e667..010d5aed7a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ ### General - Enhance: サーバーごとにモデレーションノートを残せるように - Enhance: コンディショナルロールの条件に「マニュアルロールへのアサイン」を追加 +- Enhance: 通知の受信設定に「フォロー中またはフォロワー」を追加 ### Client - Enhance: ノート作成画面のファイル添付メニューの区切り線の位置を調整 @@ -33,6 +34,7 @@ - 必須パラメータを`id`または`name`のいずれかのみに - `id`の代わりに`name`で絵文字を指定可能に（`id`・`name`両指定時は従来通り`name`を変更する挙動） - `category`および`licence`が指定なしの時勝手にnullに上書きされる挙動を修正 +- Fix: 通知の受信設定で「相互フォロー」が正しく動作しない問題を修正 ## 2024.2.0 diff --git a/locales/index.d.ts b/locales/index.d.ts index 7d5f8ce732..3edc9d235e 100644 --- a/locales/index.d.ts +++ b/locales/index.d.ts @@ -4656,6 +4656,10 @@ export interface Locale extends ILocale { * 相互フォロー */ "mutualFollow": string; + /** + * フォロー中またはフォロワー + */ + "followingOrFollower": string; /** * ファイル付きのみ */ diff --git a/locales/ja-JP.yml b/locales/ja-JP.yml index 1bb56738c6..66ddf6a46d 100644 --- a/locales/ja-JP.yml +++ b/locales/ja-JP.yml @@ -1160,6 +1160,7 @@ showRenotes: "リノートを表示" edited: "編集済み" notificationRecieveConfig: "通知の受信設定" mutualFollow: "相互フォロー" +followingOrFollower: "フォロー中またはフォロワー" fileAttachedOnly: "ファイル付きのみ" showRepliesToOthersInTimeline: "TLに他の人への返信を含める" hideRepliesToOthersInTimeline: "TLに他の人への返信を含めない" diff --git a/packages/backend/src/core/NotificationService.ts b/packages/backend/src/core/NotificationService.ts index ee16193579..7224341991 100644 --- a/packages/backend/src/core/NotificationService.ts +++ b/packages/backend/src/core/NotificationService.ts @@ -122,6 +122,14 @@ export class NotificationService implements OnApplicationShutdown { return null; } } else if (recieveConfig?.type === 'mutualFollow') { + const [isFollowing, isFollower] = await Promise.all([ + this.cacheService.userFollowingsCache.fetch(notifieeId).then(followings => Object.hasOwn(followings, notifierId)), + this.cacheService.userFollowingsCache.fetch(notifierId).then(followings => Object.hasOwn(followings, notifieeId)), + ]); + if (!(isFollowing && isFollower)) { + return null; + } + } else if (recieveConfig?.type === 'followingOrFollower') { const [isFollowing, isFollower] = await Promise.all([ this.cacheService.userFollowingsCache.fetch(notifieeId).then(followings => Object.hasOwn(followings, notifierId)), this.cacheService.userFollowingsCache.fetch(notifierId).then(followings => Object.hasOwn(followings, notifieeId)), diff --git a/packages/backend/src/models/UserProfile.ts b/packages/backend/src/models/UserProfile.ts index 1ca2f55850..7dbe0b3717 100644 --- a/packages/backend/src/models/UserProfile.ts +++ b/packages/backend/src/models/UserProfile.ts @@ -249,6 +249,8 @@ export class MiUserProfile { type: 'follower'; } | { type: 'mutualFollow'; + } | { + type: 'followingOrFollower'; } | { type: 'list'; userListId: MiUserList['id']; diff --git a/packages/backend/src/models/json-schema/user.ts b/packages/backend/src/models/json-schema/user.ts index 952cd6bf80..947a9317d7 100644 --- a/packages/backend/src/models/json-schema/user.ts +++ b/packages/backend/src/models/json-schema/user.ts @@ -13,7 +13,7 @@ export const notificationRecieveConfig = { type: { type: 'string', nullable: false, - enum: ['all', 'following', 'follower', 'mutualFollow', 'never'], + enum: ['all', 'following', 'follower', 'mutualFollow', 'followingOrFollower', 'never'], }, }, required: ['type'], diff --git a/packages/frontend/src/pages/settings/notifications.notification-config.vue b/packages/frontend/src/pages/settings/notifications.notification-config.vue index d6aac63674..a36f036303 100644 --- a/packages/frontend/src/pages/settings/notifications.notification-config.vue +++ b/packages/frontend/src/pages/settings/notifications.notification-config.vue @@ -10,6 +10,7 @@ SPDX-License-Identifier: AGPL-3.0-only + diff --git a/packages/frontend/src/pages/settings/notifications.vue b/packages/frontend/src/pages/settings/notifications.vue index febcfa32ed..bbcef65283 100644 --- a/packages/frontend/src/pages/settings/notifications.vue +++ b/packages/frontend/src/pages/settings/notifications.vue @@ -16,6 +16,7 @@ SPDX-License-Identifier: AGPL-3.0-only $i.notificationRecieveConfig[type]?.type === 'following' ? i18n.ts.following : $i.notificationRecieveConfig[type]?.type === 'follower' ? i18n.ts.followers : $i.notificationRecieveConfig[type]?.type === 'mutualFollow' ? i18n.ts.mutualFollow : + $i.notificationRecieveConfig[type]?.type === 'followingOrFollower' ? i18n.ts.followingOrFollower : $i.notificationRecieveConfig[type]?.type === 'list' ? i18n.ts.userList : i18n.ts.all }} diff --git a/packages/misskey-js/src/autogen/types.ts b/packages/misskey-js/src/autogen/types.ts index 8d700fb828..a3597e4635 100644 --- a/packages/misskey-js/src/autogen/types.ts +++ b/packages/misskey-js/src/autogen/types.ts @@ -3700,7 +3700,7 @@ export type components = { notificationRecieveConfig: { note?: OneOf<[{ /** @enum {string} */ - type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'never'; + type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'followingOrFollower' | 'never'; }, { /** @enum {string} */ type: 'list'; @@ -3709,7 +3709,7 @@ export type components = { }]>; follow?: OneOf<[{ /** @enum {string} */ - type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'never'; + type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'followingOrFollower' | 'never'; }, { /** @enum {string} */ type: 'list'; @@ -3718,7 +3718,7 @@ export type components = { }]>; mention?: OneOf<[{ /** @enum {string} */ - type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'never'; + type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'followingOrFollower' | 'never'; }, { /** @enum {string} */ type: 'list'; @@ -3727,7 +3727,7 @@ export type components = { }]>; reply?: OneOf<[{ /** @enum {string} */ - type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'never'; + type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'followingOrFollower' | 'never'; }, { /** @enum {string} */ type: 'list'; @@ -3736,7 +3736,7 @@ export type components = { }]>; renote?: OneOf<[{ /** @enum {string} */ - type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'never'; + type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'followingOrFollower' | 'never'; }, { /** @enum {string} */ type: 'list'; @@ -3745,7 +3745,7 @@ export type components = { }]>; quote?: OneOf<[{ /** @enum {string} */ - type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'never'; + type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'followingOrFollower' | 'never'; }, { /** @enum {string} */ type: 'list'; @@ -3754,7 +3754,7 @@ export type components = { }]>; reaction?: OneOf<[{ /** @enum {string} */ - type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'never'; + type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'followingOrFollower' | 'never'; }, { /** @enum {string} */ type: 'list'; @@ -3763,7 +3763,7 @@ export type components = { }]>; pollEnded?: OneOf<[{ /** @enum {string} */ - type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'never'; + type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'followingOrFollower' | 'never'; }, { /** @enum {string} */ type: 'list'; @@ -3772,7 +3772,7 @@ export type components = { }]>; receiveFollowRequest?: OneOf<[{ /** @enum {string} */ - type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'never'; + type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'followingOrFollower' | 'never'; }, { /** @enum {string} */ type: 'list'; @@ -3781,7 +3781,7 @@ export type components = { }]>; followRequestAccepted?: OneOf<[{ /** @enum {string} */ - type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'never'; + type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'followingOrFollower' | 'never'; }, { /** @enum {string} */ type: 'list'; @@ -3790,7 +3790,7 @@ export type components = { }]>; roleAssigned?: OneOf<[{ /** @enum {string} */ - type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'never'; + type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'followingOrFollower' | 'never'; }, { /** @enum {string} */ type: 'list'; @@ -3799,7 +3799,7 @@ export type components = { }]>; achievementEarned?: OneOf<[{ /** @enum {string} */ - type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'never'; + type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'followingOrFollower' | 'never'; }, { /** @enum {string} */ type: 'list'; @@ -3808,7 +3808,7 @@ export type components = { }]>; app?: OneOf<[{ /** @enum {string} */ - type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'never'; + type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'followingOrFollower' | 'never'; }, { /** @enum {string} */ type: 'list'; @@ -3817,7 +3817,7 @@ export type components = { }]>; test?: OneOf<[{ /** @enum {string} */ - type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'never'; + type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'followingOrFollower' | 'never'; }, { /** @enum {string} */ type: 'list'; @@ -8436,7 +8436,7 @@ export type operations = { notificationRecieveConfig: { note?: OneOf<[{ /** @enum {string} */ - type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'never'; + type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'followingOrFollower' | 'never'; }, { /** @enum {string} */ type: 'list'; @@ -8445,7 +8445,7 @@ export type operations = { }]>; follow?: OneOf<[{ /** @enum {string} */ - type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'never'; + type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'followingOrFollower' | 'never'; }, { /** @enum {string} */ type: 'list'; @@ -8454,7 +8454,7 @@ export type operations = { }]>; mention?: OneOf<[{ /** @enum {string} */ - type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'never'; + type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'followingOrFollower' | 'never'; }, { /** @enum {string} */ type: 'list'; @@ -8463,7 +8463,7 @@ export type operations = { }]>; reply?: OneOf<[{ /** @enum {string} */ - type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'never'; + type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'followingOrFollower' | 'never'; }, { /** @enum {string} */ type: 'list'; @@ -8472,7 +8472,7 @@ export type operations = { }]>; renote?: OneOf<[{ /** @enum {string} */ - type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'never'; + type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'followingOrFollower' | 'never'; }, { /** @enum {string} */ type: 'list'; @@ -8481,7 +8481,7 @@ export type operations = { }]>; quote?: OneOf<[{ /** @enum {string} */ - type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'never'; + type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'followingOrFollower' | 'never'; }, { /** @enum {string} */ type: 'list'; @@ -8490,7 +8490,7 @@ export type operations = { }]>; reaction?: OneOf<[{ /** @enum {string} */ - type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'never'; + type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'followingOrFollower' | 'never'; }, { /** @enum {string} */ type: 'list'; @@ -8499,7 +8499,7 @@ export type operations = { }]>; pollEnded?: OneOf<[{ /** @enum {string} */ - type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'never'; + type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'followingOrFollower' | 'never'; }, { /** @enum {string} */ type: 'list'; @@ -8508,7 +8508,7 @@ export type operations = { }]>; receiveFollowRequest?: OneOf<[{ /** @enum {string} */ - type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'never'; + type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'followingOrFollower' | 'never'; }, { /** @enum {string} */ type: 'list'; @@ -8517,7 +8517,7 @@ export type operations = { }]>; followRequestAccepted?: OneOf<[{ /** @enum {string} */ - type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'never'; + type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'followingOrFollower' | 'never'; }, { /** @enum {string} */ type: 'list'; @@ -8526,7 +8526,7 @@ export type operations = { }]>; roleAssigned?: OneOf<[{ /** @enum {string} */ - type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'never'; + type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'followingOrFollower' | 'never'; }, { /** @enum {string} */ type: 'list'; @@ -8535,7 +8535,7 @@ export type operations = { }]>; achievementEarned?: OneOf<[{ /** @enum {string} */ - type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'never'; + type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'followingOrFollower' | 'never'; }, { /** @enum {string} */ type: 'list'; @@ -8544,7 +8544,7 @@ export type operations = { }]>; app?: OneOf<[{ /** @enum {string} */ - type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'never'; + type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'followingOrFollower' | 'never'; }, { /** @enum {string} */ type: 'list'; @@ -8553,7 +8553,7 @@ export type operations = { }]>; test?: OneOf<[{ /** @enum {string} */ - type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'never'; + type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'followingOrFollower' | 'never'; }, { /** @enum {string} */ type: 'list'; @@ -18787,7 +18787,7 @@ export type operations = { notificationRecieveConfig?: { note?: OneOf<[{ /** @enum {string} */ - type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'never'; + type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'followingOrFollower' | 'never'; }, { /** @enum {string} */ type: 'list'; @@ -18796,7 +18796,7 @@ export type operations = { }]>; follow?: OneOf<[{ /** @enum {string} */ - type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'never'; + type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'followingOrFollower' | 'never'; }, { /** @enum {string} */ type: 'list'; @@ -18805,7 +18805,7 @@ export type operations = { }]>; mention?: OneOf<[{ /** @enum {string} */ - type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'never'; + type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'followingOrFollower' | 'never'; }, { /** @enum {string} */ type: 'list'; @@ -18814,7 +18814,7 @@ export type operations = { }]>; reply?: OneOf<[{ /** @enum {string} */ - type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'never'; + type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'followingOrFollower' | 'never'; }, { /** @enum {string} */ type: 'list'; @@ -18823,7 +18823,7 @@ export type operations = { }]>; renote?: OneOf<[{ /** @enum {string} */ - type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'never'; + type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'followingOrFollower' | 'never'; }, { /** @enum {string} */ type: 'list'; @@ -18832,7 +18832,7 @@ export type operations = { }]>; quote?: OneOf<[{ /** @enum {string} */ - type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'never'; + type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'followingOrFollower' | 'never'; }, { /** @enum {string} */ type: 'list'; @@ -18841,7 +18841,7 @@ export type operations = { }]>; reaction?: OneOf<[{ /** @enum {string} */ - type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'never'; + type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'followingOrFollower' | 'never'; }, { /** @enum {string} */ type: 'list'; @@ -18850,7 +18850,7 @@ export type operations = { }]>; pollEnded?: OneOf<[{ /** @enum {string} */ - type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'never'; + type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'followingOrFollower' | 'never'; }, { /** @enum {string} */ type: 'list'; @@ -18859,7 +18859,7 @@ export type operations = { }]>; receiveFollowRequest?: OneOf<[{ /** @enum {string} */ - type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'never'; + type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'followingOrFollower' | 'never'; }, { /** @enum {string} */ type: 'list'; @@ -18868,7 +18868,7 @@ export type operations = { }]>; followRequestAccepted?: OneOf<[{ /** @enum {string} */ - type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'never'; + type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'followingOrFollower' | 'never'; }, { /** @enum {string} */ type: 'list'; @@ -18877,7 +18877,7 @@ export type operations = { }]>; roleAssigned?: OneOf<[{ /** @enum {string} */ - type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'never'; + type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'followingOrFollower' | 'never'; }, { /** @enum {string} */ type: 'list'; @@ -18886,7 +18886,7 @@ export type operations = { }]>; achievementEarned?: OneOf<[{ /** @enum {string} */ - type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'never'; + type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'followingOrFollower' | 'never'; }, { /** @enum {string} */ type: 'list'; @@ -18895,7 +18895,7 @@ export type operations = { }]>; app?: OneOf<[{ /** @enum {string} */ - type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'never'; + type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'followingOrFollower' | 'never'; }, { /** @enum {string} */ type: 'list'; @@ -18904,7 +18904,7 @@ export type operations = { }]>; test?: OneOf<[{ /** @enum {string} */ - type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'never'; + type: 'all' | 'following' | 'follower' | 'mutualFollow' | 'followingOrFollower' | 'never'; }, { /** @enum {string} */ type: 'list'; From b7d9d1620161a728e34ab20d8ea99160b3eb4196 Mon Sep 17 00:00:00 2001 From: okayurisotto <47853651+okayurisotto@users.noreply.github.com> Date: Wed, 28 Feb 2024 15:34:58 +0900 Subject: [PATCH 03/21] =?UTF-8?q?refactor(backend):=20=E3=83=8E=E3=83=BC?= =?UTF-8?q?=E3=83=88=E3=81=AE=E3=82=A8=E3=82=AF=E3=82=B9=E3=83=9D=E3=83=BC?= =?UTF-8?q?=E3=83=88=E5=87=A6=E7=90=86=E3=81=A7Streams=20API=E3=82=92?= =?UTF-8?q?=E4=BD=BF=E3=81=86=E3=82=88=E3=81=86=E3=81=AB=20(#13465)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * refactor(backend): ノートのエクスポート処理でStreams APIを使うように * fixup! refactor(backend): ノートのエクスポート処理でStreams APIを使うように `await`忘れにより、ジョブがすぐに完了したことになり削除されてしまっていた。 それによって、`NoteStream`内での`updateProgress`メソッドの呼び出しで、`Missing key for job`のエラーが発生することがあった。 --------- Co-authored-by: syuilo <4439005+syuilo@users.noreply.github.com> --- packages/backend/src/misc/FileWriterStream.ts | 31 ++++ packages/backend/src/misc/JsonArrayStream.ts | 30 ++++ .../processors/ExportNotesProcessorService.ts | 164 +++++++++--------- 3 files changed, 146 insertions(+), 79 deletions(-) create mode 100644 packages/backend/src/misc/FileWriterStream.ts create mode 100644 packages/backend/src/misc/JsonArrayStream.ts diff --git a/packages/backend/src/misc/FileWriterStream.ts b/packages/backend/src/misc/FileWriterStream.ts new file mode 100644 index 0000000000..828851df0e --- /dev/null +++ b/packages/backend/src/misc/FileWriterStream.ts @@ -0,0 +1,31 @@ +import * as fs from 'node:fs/promises'; +import type { PathLike } from 'node:fs'; + +/** + * `fs.createWriteStream()`相当のことを行う`WritableStream` (Web標準) + */ +export class FileWriterStream extends WritableStream { + constructor(path: PathLike) { + let file: fs.FileHandle | null = null; + + super({ + start: async () => { + file = await fs.open(path, 'a'); + }, + write: async (chunk, controller) => { + if (file === null) { + controller.error(); + throw new Error(); + } + + await file.write(chunk); + }, + close: async () => { + await file?.close(); + }, + abort: async () => { + await file?.close(); + }, + }); + } +} diff --git a/packages/backend/src/misc/JsonArrayStream.ts b/packages/backend/src/misc/JsonArrayStream.ts new file mode 100644 index 0000000000..ad35bb3a79 --- /dev/null +++ b/packages/backend/src/misc/JsonArrayStream.ts @@ -0,0 +1,30 @@ +import { TransformStream } from 'node:stream/web'; + +/** + * ストリームに流れてきた各データについて`JSON.stringify()`した上で、それらを一つの配列にまとめる + */ +export class JsonArrayStream extends TransformStream { + constructor() { + /** 最初の要素かどうかを変数に記録 */ + let isFirst = true; + + super({ + start(controller) { + controller.enqueue('['); + }, + flush(controller) { + controller.enqueue(']'); + }, + transform(chunk, controller) { + if (isFirst) { + isFirst = false; + } else { + // 妥当なJSON配列にするためには最初以外の要素の前に`,`を挿入しなければならない + controller.enqueue(',\n'); + } + + controller.enqueue(JSON.stringify(chunk)); + }, + }); + } +} diff --git a/packages/backend/src/queue/processors/ExportNotesProcessorService.ts b/packages/backend/src/queue/processors/ExportNotesProcessorService.ts index f2ae0ce4b4..c7611012d7 100644 --- a/packages/backend/src/queue/processors/ExportNotesProcessorService.ts +++ b/packages/backend/src/queue/processors/ExportNotesProcessorService.ts @@ -3,7 +3,7 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import * as fs from 'node:fs'; +import { ReadableStream, TextEncoderStream } from 'node:stream/web'; import { Inject, Injectable } from '@nestjs/common'; import { MoreThan } from 'typeorm'; import { format as dateFormat } from 'date-fns'; @@ -18,10 +18,82 @@ import { bindThis } from '@/decorators.js'; import { DriveFileEntityService } from '@/core/entities/DriveFileEntityService.js'; import { Packed } from '@/misc/json-schema.js'; import { IdService } from '@/core/IdService.js'; +import { JsonArrayStream } from '@/misc/JsonArrayStream.js'; +import { FileWriterStream } from '@/misc/FileWriterStream.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; import type * as Bull from 'bullmq'; import type { DbJobDataWithUser } from '../types.js'; +class NoteStream extends ReadableStream> { + constructor( + job: Bull.Job, + notesRepository: NotesRepository, + pollsRepository: PollsRepository, + driveFileEntityService: DriveFileEntityService, + idService: IdService, + userId: string, + ) { + let exportedNotesCount = 0; + let cursor: MiNote['id'] | null = null; + + const serialize = ( + note: MiNote, + poll: MiPoll | null, + files: Packed<'DriveFile'>[], + ): Record => { + return { + id: note.id, + text: note.text, + createdAt: idService.parse(note.id).date.toISOString(), + fileIds: note.fileIds, + files: files, + replyId: note.replyId, + renoteId: note.renoteId, + poll: poll, + cw: note.cw, + visibility: note.visibility, + visibleUserIds: note.visibleUserIds, + localOnly: note.localOnly, + reactionAcceptance: note.reactionAcceptance, + }; + }; + + super({ + async pull(controller): Promise { + const notes = await notesRepository.find({ + where: { + userId, + ...(cursor !== null ? { id: MoreThan(cursor) } : {}), + }, + take: 100, // 100件ずつ取得 + order: { id: 1 }, + }); + + if (notes.length === 0) { + job.updateProgress(100); + controller.close(); + } + + cursor = notes.at(-1)?.id ?? null; + + for (const note of notes) { + const poll = note.hasPoll + ? await pollsRepository.findOneByOrFail({ noteId: note.id }) // N+1 + : null; + const files = await driveFileEntityService.packManyByIds(note.fileIds); // N+1 + const content = serialize(note, poll, files); + + controller.enqueue(content); + exportedNotesCount++; + } + + const total = await notesRepository.countBy({ userId }); + job.updateProgress(exportedNotesCount / total); + }, + }); + } +} + @Injectable() export class ExportNotesProcessorService { private logger: Logger; @@ -59,67 +131,19 @@ export class ExportNotesProcessorService { this.logger.info(`Temp file is ${path}`); try { - const stream = fs.createWriteStream(path, { flags: 'a' }); + // メモリが足りなくならないようにストリームで処理する + await new NoteStream( + job, + this.notesRepository, + this.pollsRepository, + this.driveFileEntityService, + this.idService, + user.id, + ) + .pipeThrough(new JsonArrayStream()) + .pipeThrough(new TextEncoderStream()) + .pipeTo(new FileWriterStream(path)); - const write = (text: string): Promise => { - return new Promise((res, rej) => { - stream.write(text, err => { - if (err) { - this.logger.error(err); - rej(err); - } else { - res(); - } - }); - }); - }; - - await write('['); - - let exportedNotesCount = 0; - let cursor: MiNote['id'] | null = null; - - while (true) { - const notes = await this.notesRepository.find({ - where: { - userId: user.id, - ...(cursor ? { id: MoreThan(cursor) } : {}), - }, - take: 100, - order: { - id: 1, - }, - }) as MiNote[]; - - if (notes.length === 0) { - job.updateProgress(100); - break; - } - - cursor = notes.at(-1)?.id ?? null; - - for (const note of notes) { - let poll: MiPoll | undefined; - if (note.hasPoll) { - poll = await this.pollsRepository.findOneByOrFail({ noteId: note.id }); - } - const files = await this.driveFileEntityService.packManyByIds(note.fileIds); - const content = JSON.stringify(this.serialize(note, poll, files)); - const isFirst = exportedNotesCount === 0; - await write(isFirst ? content : ',\n' + content); - exportedNotesCount++; - } - - const total = await this.notesRepository.countBy({ - userId: user.id, - }); - - job.updateProgress(exportedNotesCount / total); - } - - await write(']'); - - stream.end(); this.logger.succ(`Exported to: ${path}`); const fileName = 'notes-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.json'; @@ -130,22 +154,4 @@ export class ExportNotesProcessorService { cleanup(); } } - - private serialize(note: MiNote, poll: MiPoll | null = null, files: Packed<'DriveFile'>[]): Record { - return { - id: note.id, - text: note.text, - createdAt: this.idService.parse(note.id).date.toISOString(), - fileIds: note.fileIds, - files: files, - replyId: note.replyId, - renoteId: note.renoteId, - poll: poll, - cw: note.cw, - visibility: note.visibility, - visibleUserIds: note.visibleUserIds, - localOnly: note.localOnly, - reactionAcceptance: note.reactionAcceptance, - }; - } } From 664aeb3ced65f3911c8a21c2d5ffbd1035aec31a Mon Sep 17 00:00:00 2001 From: zyoshoka <107108195+zyoshoka@users.noreply.github.com> Date: Wed, 28 Feb 2024 17:43:17 +0900 Subject: [PATCH 04/21] =?UTF-8?q?fix(backend):=20=E3=83=AA=E3=83=8E?= =?UTF-8?q?=E3=83=BC=E3=83=88=E6=99=82=E3=81=AEHTL=E3=81=B8=E3=81=AE?= =?UTF-8?q?=E3=82=B9=E3=83=88=E3=83=AA=E3=83=BC=E3=83=9F=E3=83=B3=E3=82=B0?= =?UTF-8?q?=E3=81=AE=E6=84=8F=E5=9B=B3=E3=81=97=E3=81=AA=E3=81=84=E6=8C=99?= =?UTF-8?q?=E5=8B=95=E3=82=92=E4=BF=AE=E6=AD=A3=20(#13425)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(backend): リノート時のストリーミングの意図しない挙動を修正 * Update CHANGELOG.md * fix: 不要な返り値 * fix: 不適切な条件分岐を修正 * test(backend): add htl tests --------- Co-authored-by: syuilo <4439005+syuilo@users.noreply.github.com> --- CHANGELOG.md | 2 + .../api/stream/channels/home-timeline.ts | 10 ++- packages/backend/test/e2e/streaming.ts | 65 +++++++++++++++++-- packages/backend/test/utils.ts | 6 +- 4 files changed, 75 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 010d5aed7a..f52226a635 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,8 @@ - Fix: nodeinfoにenableMcaptchaとenableTurnstileが無いのを修正 - エンドポイント`flash/update`の`flashId`以外のパラメータは必須ではなくなりました - Fix: 禁止キーワードを含むノートがDelayed Queueに追加されて再処理される問題を修正 +- Fix: 自分がフォローしていないアカウントのフォロワー限定ノートが閲覧できることがある問題を修正 +- Fix: タイムラインのオプションで「リノートを表示」を無効にしている際、投票のみの引用リノートが流れてこない問題を修正 - エンドポイント`admin/emoji/update`の各種修正 - 必須パラメータを`id`または`name`のいずれかのみに - `id`の代わりに`name`で絵文字を指定可能に（`id`・`name`両指定時は従来通り`name`を変更する挙動） diff --git a/packages/backend/src/server/api/stream/channels/home-timeline.ts b/packages/backend/src/server/api/stream/channels/home-timeline.ts index ce9d7f5647..f45bf8622e 100644 --- a/packages/backend/src/server/api/stream/channels/home-timeline.ts +++ b/packages/backend/src/server/api/stream/channels/home-timeline.ts @@ -71,7 +71,15 @@ class HomeTimelineChannel extends Channel { } } - if (note.renote && note.text == null && (note.fileIds == null || note.fileIds.length === 0) && !this.withRenotes) return; + // 純粋なリノート（引用リノートでないリノート）の場合 + if (note.renote && note.text == null && (note.fileIds == null || note.fileIds.length === 0) && note.poll == null) { + if (!this.withRenotes) return; + if (note.renote.reply) { + const reply = note.renote.reply; + // 自分のフォローしていないユーザーの visibility: followers な投稿への返信のリノートは弾く + if (reply.visibility === 'followers' && !Object.hasOwn(this.following, reply.userId)) return; + } + } // 流れてきたNoteがミュートしているユーザーが関わるものだったら無視する if (isUserRelated(note, this.userIdsWhoMeMuting)) return; diff --git a/packages/backend/test/e2e/streaming.ts b/packages/backend/test/e2e/streaming.ts index 071daa275f..13d5a683ba 100644 --- a/packages/backend/test/e2e/streaming.ts +++ b/packages/backend/test/e2e/streaming.ts @@ -40,9 +40,9 @@ describe('Streaming', () => { let chinatsu: misskey.entities.SignupResponse; let takumi: misskey.entities.SignupResponse; - let kyokoNote: any; - let kanakoNote: any; - let takumiNote: any; + let kyokoNote: misskey.entities.Note; + let kanakoNote: misskey.entities.Note; + let takumiNote: misskey.entities.Note; let list: any; beforeAll(async () => { @@ -68,6 +68,9 @@ describe('Streaming', () => { // Follow: ayano => akari await follow(ayano, akari); + // Follow: kyoko => chitose + await api('following/create', { userId: chitose.id }, kyoko); + // Mute: chitose => kanako await api('mute/create', { userId: kanako.id }, chitose); @@ -170,7 +173,28 @@ describe('Streaming', () => { */ test('フォローしているユーザーのフォローしていないユーザーの visibility: followers な投稿への返信が流れない', async () => { - // TODO + const chitoseNote = await post(chitose, { text: 'followers-only post', visibility: 'followers' }); + + const fired = await waitFire( + ayano, 'homeTimeline', // ayano:home + () => api('notes/create', { text: 'reply to chitose\'s followers-only post', replyId: chitoseNote.id }, kyoko), // kyoko's reply to chitose's followers-only post + msg => msg.type === 'note' && msg.body.userId === kyoko.id, // wait kyoko + ); + + assert.strictEqual(fired, false); + }); + + test('フォローしているユーザーのフォローしていないユーザーの visibility: followers な投稿への返信のリノートが流れない', async () => { + const chitoseNote = await post(chitose, { text: 'followers-only post', visibility: 'followers' }); + const kyokoReply = await post(kyoko, { text: 'reply to followers-only post', replyId: chitoseNote.id }); + + const fired = await waitFire( + ayano, 'homeTimeline', // ayano:home + () => api('notes/create', { renoteId: kyokoReply.id }, kyoko), // kyoko's renote of kyoko's reply to chitose's followers-only post + msg => msg.type === 'note' && msg.body.userId === kyoko.id, // wait kyoko + ); + + assert.strictEqual(fired, false); }); test('フォローしていないユーザーの投稿は流れない', async () => { @@ -202,6 +226,39 @@ describe('Streaming', () => { assert.strictEqual(fired, false); }); + + test('withRenotes: false のときリノートが流れない', async () => { + const fired = await waitFire( + ayano, 'homeTimeline', // ayano:home + () => api('notes/create', { renoteId: kyokoNote.id }, kyoko), // kyoko renote + msg => msg.type === 'note' && msg.body.userId === kyoko.id, // wait kyoko + { withRenotes: false }, + ); + + assert.strictEqual(fired, false); + }); + + test('withRenotes: false のとき引用リノートが流れる', async () => { + const fired = await waitFire( + ayano, 'homeTimeline', // ayano:home + () => api('notes/create', { text: 'quote', renoteId: kyokoNote.id }, kyoko), // kyoko quote + msg => msg.type === 'note' && msg.body.userId === kyoko.id, // wait kyoko + { withRenotes: false }, + ); + + assert.strictEqual(fired, true); + }); + + test('withRenotes: false のとき投票のみのリノートが流れる', async () => { + const fired = await waitFire( + ayano, 'homeTimeline', // ayano:home + () => api('notes/create', { poll: { choices: ['kinoko', 'takenoko'] }, renoteId: kyokoNote.id }, kyoko), // kyoko renote with poll + msg => msg.type === 'note' && msg.body.userId === kyoko.id, // wait kyoko + { withRenotes: false }, + ); + + assert.strictEqual(fired, true); + }); }); // Home describe('Local Timeline', () => { diff --git a/packages/backend/test/utils.ts b/packages/backend/test/utils.ts index a2220ffae6..cd5dddd68d 100644 --- a/packages/backend/test/utils.ts +++ b/packages/backend/test/utils.ts @@ -355,7 +355,7 @@ export const uploadUrl = async (user: UserToken, url: string): Promise) => any, params?: any): Promise { +export function connectStream(user: UserToken, channel: C, listener: (message: Record) => any, params?: misskey.Channels[C]['params']): Promise { return new Promise((res, rej) => { const url = new URL(`ws://127.0.0.1:${port}/streaming`); const options: ClientOptions = {}; @@ -390,7 +390,7 @@ export function connectStream(user: UserToken, channel: string, listener: (messa }); } -export const waitFire = async (user: UserToken, channel: string, trgr: () => any, cond: (msg: Record) => boolean, params?: any) => { +export const waitFire = async (user: UserToken, channel: C, trgr: () => any, cond: (msg: Record) => boolean, params?: misskey.Channels[C]['params']) => { return new Promise(async (res, rej) => { let timer: NodeJS.Timeout | null = null; @@ -435,7 +435,7 @@ export const waitFire = async (user: UserToken, channel: string, trgr: () => any */ export function makeStreamCatcher( user: UserToken, - channel: string, + channel: keyof misskey.Channels, cond: (message: Record) => boolean, extractor: (message: Record) => T, timeout = 60 * 1000): Promise { From 29350c9f334f426567e71eed479ae60ab4dea690 Mon Sep 17 00:00:00 2001 From: zyoshoka <107108195+zyoshoka@users.noreply.github.com> Date: Wed, 28 Feb 2024 18:26:38 +0900 Subject: [PATCH 05/21] =?UTF-8?q?refactor(frontend):=20`os.ts`=E5=91=A8?= =?UTF-8?q?=E3=82=8A=E3=81=AE=E3=83=AA=E3=83=95=E3=82=A1=E3=82=AF=E3=82=BF?= =?UTF-8?q?=E3=83=AA=E3=83=B3=E3=82=B0=20(#13186)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * refactor(frontend): `os.ts`周りのリファクタリング * refactor: apiWithDialogのdataの型付け * refactor: 不要なas anyを除去 * refactor: 返り値の型を明記、`selectDriveFolder`は`File`のほうに合わせるよう返り値を変更 * refactor: 返り値の型を改善 * refactor: フォームの型を改善 * refactor: 良い感じのimportに修正 * refactor: フォームの返り値の型を改善 * refactor: `popup()`の`props`に`ref`な値を入れるのを許可するように * fix: `os.input`系と`os.select`の返り値の型がおかしい問題とそれによるバグを修正 * Update CHANGELOG.md * Update CHANGELOG.md --------- Co-authored-by: syuilo <4439005+syuilo@users.noreply.github.com> --- CHANGELOG.md | 2 + packages/frontend/src/account.ts | 2 +- packages/frontend/src/components/MkDialog.vue | 29 +- .../src/components/MkDriveSelectDialog.vue | 8 +- .../src/components/MkEmojiPickerDialog.vue | 4 +- .../src/components/MkEmojiPickerWindow.vue | 49 --- .../frontend/src/components/MkFormDialog.vue | 56 ++-- packages/frontend/src/os.ts | 304 ++++++++++-------- .../frontend/src/pages/emoji-edit-dialog.vue | 2 +- packages/frontend/src/pages/notifications.vue | 4 +- .../frontend/src/pages/settings/drive.vue | 2 +- .../src/pages/settings/emoji-picker.vue | 2 +- .../pages/settings/preferences-backups.vue | 2 + packages/frontend/src/scripts/form.ts | 17 +- packages/frontend/src/ui/deck.vue | 20 +- .../frontend/src/widgets/WidgetSlideshow.vue | 4 +- 16 files changed, 257 insertions(+), 250 deletions(-) delete mode 100644 packages/frontend/src/components/MkEmojiPickerWindow.vue diff --git a/CHANGELOG.md b/CHANGELOG.md index f52226a635..bd35691871 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,8 @@ - Fix: MFMのオートコンプリートが出るべき状況で出ないことがある問題を修正 - Fix: チャートのラベルが消えている問題を修正 - Fix: 画面表示後最初の音声再生が爆音になることがある問題を修正 +- Fix: 設定のバックアップ作成時に名前を入力しなかった場合、ローカライゼーションがおかしくなる問題を修正 +- Fix: ページ`/admin/emojis`の絵文字編集ダイアログで「リアクションとして使えるロール」を追加する際に何も選択せずOKを押下すると画面が固まる問題を修正 - Fix: 絵文字サジェストの順位で、絵文字自体の名前が同じものよりもタグで一致しているものが優先されてしまう問題を修正 ### Server diff --git a/packages/frontend/src/account.ts b/packages/frontend/src/account.ts index e606fe368c..7f20e0b1a2 100644 --- a/packages/frontend/src/account.ts +++ b/packages/frontend/src/account.ts @@ -290,7 +290,7 @@ export async function openAccountMenu(opts: { text: i18n.ts.profile, to: `/@${ $i.username }`, avatar: $i, - }, { type: 'divider' }, ...(opts.includeCurrentAccount ? [createItem($i)] : []), ...accountItemPromises, { + }, { type: 'divider' as const }, ...(opts.includeCurrentAccount ? [createItem($i)] : []), ...accountItemPromises, { type: 'parent' as const, icon: 'ti ti-plus', text: i18n.ts.addAccount, diff --git a/packages/frontend/src/components/MkDialog.vue b/packages/frontend/src/components/MkDialog.vue index 4b7584faaa..4577d37c08 100644 --- a/packages/frontend/src/components/MkDialog.vue +++ b/packages/frontend/src/components/MkDialog.vue @@ -38,11 +38,6 @@ SPDX-License-Identifier: AGPL-3.0-only -
{{ okText ?? ((showCancelButton || input || select) ? i18n.ts.ok : i18n.ts.gotIt) }} @@ -64,7 +59,7 @@ import MkSelect from '@/components/MkSelect.vue'; import { i18n } from '@/i18n.js'; type Input = { - type: 'text' | 'number' | 'password' | 'email' | 'url' | 'date' | 'time' | 'search' | 'datetime-local'; + type?: 'text' | 'number' | 'password' | 'email' | 'url' | 'date' | 'time' | 'search' | 'datetime-local'; placeholder?: string | null; autocomplete?: string; default: string | number | null; @@ -74,22 +69,17 @@ type Input = { type Select = { items: { - value: string; + value: any; text: string; }[]; - groupedItems: { - label: string; - items: { - value: string; - text: string; - }[]; - }[]; default: string | null; }; +type Result = string | number | true | null; + const props = withDefaults(defineProps<{ type?: 'success' | 'error' | 'warning' | 'info' | 'question' | 'waiting'; - title: string; + title?: string; text?: string; input?: Input; select?: Select; @@ -113,7 +103,7 @@ const props = withDefaults(defineProps<{ }); const emit = defineEmits<{ - (ev: 'done', v: { canceled: boolean; result: any }): void; + (ev: 'done', v: { canceled: true } | { canceled: false, result: Result }): void; (ev: 'closed'): void; }>(); @@ -139,8 +129,11 @@ const okButtonDisabledReason = computed(); const dialog = shallowRef>(); -const selected = ref([]); +const selected = ref([]); function ok() { emit('done', selected.value); @@ -57,7 +57,7 @@ function cancel() { dialog.value?.close(); } -function onChangeSelection(files: Misskey.entities.DriveFile[]) { - selected.value = files; +function onChangeSelection(v: Misskey.entities.DriveFile[] | Misskey.entities.DriveFolder[]) { + selected.value = v; } diff --git a/packages/frontend/src/components/MkEmojiPickerDialog.vue b/packages/frontend/src/components/MkEmojiPickerDialog.vue index 59f4b51522..adcea839ee 100644 --- a/packages/frontend/src/components/MkEmojiPickerDialog.vue +++ b/packages/frontend/src/components/MkEmojiPickerDialog.vue @@ -56,7 +56,7 @@ const props = withDefaults(defineProps<{ }); const emit = defineEmits<{ - (ev: 'done', v: any): void; + (ev: 'done', v: string): void; (ev: 'close'): void; (ev: 'closed'): void; }>(); @@ -64,7 +64,7 @@ const emit = defineEmits<{ const modal = shallowRef>(); const picker = shallowRef>(); -function chosen(emoji: any) { +function chosen(emoji: string) { emit('done', emoji); if (props.choseAndClose) { modal.value?.close(); diff --git a/packages/frontend/src/components/MkEmojiPickerWindow.vue b/packages/frontend/src/components/MkEmojiPickerWindow.vue deleted file mode 100644 index 6952943345..0000000000 --- a/packages/frontend/src/components/MkEmojiPickerWindow.vue +++ /dev/null @@ -1,49 +0,0 @@ - - - - - - - diff --git a/packages/frontend/src/components/MkFormDialog.vue b/packages/frontend/src/components/MkFormDialog.vue index 0d8734799c..deedc5badb 100644 --- a/packages/frontend/src/components/MkFormDialog.vue +++ b/packages/frontend/src/components/MkFormDialog.vue @@ -21,37 +21,37 @@ SPDX-License-Identifier: AGPL-3.0-only
-