From f4838e50b4043f917020dd1cfa7b75da087ff8f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=81=8B=E3=81=A3=E3=81=93=E3=81=8B=E3=82=8A?= <67428053+kakkokari-gtyih@users.noreply.github.com> Date: Thu, 21 Mar 2024 07:51:01 +0900 Subject: [PATCH 1/3] =?UTF-8?q?enhance(antenna):=20Bot=E3=81=AE=E6=8A=95?= =?UTF-8?q?=E7=A8=BF=E3=82=92=E9=99=A4=E5=A4=96=E3=81=A7=E3=81=8D=E3=82=8B?= =?UTF-8?q?=E3=82=88=E3=81=86=E3=81=AB=20(#13603)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * enhance(antenna): Botの投稿を除外できるように (MisskeyIO#545) (cherry picked from commit a95ce067c6cf0a93647e358aabc984bdbe99e952) * Update Changelog * remove translations * spdx --------- Co-authored-by: まっちゃとーにゅ <17376330+u1-liquid@users.noreply.github.com> --- CHANGELOG.md | 2 ++ locales/index.d.ts | 4 ++++ locales/ja-JP.yml | 1 + .../1710919614510-antenna-exclude-bots.js | 16 ++++++++++++++++ packages/backend/src/core/AntennaService.ts | 6 ++++-- .../src/core/entities/AntennaEntityService.ts | 1 + packages/backend/src/models/Antenna.ts | 5 +++++ .../backend/src/models/json-schema/antenna.ts | 5 +++++ .../processors/ExportAntennasProcessorService.ts | 1 + .../processors/ImportAntennasProcessorService.ts | 2 ++ .../src/server/api/endpoints/antennas/create.ts | 2 ++ .../src/server/api/endpoints/antennas/update.ts | 2 ++ packages/backend/test/e2e/antennas.ts | 2 ++ .../frontend/src/pages/my-antennas/create.vue | 1 + .../frontend/src/pages/my-antennas/editor.vue | 3 +++ packages/misskey-js/src/autogen/types.ts | 4 ++++ 16 files changed, 55 insertions(+), 2 deletions(-) create mode 100644 packages/backend/migration/1710919614510-antenna-exclude-bots.js diff --git a/CHANGELOG.md b/CHANGELOG.md index 18dd07f1c3..0dce1a0496 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,8 @@ ## Unreleased ### General +- Enhance: アンテナでBotによるノートを除外できるように + (Cherry-picked from https://github.com/MisskeyIO/misskey/pull/545) - Fix: Play作成時に設定した公開範囲が機能していない問題を修正 ### Client diff --git a/locales/index.d.ts b/locales/index.d.ts index 7f4ec7ecb0..afb4adac6c 100644 --- a/locales/index.d.ts +++ b/locales/index.d.ts @@ -1616,6 +1616,10 @@ export interface Locale extends ILocale { * 除外キーワード */ "antennaExcludeKeywords": string; + /** + * Botアカウントを除外 + */ + "antennaExcludeBots": string; /** * スペースで区切るとAND指定になり、改行で区切るとOR指定になります */ diff --git a/locales/ja-JP.yml b/locales/ja-JP.yml index 8b44ac2121..a64c83b10f 100644 --- a/locales/ja-JP.yml +++ b/locales/ja-JP.yml @@ -400,6 +400,7 @@ name: "名前" antennaSource: "受信ソース" antennaKeywords: "受信キーワード" antennaExcludeKeywords: "除外キーワード" +antennaExcludeBots: "Botアカウントを除外" antennaKeywordsDescription: "スペースで区切るとAND指定になり、改行で区切るとOR指定になります" notifyAntenna: "新しいノートを通知する" withFileAntenna: "ファイルが添付されたノートのみ" diff --git a/packages/backend/migration/1710919614510-antenna-exclude-bots.js b/packages/backend/migration/1710919614510-antenna-exclude-bots.js new file mode 100644 index 0000000000..fac84317cc --- /dev/null +++ b/packages/backend/migration/1710919614510-antenna-exclude-bots.js @@ -0,0 +1,16 @@ +/* + * SPDX-FileCopyrightText: syuilo and misskey-project + * SPDX-License-Identifier: AGPL-3.0-only + */ + +export class AntennaExcludeBots1710919614510 { + name = 'AntennaExcludeBots1710919614510' + + async up(queryRunner) { + await queryRunner.query(`ALTER TABLE "antenna" ADD "excludeBots" boolean NOT NULL DEFAULT false`); + } + + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "antenna" DROP COLUMN "excludeBots"`); + } +} diff --git a/packages/backend/src/core/AntennaService.ts b/packages/backend/src/core/AntennaService.ts index 4f956a43ed..793d8974b3 100644 --- a/packages/backend/src/core/AntennaService.ts +++ b/packages/backend/src/core/AntennaService.ts @@ -92,7 +92,7 @@ export class AntennaService implements OnApplicationShutdown { } @bindThis - public async addNoteToAntennas(note: MiNote, noteUser: { id: MiUser['id']; username: string; host: string | null; }): Promise { + public async addNoteToAntennas(note: MiNote, noteUser: { id: MiUser['id']; username: string; host: string | null; isBot: boolean; }): Promise { const antennas = await this.getAntennas(); const antennasWithMatchResult = await Promise.all(antennas.map(antenna => this.checkHitAntenna(antenna, note, noteUser).then(hit => [antenna, hit] as const))); const matchedAntennas = antennasWithMatchResult.filter(([, hit]) => hit).map(([antenna]) => antenna); @@ -110,10 +110,12 @@ export class AntennaService implements OnApplicationShutdown { // NOTE: フォローしているユーザーのノート、リストのユーザーのノート、グループのユーザーのノート指定はパフォーマンス上の理由で無効になっている @bindThis - public async checkHitAntenna(antenna: MiAntenna, note: (MiNote | Packed<'Note'>), noteUser: { id: MiUser['id']; username: string; host: string | null; }): Promise { + public async checkHitAntenna(antenna: MiAntenna, note: (MiNote | Packed<'Note'>), noteUser: { id: MiUser['id']; username: string; host: string | null; isBot: boolean; }): Promise { if (note.visibility === 'specified') return false; if (note.visibility === 'followers') return false; + if (antenna.excludeBots && noteUser.isBot) return false; + if (antenna.localOnly && noteUser.host != null) return false; if (!antenna.withReplies && note.replyId != null) return false; diff --git a/packages/backend/src/core/entities/AntennaEntityService.ts b/packages/backend/src/core/entities/AntennaEntityService.ts index 64d6a3c978..3ec8efa6bf 100644 --- a/packages/backend/src/core/entities/AntennaEntityService.ts +++ b/packages/backend/src/core/entities/AntennaEntityService.ts @@ -39,6 +39,7 @@ export class AntennaEntityService { caseSensitive: antenna.caseSensitive, localOnly: antenna.localOnly, notify: antenna.notify, + excludeBots: antenna.excludeBots, withReplies: antenna.withReplies, withFile: antenna.withFile, isActive: antenna.isActive, diff --git a/packages/backend/src/models/Antenna.ts b/packages/backend/src/models/Antenna.ts index 332a899768..f5e819059e 100644 --- a/packages/backend/src/models/Antenna.ts +++ b/packages/backend/src/models/Antenna.ts @@ -72,6 +72,11 @@ export class MiAntenna { }) public caseSensitive: boolean; + @Column('boolean', { + default: false, + }) + public excludeBots: boolean; + @Column('boolean', { default: false, }) diff --git a/packages/backend/src/models/json-schema/antenna.ts b/packages/backend/src/models/json-schema/antenna.ts index 74622b6193..78cf6d3ba2 100644 --- a/packages/backend/src/models/json-schema/antenna.ts +++ b/packages/backend/src/models/json-schema/antenna.ts @@ -76,6 +76,11 @@ export const packedAntennaSchema = { type: 'boolean', optional: false, nullable: false, }, + excludeBots: { + type: 'boolean', + optional: false, nullable: false, + default: false, + }, withReplies: { type: 'boolean', optional: false, nullable: false, diff --git a/packages/backend/src/queue/processors/ExportAntennasProcessorService.ts b/packages/backend/src/queue/processors/ExportAntennasProcessorService.ts index af48bad417..1d8e90f367 100644 --- a/packages/backend/src/queue/processors/ExportAntennasProcessorService.ts +++ b/packages/backend/src/queue/processors/ExportAntennasProcessorService.ts @@ -81,6 +81,7 @@ export class ExportAntennasProcessorService { }) : null, caseSensitive: antenna.caseSensitive, localOnly: antenna.localOnly, + excludeBots: antenna.excludeBots, withReplies: antenna.withReplies, withFile: antenna.withFile, notify: antenna.notify, diff --git a/packages/backend/src/queue/processors/ImportAntennasProcessorService.ts b/packages/backend/src/queue/processors/ImportAntennasProcessorService.ts index 951b560597..ff1c04de06 100644 --- a/packages/backend/src/queue/processors/ImportAntennasProcessorService.ts +++ b/packages/backend/src/queue/processors/ImportAntennasProcessorService.ts @@ -44,6 +44,7 @@ const validate = new Ajv().compile({ } }, caseSensitive: { type: 'boolean' }, localOnly: { type: 'boolean' }, + excludeBots: { type: 'boolean' }, withReplies: { type: 'boolean' }, withFile: { type: 'boolean' }, notify: { type: 'boolean' }, @@ -88,6 +89,7 @@ export class ImportAntennasProcessorService { users: (antenna.src === 'list' && antenna.userListAccts !== null ? antenna.userListAccts : antenna.users).filter(Boolean), caseSensitive: antenna.caseSensitive, localOnly: antenna.localOnly, + excludeBots: antenna.excludeBots, withReplies: antenna.withReplies, withFile: antenna.withFile, notify: antenna.notify, diff --git a/packages/backend/src/server/api/endpoints/antennas/create.ts b/packages/backend/src/server/api/endpoints/antennas/create.ts index 191de8f833..57c8eb4958 100644 --- a/packages/backend/src/server/api/endpoints/antennas/create.ts +++ b/packages/backend/src/server/api/endpoints/antennas/create.ts @@ -64,6 +64,7 @@ export const paramDef = { } }, caseSensitive: { type: 'boolean' }, localOnly: { type: 'boolean' }, + excludeBots: { type: 'boolean' }, withReplies: { type: 'boolean' }, withFile: { type: 'boolean' }, notify: { type: 'boolean' }, @@ -124,6 +125,7 @@ export default class extends Endpoint { // eslint- users: ps.users, caseSensitive: ps.caseSensitive, localOnly: ps.localOnly, + excludeBots: ps.excludeBots, withReplies: ps.withReplies, withFile: ps.withFile, notify: ps.notify, diff --git a/packages/backend/src/server/api/endpoints/antennas/update.ts b/packages/backend/src/server/api/endpoints/antennas/update.ts index 76a34924a0..e6720aacf8 100644 --- a/packages/backend/src/server/api/endpoints/antennas/update.ts +++ b/packages/backend/src/server/api/endpoints/antennas/update.ts @@ -63,6 +63,7 @@ export const paramDef = { } }, caseSensitive: { type: 'boolean' }, localOnly: { type: 'boolean' }, + excludeBots: { type: 'boolean' }, withReplies: { type: 'boolean' }, withFile: { type: 'boolean' }, notify: { type: 'boolean' }, @@ -120,6 +121,7 @@ export default class extends Endpoint { // eslint- users: ps.users, caseSensitive: ps.caseSensitive, localOnly: ps.localOnly, + excludeBots: ps.excludeBots, withReplies: ps.withReplies, withFile: ps.withFile, notify: ps.notify, diff --git a/packages/backend/test/e2e/antennas.ts b/packages/backend/test/e2e/antennas.ts index 7370b1963c..cf5c7dd130 100644 --- a/packages/backend/test/e2e/antennas.ts +++ b/packages/backend/test/e2e/antennas.ts @@ -44,6 +44,7 @@ describe('アンテナ', () => { users: [''], withFile: false, withReplies: false, + excludeBots: false, }; let root: User; @@ -156,6 +157,7 @@ describe('アンテナ', () => { users: [''], withFile: false, withReplies: false, + excludeBots: false, localOnly: false, }; assert.deepStrictEqual(response, expected); diff --git a/packages/frontend/src/pages/my-antennas/create.vue b/packages/frontend/src/pages/my-antennas/create.vue index 8b3b3cfbfd..2d026d2fa9 100644 --- a/packages/frontend/src/pages/my-antennas/create.vue +++ b/packages/frontend/src/pages/my-antennas/create.vue @@ -26,6 +26,7 @@ const draft = ref({ users: [], keywords: [], excludeKeywords: [], + excludeBots: false, withReplies: false, caseSensitive: false, localOnly: false, diff --git a/packages/frontend/src/pages/my-antennas/editor.vue b/packages/frontend/src/pages/my-antennas/editor.vue index c6dcbadd9b..97edbc44ce 100644 --- a/packages/frontend/src/pages/my-antennas/editor.vue +++ b/packages/frontend/src/pages/my-antennas/editor.vue @@ -26,6 +26,7 @@ SPDX-License-Identifier: AGPL-3.0-only + {{ i18n.ts.antennaExcludeBots }} {{ i18n.ts.withReplies }} @@ -78,6 +79,7 @@ const keywords = ref(props.antenna.keywords.map(x => x.join(' ')).join(' const excludeKeywords = ref(props.antenna.excludeKeywords.map(x => x.join(' ')).join('\n')); const caseSensitive = ref(props.antenna.caseSensitive); const localOnly = ref(props.antenna.localOnly); +const excludeBots = ref(props.antenna.excludeBots); const withReplies = ref(props.antenna.withReplies); const withFile = ref(props.antenna.withFile); const notify = ref(props.antenna.notify); @@ -94,6 +96,7 @@ async function saveAntenna() { name: name.value, src: src.value, userListId: userListId.value, + excludeBots: excludeBots.value, withReplies: withReplies.value, withFile: withFile.value, notify: notify.value, diff --git a/packages/misskey-js/src/autogen/types.ts b/packages/misskey-js/src/autogen/types.ts index 3c862f690e..636bc62aaa 100644 --- a/packages/misskey-js/src/autogen/types.ts +++ b/packages/misskey-js/src/autogen/types.ts @@ -4434,6 +4434,8 @@ export type components = { localOnly: boolean; notify: boolean; /** @default false */ + excludeBots: boolean; + /** @default false */ withReplies: boolean; withFile: boolean; isActive: boolean; @@ -9654,6 +9656,7 @@ export type operations = { users: string[]; caseSensitive: boolean; localOnly?: boolean; + excludeBots?: boolean; withReplies: boolean; withFile: boolean; notify: boolean; @@ -9935,6 +9938,7 @@ export type operations = { users?: string[]; caseSensitive?: boolean; localOnly?: boolean; + excludeBots?: boolean; withReplies?: boolean; withFile?: boolean; notify?: boolean; From 831c74a25b2db0ba3f6d43a9a1a9072d342b2822 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=81=8A=E3=81=95=E3=82=80=E3=81=AE=E3=81=B2=E3=81=A8?= <46447427+samunohito@users.noreply.github.com> Date: Thu, 21 Mar 2024 18:46:42 +0900 Subject: [PATCH 2/3] =?UTF-8?q?fix:=20URL=E3=83=97=E3=83=AC=E3=83=93?= =?UTF-8?q?=E3=83=A5=E3=83=BC=E3=81=AE=E5=8B=95=E4=BD=9C=E6=94=B9=E5=96=84?= =?UTF-8?q?=EF=BC=8B=E5=8B=95=E4=BD=9C=E8=A8=AD=E5=AE=9A=E3=82=92=E5=8F=AF?= =?UTF-8?q?=E8=83=BD=E3=81=AB=E3=81=99=E3=82=8B=20(#13579)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * wip * support new version * URLプレビュー無効化時、フロント側も非表示にしてリクエストをしないようにする * fix lint * fix lint * tweak preview request error handles * fix: CHANGELOG.md * fix * fix --------- Co-authored-by: syuilo <4439005+syuilo@users.noreply.github.com> --- CHANGELOG.md | 5 ++ locales/index.d.ts | 58 +++++++++++++++ locales/ja-JP.yml | 15 ++++ .../1710512074000-url-preview-meta.js | 42 +++++++++++ packages/backend/package.json | 2 +- .../src/core/entities/MetaEntityService.ts | 1 + packages/backend/src/models/Meta.ts | 38 ++++++++-- .../backend/src/models/json-schema/meta.ts | 4 + .../src/server/api/endpoints/admin/meta.ts | 34 ++++++++- .../server/api/endpoints/admin/update-meta.ts | 41 ++++++++-- .../src/server/web/UrlPreviewService.ts | 67 +++++++++++++---- packages/frontend/src/components/MkLink.vue | 17 +++-- packages/frontend/src/components/MkNote.vue | 7 +- .../src/components/MkNoteDetailed.vue | 5 +- .../frontend/src/components/MkUrlPreview.vue | 12 +-- .../frontend/src/components/global/MkUrl.vue | 3 +- .../src/components/page/page.text.vue | 5 +- packages/frontend/src/instance.ts | 2 + .../frontend/src/pages/admin/security.vue | 16 ---- .../frontend/src/pages/admin/settings.vue | 74 ++++++++++++++++++- packages/misskey-js/src/autogen/types.ts | 20 ++++- pnpm-lock.yaml | 18 ++++- 22 files changed, 420 insertions(+), 66 deletions(-) create mode 100644 packages/backend/migration/1710512074000-url-preview-meta.js diff --git a/CHANGELOG.md b/CHANGELOG.md index 0dce1a0496..188e146cdd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,10 @@ ## Unreleased +### Note +- コントロールパネル内にあるサマリープロキシの設定個所がセキュリティから全般へ変更となります。 + ### General +- Enhance: URLプレビューの有効化・無効化を設定できるように #13569 - Enhance: アンテナでBotによるノートを除外できるように (Cherry-picked from https://github.com/MisskeyIO/misskey/pull/545) - Fix: Play作成時に設定した公開範囲が機能していない問題を修正 @@ -25,6 +29,7 @@ ### Server - Enhance: エンドポイント`antennas/update`の必須項目を`antennaId`のみに +- Enhance: misskey-dev/summaly@5.1.0の取り込み（プレビュー生成処理の効率化） - Fix: フォローリクエストを作成する際に既存のものは削除するように (Cherry-picked from https://activitypub.software/TransFem-org/Sharkey/-/merge_requests/440) diff --git a/locales/index.d.ts b/locales/index.d.ts index afb4adac6c..70586d7a87 100644 --- a/locales/index.d.ts +++ b/locales/index.d.ts @@ -4916,6 +4916,10 @@ export interface Locale extends ILocale { * リトライ */ "gameRetry": string; + /** + * 使用しない場合は空欄にしてください + */ + "notUsePleaseLeaveBlank": string; "_bubbleGame": { /** * 遊び方 @@ -9768,6 +9772,60 @@ export interface Locale extends ILocale { */ "header": string; }; + "_urlPreviewSetting": { + /** + * URLプレビューの設定 + */ + "title": string; + /** + * URLプレビューを有効にする + */ + "enable": string; + /** + * プレビュー取得時のタイムアウト(ms) + */ + "timeout": string; + /** + * プレビュー取得の所要時間がこの値を超えた場合、プレビューは生成されません。 + */ + "timeoutDescription": string; + /** + * Content-Lengthの最大値(byte) + */ + "maximumContentLength": string; + /** + * Content-Lengthがこの値を超えた場合、プレビューは生成されません。 + */ + "maximumContentLengthDescription": string; + /** + * Content-Lengthが取得できた場合のみプレビューを生成 + */ + "requireContentLength": string; + /** + * 相手サーバがContent-Lengthを返さない場合、プレビューは生成されません。 + */ + "requireContentLengthDescription": string; + /** + * User-Agent + */ + "userAgent": string; + /** + * プレビュー取得時に使用されるUser-Agentを設定します。空欄の場合、デフォルトのUser-Agentが使用されます。 + */ + "userAgentDescription": string; + /** + * プレビューを生成するプロキシのエンドポイント + */ + "summaryProxy": string; + /** + * Misskey本体ではなく、サマリープロキシを使用してプレビューを生成します。 + */ + "summaryProxyDescription": string; + /** + * プロキシには下記パラメータがクエリ文字列として連携されます。プロキシ側がこれらをサポートしない場合、設定値は無視されます。 + */ + "summaryProxyDescription2": string; + }; } declare const locales: { [lang: string]: Locale; diff --git a/locales/ja-JP.yml b/locales/ja-JP.yml index a64c83b10f..cada6d855f 100644 --- a/locales/ja-JP.yml +++ b/locales/ja-JP.yml @@ -1225,6 +1225,7 @@ enableHorizontalSwipe: "スワイプしてタブを切り替える" loading: "読み込み中" surrender: "やめる" gameRetry: "リトライ" +notUsePleaseLeaveBlank: "使用しない場合は空欄にしてください" _bubbleGame: howToPlay: "遊び方" @@ -2602,3 +2603,17 @@ _offlineScreen: title: "オフライン - サーバーに接続できません" header: "サーバーに接続できません" +_urlPreviewSetting: + title: "URLプレビューの設定" + enable: "URLプレビューを有効にする" + timeout: "プレビュー取得時のタイムアウト(ms)" + timeoutDescription: "プレビュー取得の所要時間がこの値を超えた場合、プレビューは生成されません。" + maximumContentLength: "Content-Lengthの最大値(byte)" + maximumContentLengthDescription: "Content-Lengthがこの値を超えた場合、プレビューは生成されません。" + requireContentLength: "Content-Lengthが取得できた場合のみプレビューを生成" + requireContentLengthDescription: "相手サーバがContent-Lengthを返さない場合、プレビューは生成されません。" + userAgent: "User-Agent" + userAgentDescription: "プレビュー取得時に使用されるUser-Agentを設定します。空欄の場合、デフォルトのUser-Agentが使用されます。" + summaryProxy: "プレビューを生成するプロキシのエンドポイント" + summaryProxyDescription: "Misskey本体ではなく、サマリープロキシを使用してプレビューを生成します。" + summaryProxyDescription2: "プロキシには下記パラメータがクエリ文字列として連携されます。プロキシ側がこれらをサポートしない場合、設定値は無視されます。" diff --git a/packages/backend/migration/1710512074000-url-preview-meta.js b/packages/backend/migration/1710512074000-url-preview-meta.js new file mode 100644 index 0000000000..8af521bbf4 --- /dev/null +++ b/packages/backend/migration/1710512074000-url-preview-meta.js @@ -0,0 +1,42 @@ +/* + * SPDX-FileCopyrightText: syuilo and misskey-project + * SPDX-License-Identifier: AGPL-3.0-only + */ + +export class UrlPreviewMeta1710512074000 { + name = 'UrlPreviewMeta1710512074000' + + async up(queryRunner) { + await queryRunner.query(` + alter table meta + rename column "summalyProxy" to "urlPreviewSummaryProxyUrl"; + alter table meta + add "urlPreviewEnabled" boolean default true not null; + alter table meta + add "urlPreviewTimeout" integer default 10000 not null; + alter table meta + add "urlPreviewMaximumContentLength" bigint default 10485760 not null; + alter table meta + add "urlPreviewRequireContentLength" boolean default false not null; + alter table meta + add "urlPreviewUserAgent" varchar(1024) default null; + `); + } + + async down(queryRunner) { + await queryRunner.query(` + alter table meta + rename column "urlPreviewSummaryProxyUrl" to "summalyProxy"; + alter table meta + drop column "urlPreviewEnabled"; + alter table meta + drop column "urlPreviewTimeout"; + alter table meta + drop column "urlPreviewMaximumContentLength"; + alter table meta + drop column "urlPreviewRequireContentLength"; + alter table meta + drop column "urlPreviewUserAgent"; + `); + } +} diff --git a/packages/backend/package.json b/packages/backend/package.json index eaad96d5f6..d64fcc3d2a 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -80,7 +80,7 @@ "@fastify/static": "6.12.0", "@fastify/view": "8.2.0", "@misskey-dev/sharp-read-bmp": "1.2.0", - "@misskey-dev/summaly": "5.0.3", + "@misskey-dev/summaly": "5.1.0", "@nestjs/common": "10.3.3", "@nestjs/core": "10.3.3", "@nestjs/testing": "10.3.3", diff --git a/packages/backend/src/core/entities/MetaEntityService.ts b/packages/backend/src/core/entities/MetaEntityService.ts index b50d76288f..9d054ab6a1 100644 --- a/packages/backend/src/core/entities/MetaEntityService.ts +++ b/packages/backend/src/core/entities/MetaEntityService.ts @@ -111,6 +111,7 @@ export class MetaEntityService { policies: { ...DEFAULT_POLICIES, ...instance.policies }, mediaProxy: this.config.mediaProxy, + enableUrlPreview: instance.urlPreviewEnabled, }; return packed; diff --git a/packages/backend/src/models/Meta.ts b/packages/backend/src/models/Meta.ts index 66f19ce197..04a34bbbb4 100644 --- a/packages/backend/src/models/Meta.ts +++ b/packages/backend/src/models/Meta.ts @@ -277,12 +277,6 @@ export class MiMeta { }) public enableSensitiveMediaDetectionForVideos: boolean; - @Column('varchar', { - length: 1024, - nullable: true, - }) - public summalyProxy: string | null; - @Column('boolean', { default: false, }) @@ -588,4 +582,36 @@ export class MiMeta { default: 0, }) public notesPerOneAd: number; + + @Column('boolean', { + default: true, + }) + public urlPreviewEnabled: boolean; + + @Column('integer', { + default: 10000, + }) + public urlPreviewTimeout: number; + + @Column('bigint', { + default: 1024 * 1024 * 10, + }) + public urlPreviewMaximumContentLength: number; + + @Column('boolean', { + default: true, + }) + public urlPreviewRequireContentLength: boolean; + + @Column('varchar', { + length: 1024, + nullable: true, + }) + public urlPreviewSummaryProxyUrl: string | null; + + @Column('varchar', { + length: 1024, + nullable: true, + }) + public urlPreviewUserAgent: string | null; } diff --git a/packages/backend/src/models/json-schema/meta.ts b/packages/backend/src/models/json-schema/meta.ts index 17789f3b46..473339a1ad 100644 --- a/packages/backend/src/models/json-schema/meta.ts +++ b/packages/backend/src/models/json-schema/meta.ts @@ -207,6 +207,10 @@ export const packedMetaLiteSchema = { type: 'string', optional: false, nullable: false, }, + enableUrlPreview: { + type: 'boolean', + optional: false, nullable: false, + }, backgroundImageUrl: { type: 'string', optional: false, nullable: true, diff --git a/packages/backend/src/server/api/endpoints/admin/meta.ts b/packages/backend/src/server/api/endpoints/admin/meta.ts index 88c5907bcc..f4ff573271 100644 --- a/packages/backend/src/server/api/endpoints/admin/meta.ts +++ b/packages/backend/src/server/api/endpoints/admin/meta.ts @@ -434,6 +434,8 @@ export const meta = { summalyProxy: { type: 'string', optional: false, nullable: true, + deprecated: true, + description: '[Deprecated] Use "urlPreviewSummaryProxyUrl" instead.', }, themeColor: { type: 'string', @@ -451,6 +453,30 @@ export const meta = { type: 'string', optional: false, nullable: false, }, + urlPreviewEnabled: { + type: 'boolean', + optional: false, nullable: false, + }, + urlPreviewTimeout: { + type: 'number', + optional: false, nullable: false, + }, + urlPreviewMaximumContentLength: { + type: 'number', + optional: false, nullable: false, + }, + urlPreviewRequireContentLength: { + type: 'boolean', + optional: false, nullable: false, + }, + urlPreviewUserAgent: { + type: 'string', + optional: false, nullable: true, + }, + urlPreviewSummaryProxyUrl: { + type: 'string', + optional: false, nullable: true, + }, }, }, } as const; @@ -533,7 +559,6 @@ export default class extends Endpoint { // eslint- setSensitiveFlagAutomatically: instance.setSensitiveFlagAutomatically, enableSensitiveMediaDetectionForVideos: instance.enableSensitiveMediaDetectionForVideos, proxyAccountId: instance.proxyAccountId, - summalyProxy: instance.summalyProxy, email: instance.email, smtpSecure: instance.smtpSecure, smtpHost: instance.smtpHost, @@ -577,6 +602,13 @@ export default class extends Endpoint { // eslint- perUserHomeTimelineCacheMax: instance.perUserHomeTimelineCacheMax, perUserListTimelineCacheMax: instance.perUserListTimelineCacheMax, notesPerOneAd: instance.notesPerOneAd, + summalyProxy: instance.urlPreviewSummaryProxyUrl, + urlPreviewEnabled: instance.urlPreviewEnabled, + urlPreviewTimeout: instance.urlPreviewTimeout, + urlPreviewMaximumContentLength: instance.urlPreviewMaximumContentLength, + urlPreviewRequireContentLength: instance.urlPreviewRequireContentLength, + urlPreviewUserAgent: instance.urlPreviewUserAgent, + urlPreviewSummaryProxyUrl: instance.urlPreviewSummaryProxyUrl, }; }); } diff --git a/packages/backend/src/server/api/endpoints/admin/update-meta.ts b/packages/backend/src/server/api/endpoints/admin/update-meta.ts index bffceef815..2f62d30ada 100644 --- a/packages/backend/src/server/api/endpoints/admin/update-meta.ts +++ b/packages/backend/src/server/api/endpoints/admin/update-meta.ts @@ -90,7 +90,6 @@ export const paramDef = { type: 'string', }, }, - summalyProxy: { type: 'string', nullable: true }, deeplAuthKey: { type: 'string', nullable: true }, deeplIsPro: { type: 'boolean' }, enableEmail: { type: 'boolean' }, @@ -150,6 +149,16 @@ export const paramDef = { type: 'string', }, }, + summalyProxy: { + type: 'string', nullable: true, + description: '[Deprecated] Use "urlPreviewSummaryProxyUrl" instead.', + }, + urlPreviewEnabled: { type: 'boolean' }, + urlPreviewTimeout: { type: 'integer' }, + urlPreviewMaximumContentLength: { type: 'integer' }, + urlPreviewRequireContentLength: { type: 'boolean' }, + urlPreviewUserAgent: { type: 'string', nullable: true }, + urlPreviewSummaryProxyUrl: { type: 'string', nullable: true }, }, required: [], } as const; @@ -353,10 +362,6 @@ export default class extends Endpoint { // eslint- set.langs = ps.langs.filter(Boolean); } - if (ps.summalyProxy !== undefined) { - set.summalyProxy = ps.summalyProxy; - } - if (ps.enableEmail !== undefined) { set.enableEmail = ps.enableEmail; } @@ -581,6 +586,32 @@ export default class extends Endpoint { // eslint- set.bannedEmailDomains = ps.bannedEmailDomains; } + if (ps.urlPreviewEnabled !== undefined) { + set.urlPreviewEnabled = ps.urlPreviewEnabled; + } + + if (ps.urlPreviewTimeout !== undefined) { + set.urlPreviewTimeout = ps.urlPreviewTimeout; + } + + if (ps.urlPreviewMaximumContentLength !== undefined) { + set.urlPreviewMaximumContentLength = ps.urlPreviewMaximumContentLength; + } + + if (ps.urlPreviewRequireContentLength !== undefined) { + set.urlPreviewRequireContentLength = ps.urlPreviewRequireContentLength; + } + + if (ps.urlPreviewUserAgent !== undefined) { + const value = (ps.urlPreviewUserAgent ?? '').trim(); + set.urlPreviewUserAgent = value === '' ? null : ps.urlPreviewUserAgent; + } + + if (ps.summalyProxy !== undefined || ps.urlPreviewSummaryProxyUrl !== undefined) { + const value = ((ps.urlPreviewSummaryProxyUrl ?? ps.summalyProxy) ?? '').trim(); + set.urlPreviewSummaryProxyUrl = value === '' ? null : value; + } + const before = await this.metaService.fetch(true); await this.metaService.update(set); diff --git a/packages/backend/src/server/web/UrlPreviewService.ts b/packages/backend/src/server/web/UrlPreviewService.ts index c6a96e94cb..8f8f08a305 100644 --- a/packages/backend/src/server/web/UrlPreviewService.ts +++ b/packages/backend/src/server/web/UrlPreviewService.ts @@ -5,6 +5,7 @@ import { Inject, Injectable } from '@nestjs/common'; import { summaly } from '@misskey-dev/summaly'; +import { SummalyResult } from '@misskey-dev/summaly/built/summary.js'; import { DI } from '@/di-symbols.js'; import type { Config } from '@/config.js'; import { MetaService } from '@/core/MetaService.js'; @@ -14,6 +15,7 @@ import { query } from '@/misc/prelude/url.js'; import { LoggerService } from '@/core/LoggerService.js'; import { bindThis } from '@/decorators.js'; import { ApiError } from '@/server/api/error.js'; +import { MiMeta } from '@/models/Meta.js'; import type { FastifyRequest, FastifyReply } from 'fastify'; @Injectable() @@ -62,24 +64,25 @@ export class UrlPreviewService { const meta = await this.metaService.fetch(); - this.logger.info(meta.summalyProxy + if (!meta.urlPreviewEnabled) { + reply.code(403); + return { + error: new ApiError({ + message: 'URL preview is disabled', + code: 'URL_PREVIEW_DISABLED', + id: '58b36e13-d2f5-0323-b0c6-76aa9dabefb8', + }), + }; + } + + this.logger.info(meta.urlPreviewSummaryProxyUrl ? `(Proxy) Getting preview of ${url}@${lang} ...` : `Getting preview of ${url}@${lang} ...`); + try { - const summary = meta.summalyProxy ? - await this.httpRequestService.getJson>(`${meta.summalyProxy}?${query({ - url: url, - lang: lang ?? 'ja-JP', - })}`) - : - await summaly(url, { - followRedirects: false, - lang: lang ?? 'ja-JP', - agent: this.config.proxy ? { - http: this.httpRequestService.httpAgent, - https: this.httpRequestService.httpsAgent, - } : undefined, - }); + const summary = meta.urlPreviewSummaryProxyUrl + ? await this.fetchSummaryFromProxy(url, meta, lang) + : await this.fetchSummary(url, meta, lang); this.logger.succ(`Got preview of ${url}: ${summary.title}`); @@ -100,6 +103,7 @@ export class UrlPreviewService { return summary; } catch (err) { this.logger.warn(`Failed to get preview of ${url}: ${err}`); + reply.code(422); reply.header('Cache-Control', 'max-age=86400, immutable'); return { @@ -111,4 +115,37 @@ export class UrlPreviewService { }; } } + + private fetchSummary(url: string, meta: MiMeta, lang?: string): Promise { + const agent = this.config.proxy + ? { + http: this.httpRequestService.httpAgent, + https: this.httpRequestService.httpsAgent, + } + : undefined; + + return summaly(url, { + followRedirects: false, + lang: lang ?? 'ja-JP', + agent: agent, + userAgent: meta.urlPreviewUserAgent ?? undefined, + operationTimeout: meta.urlPreviewTimeout, + contentLengthLimit: meta.urlPreviewMaximumContentLength, + contentLengthRequired: meta.urlPreviewRequireContentLength, + }); + } + + private fetchSummaryFromProxy(url: string, meta: MiMeta, lang?: string): Promise { + const proxy = meta.urlPreviewSummaryProxyUrl!; + const queryStr = query({ + url: url, + lang: lang ?? 'ja-JP', + userAgent: meta.urlPreviewUserAgent ?? undefined, + operationTimeout: meta.urlPreviewTimeout, + contentLengthLimit: meta.urlPreviewMaximumContentLength, + contentLengthRequired: meta.urlPreviewRequireContentLength, + }); + + return this.httpRequestService.getJson(`${proxy}?${queryStr}`); + } } diff --git a/packages/frontend/src/components/MkLink.vue b/packages/frontend/src/components/MkLink.vue index 3f7aba2fe4..ca875242b4 100644 --- a/packages/frontend/src/components/MkLink.vue +++ b/packages/frontend/src/components/MkLink.vue @@ -18,6 +18,7 @@ import { defineAsyncComponent, ref } from 'vue'; import { url as local } from '@/config.js'; import { useTooltip } from '@/scripts/use-tooltip.js'; import * as os from '@/os.js'; +import { isEnabledUrlPreview } from '@/instance.js'; const props = withDefaults(defineProps<{ url: string; @@ -31,13 +32,15 @@ const target = self ? null : '_blank'; const el = ref(); -useTooltip(el, (showing) => { - os.popup(defineAsyncComponent(() => import('@/components/MkUrlPreviewPopup.vue')), { - showing, - url: props.url, - source: el.value instanceof HTMLElement ? el.value : el.value?.$el, - }, {}, 'closed'); -}); +if (isEnabledUrlPreview.value) { + useTooltip(el, (showing) => { + os.popup(defineAsyncComponent(() => import('@/components/MkUrlPreviewPopup.vue')), { + showing, + url: props.url, + source: el.value instanceof HTMLElement ? el.value : el.value?.$el, + }, {}, 'closed'); + }); +} diff --git a/packages/misskey-js/src/autogen/types.ts b/packages/misskey-js/src/autogen/types.ts index 636bc62aaa..c6e36d80aa 100644 --- a/packages/misskey-js/src/autogen/types.ts +++ b/packages/misskey-js/src/autogen/types.ts @@ -4810,6 +4810,7 @@ export type components = { enableServiceWorker: boolean; translatorAvailable: boolean; mediaProxy: string; + enableUrlPreview: boolean; backgroundImageUrl: string | null; impressumUrl: string | null; logoImageUrl: string | null; @@ -4962,11 +4963,21 @@ export type operations = { objectStorageS3ForcePathStyle: boolean; privacyPolicyUrl: string | null; repositoryUrl: string | null; + /** + * @deprecated + * @description [Deprecated] Use "urlPreviewSummaryProxyUrl" instead. + */ summalyProxy: string | null; themeColor: string | null; tosUrl: string | null; uri: string; version: string; + urlPreviewEnabled: boolean; + urlPreviewTimeout: number; + urlPreviewMaximumContentLength: number; + urlPreviewRequireContentLength: boolean; + urlPreviewUserAgent: string | null; + urlPreviewSummaryProxyUrl: string | null; }; }; }; @@ -8862,7 +8873,6 @@ export type operations = { maintainerName?: string | null; maintainerEmail?: string | null; langs?: string[]; - summalyProxy?: string | null; deeplAuthKey?: string | null; deeplIsPro?: boolean; enableEmail?: boolean; @@ -8916,6 +8926,14 @@ export type operations = { perUserListTimelineCacheMax?: number; notesPerOneAd?: number; silencedHosts?: string[] | null; + /** @description [Deprecated] Use "urlPreviewSummaryProxyUrl" instead. */ + summalyProxy?: string | null; + urlPreviewEnabled?: boolean; + urlPreviewTimeout?: number; + urlPreviewMaximumContentLength?: number; + urlPreviewRequireContentLength?: boolean; + urlPreviewUserAgent?: string | null; + urlPreviewSummaryProxyUrl?: string | null; }; }; }; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4c1e228a95..383b31b1f3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -117,8 +117,8 @@ importers: specifier: 1.2.0 version: 1.2.0 '@misskey-dev/summaly': - specifier: 5.0.3 - version: 5.0.3 + specifier: 5.1.0 + version: 5.1.0 '@nestjs/common': specifier: 10.3.3 version: 10.3.3(reflect-metadata@0.2.1)(rxjs@7.8.1) @@ -4772,6 +4772,20 @@ packages: jschardet: 3.0.0 private-ip: 2.3.3 trace-redirect: 1.0.6 + dev: true + + /@misskey-dev/summaly@5.1.0: + resolution: {integrity: sha512-WAUrgX3/z4h4aI8Y/WVwmJcJ6Fa1Zf2LJCSS651t9MHoWVGABLsQ2KCXRGmlpk4i+cMDNIwweObUroosE7j8rg==} + dependencies: + cheerio: 1.0.0-rc.12 + escape-regexp: 0.0.1 + got: 12.6.1 + html-entities: 2.3.2 + iconv-lite: 0.6.3 + jschardet: 3.0.0 + private-ip: 2.3.3 + trace-redirect: 1.0.6 + dev: false /@mole-inc/bin-wrapper@8.0.1: resolution: {integrity: sha512-sTGoeZnjI8N4KS+sW2AN95gDBErhAguvkw/tWdCjeM8bvxpz5lqrnd0vOJABA1A+Ic3zED7PYoLP/RANLgVotA==} From f7ec503b9ceb34d61a0dbd658858915eb7399c5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=81=BE=E3=81=A3=E3=81=A1=E3=82=83=E3=81=A8=E3=83=BC?= =?UTF-8?q?=E3=81=AB=E3=82=85?= <17376330+u1-liquid@users.noreply.github.com> Date: Fri, 22 Mar 2024 08:52:46 +0900 Subject: [PATCH 3/3] =?UTF-8?q?fix(MkUrlPreview):=20=E5=A4=96=E9=83=A8?= =?UTF-8?q?=E3=82=B5=E3=82=A4=E3=83=88=E3=81=B8=E3=81=AE=E3=83=AA=E3=83=B3?= =?UTF-8?q?=E3=82=AF=E3=81=AF=E7=A7=BB=E5=8B=95=E3=81=AE=E5=89=8D=E3=81=AB?= =?UTF-8?q?=E8=AD=A6=E5=91=8A=E3=82=92=E8=A1=A8=E7=A4=BA=E3=81=99=E3=82=8B?= =?UTF-8?q?=E3=82=88=E3=81=86=E3=81=AB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/frontend/src/components/MkUrlPreview.vue | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/packages/frontend/src/components/MkUrlPreview.vue b/packages/frontend/src/components/MkUrlPreview.vue index 3e5aebc608..18c47a18ab 100644 --- a/packages/frontend/src/components/MkUrlPreview.vue +++ b/packages/frontend/src/components/MkUrlPreview.vue @@ -44,7 +44,15 @@ SPDX-License-Identifier: AGPL-3.0-only
- +
@@ -92,6 +100,7 @@ import { deviceKind } from '@/scripts/device-kind.js'; import MkButton from '@/components/MkButton.vue'; import { versatileLang } from '@/scripts/intl-const.js'; import { defaultStore } from '@/store.js'; +import { warningExternalWebsite } from '@/scripts/warning-external-website.js'; type SummalyResult = Awaited>;