From a481f00cab475846f6f1e8c91223ebfe6528b972 Mon Sep 17 00:00:00 2001 From: MomentQYC Date: Fri, 30 Aug 2024 14:26:28 +0800 Subject: [PATCH] WIP --- locales/index.d.ts | 16 +++ locales/ja-JP.yml | 4 + .../backend/migration/1724683952000-tts.js | 16 +++ packages/backend/src/core/RoleService.ts | 3 + .../src/core/entities/MetaEntityService.ts | 1 + packages/backend/src/models/Meta.ts | 6 + .../backend/src/models/json-schema/meta.ts | 4 + .../backend/src/models/json-schema/role.ts | 4 + .../backend/src/server/api/EndpointsModule.ts | 4 + packages/backend/src/server/api/endpoints.ts | 2 + .../src/server/api/endpoints/admin/meta.ts | 5 + .../server/api/endpoints/admin/update-meta.ts | 9 ++ .../src/server/api/endpoints/notes/tts.ts | 109 ++++++++++++++++++ packages/frontend/src/components/MkNote.vue | 33 +++++- .../src/components/MkNoteDetailed.vue | 45 +++++++- packages/frontend/src/const.ts | 1 + .../src/pages/admin/external-services.vue | 13 +++ .../frontend/src/pages/admin/roles.editor.vue | 20 ++++ packages/frontend/src/pages/admin/roles.vue | 8 ++ .../frontend/src/scripts/get-note-menu.ts | 21 ++++ packages/misskey-js/etc/misskey-js.api.md | 8 ++ packages/misskey-js/src/autogen/endpoint.ts | 4 + packages/misskey-js/src/autogen/entities.ts | 2 + packages/misskey-js/src/autogen/types.ts | 72 ++++++++++++ 24 files changed, 406 insertions(+), 4 deletions(-) create mode 100644 packages/backend/migration/1724683952000-tts.js create mode 100644 packages/backend/src/server/api/endpoints/notes/tts.ts diff --git a/locales/index.d.ts b/locales/index.d.ts index 9fd3441ab1..9972661bf8 100644 --- a/locales/index.d.ts +++ b/locales/index.d.ts @@ -3544,10 +3544,18 @@ export interface Locale extends ILocale { * 翻訳 */ "translate": string; + /** + * 変換 + */ + "converting": string; /** * {x}から翻訳 */ "translatedFrom": ParameterizedString<"x">; + /** + * {x}から変換 + */ + "convertedFrom": ParameterizedString<"x">; /** * アカウントの削除が進行中です */ @@ -6738,6 +6746,10 @@ export interface Locale extends ILocale { * 翻訳機能の利用 */ "canUseTranslator": string; + /** + * TTS機能の利用 + */ + "canUseTTS": string; /** * アイコンデコレーションの最大取付個数 */ @@ -7155,6 +7167,10 @@ export interface Locale extends ILocale { * Misskeyを翻訳 */ "translation": string; + /** + * Misskeyを変換 + */ + "convert": string; /** * Misskeyに寄付 */ diff --git a/locales/ja-JP.yml b/locales/ja-JP.yml index 587b67d987..9a75b05a76 100644 --- a/locales/ja-JP.yml +++ b/locales/ja-JP.yml @@ -882,7 +882,9 @@ learnMore: "詳しく" misskeyUpdated: "Misskeyが更新されました！" whatIsNew: "更新情報を見る" translate: "翻訳" +converting: "変換" translatedFrom: "{x}から翻訳" +convertedFrom: "{x}から変換" accountDeletionInProgress: "アカウントの削除が進行中です" usernameInfo: "サーバー上であなたのアカウントを一意に識別するための名前。アルファベット(a~z, A~Z)、数字(0~9)、およびアンダーバー(_)が使用できます。ユーザー名は後から変更することは出来ません。" aiChanMode: "藍モード" @@ -1741,6 +1743,7 @@ _role: canHideAds: "広告の非表示" canSearchNotes: "ノート検索の利用" canUseTranslator: "翻訳機能の利用" + canUseTTS: "TTS機能の利用" avatarDecorationLimit: "アイコンデコレーションの最大取付個数" _condition: roleAssignedTo: "マニュアルロールにアサイン済み" @@ -1866,6 +1869,7 @@ _aboutMisskey: original: "オリジナル" thisIsModifiedVersion: "{name}はオリジナルのMisskeyを改変したバージョンを使用しています。" translation: "Misskeyを翻訳" + convert: "Misskeyを変換" donate: "Misskeyに寄付" morePatrons: "他にも多くの方が支援してくれています。ありがとうございます🥰" patrons: "支援者" diff --git a/packages/backend/migration/1724683952000-tts.js b/packages/backend/migration/1724683952000-tts.js new file mode 100644 index 0000000000..2b669804ae --- /dev/null +++ b/packages/backend/migration/1724683952000-tts.js @@ -0,0 +1,16 @@ +/* + * SPDX-FileCopyrightText: syuilo and misskey-project + * SPDX-License-Identifier: AGPL-3.0-only + */ + +export class TTSIntegration1724683952000 { + constructor() { + this.name = 'TTSIntegration1724683952000'; + } + async up(queryRunner) { + await queryRunner.query(`ALTER TABLE "meta" ADD "hfAuthKey" character varying(128)`); + } + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "hfAuthKey"`); + } +} diff --git a/packages/backend/src/core/RoleService.ts b/packages/backend/src/core/RoleService.ts index 0210012a03..6025fb49ba 100644 --- a/packages/backend/src/core/RoleService.ts +++ b/packages/backend/src/core/RoleService.ts @@ -44,6 +44,7 @@ export type RolePolicies = { canManageAvatarDecorations: boolean; canSearchNotes: boolean; canUseTranslator: boolean; + canUseTTS: boolean; canHideAds: boolean; driveCapacityMb: number; alwaysMarkNsfw: boolean; @@ -73,6 +74,7 @@ export const DEFAULT_POLICIES: RolePolicies = { canManageAvatarDecorations: false, canSearchNotes: false, canUseTranslator: true, + canUseTTS: true, canHideAds: false, driveCapacityMb: 100, alwaysMarkNsfw: false, @@ -373,6 +375,7 @@ export class RoleService implements OnApplicationShutdown, OnModuleInit { canManageAvatarDecorations: calc('canManageAvatarDecorations', vs => vs.some(v => v === true)), canSearchNotes: calc('canSearchNotes', vs => vs.some(v => v === true)), canUseTranslator: calc('canUseTranslator', vs => vs.some(v => v === true)), + canUseTTS: calc('canUseTTS', vs => vs.some(v => v === true)), canHideAds: calc('canHideAds', vs => vs.some(v => v === true)), driveCapacityMb: calc('driveCapacityMb', vs => Math.max(...vs)), alwaysMarkNsfw: calc('alwaysMarkNsfw', vs => vs.some(v => v === true)), diff --git a/packages/backend/src/core/entities/MetaEntityService.ts b/packages/backend/src/core/entities/MetaEntityService.ts index f4b1e302d0..514149d70e 100644 --- a/packages/backend/src/core/entities/MetaEntityService.ts +++ b/packages/backend/src/core/entities/MetaEntityService.ts @@ -121,6 +121,7 @@ export class MetaEntityService { enableServiceWorker: instance.enableServiceWorker, translatorAvailable: instance.deeplAuthKey != null, + ttsAvailable: instance.hfAuthKey != null, serverRules: instance.serverRules, diff --git a/packages/backend/src/models/Meta.ts b/packages/backend/src/models/Meta.ts index 70d41801b5..e6dc458df8 100644 --- a/packages/backend/src/models/Meta.ts +++ b/packages/backend/src/models/Meta.ts @@ -349,6 +349,12 @@ export class MiMeta { }) public deeplIsPro: boolean; + @Column('varchar', { + length: 1024, + nullable: true, + }) + public hfAuthKey: string | null; + @Column('varchar', { length: 1024, nullable: true, diff --git a/packages/backend/src/models/json-schema/meta.ts b/packages/backend/src/models/json-schema/meta.ts index 99feeaa7d7..df0853b5c7 100644 --- a/packages/backend/src/models/json-schema/meta.ts +++ b/packages/backend/src/models/json-schema/meta.ts @@ -203,6 +203,10 @@ export const packedMetaLiteSchema = { type: 'boolean', optional: false, nullable: false, }, + ttsAvailable: { + type: 'boolean', + optional: false, nullable: false, + }, mediaProxy: { type: 'string', optional: false, nullable: false, diff --git a/packages/backend/src/models/json-schema/role.ts b/packages/backend/src/models/json-schema/role.ts index 7366f05356..0616cf4975 100644 --- a/packages/backend/src/models/json-schema/role.ts +++ b/packages/backend/src/models/json-schema/role.ts @@ -216,6 +216,10 @@ export const packedRolePoliciesSchema = { type: 'boolean', optional: false, nullable: false, }, + canUseTTS: { + type: 'boolean', + optional: false, nullable: false, + }, canHideAds: { type: 'boolean', optional: false, nullable: false, diff --git a/packages/backend/src/server/api/EndpointsModule.ts b/packages/backend/src/server/api/EndpointsModule.ts index 41576bedaa..59b9d0b0e4 100644 --- a/packages/backend/src/server/api/EndpointsModule.ts +++ b/packages/backend/src/server/api/EndpointsModule.ts @@ -301,6 +301,7 @@ import * as ep___notes_threadMuting_create from './endpoints/notes/thread-muting import * as ep___notes_threadMuting_delete from './endpoints/notes/thread-muting/delete.js'; import * as ep___notes_timeline from './endpoints/notes/timeline.js'; import * as ep___notes_translate from './endpoints/notes/translate.js'; +import * as ep___notes_tts from './endpoints/notes/tts.js'; import * as ep___notes_unrenote from './endpoints/notes/unrenote.js'; import * as ep___notes_userListTimeline from './endpoints/notes/user-list-timeline.js'; import * as ep___notifications_create from './endpoints/notifications/create.js'; @@ -684,6 +685,7 @@ const $notes_threadMuting_create: Provider = { provide: 'ep:notes/thread-muting/ const $notes_threadMuting_delete: Provider = { provide: 'ep:notes/thread-muting/delete', useClass: ep___notes_threadMuting_delete.default }; const $notes_timeline: Provider = { provide: 'ep:notes/timeline', useClass: ep___notes_timeline.default }; const $notes_translate: Provider = { provide: 'ep:notes/translate', useClass: ep___notes_translate.default }; +const $notes_tts: Provider = { provide: 'ep:notes/tts', useClass: ep___notes_tts.default }; const $notes_unrenote: Provider = { provide: 'ep:notes/unrenote', useClass: ep___notes_unrenote.default }; const $notes_userListTimeline: Provider = { provide: 'ep:notes/user-list-timeline', useClass: ep___notes_userListTimeline.default }; const $notifications_create: Provider = { provide: 'ep:notifications/create', useClass: ep___notifications_create.default }; @@ -1071,6 +1073,7 @@ const $reversi_verify: Provider = { provide: 'ep:reversi/verify', useClass: ep__ $notes_threadMuting_delete, $notes_timeline, $notes_translate, + $notes_tts, $notes_unrenote, $notes_userListTimeline, $notifications_create, @@ -1452,6 +1455,7 @@ const $reversi_verify: Provider = { provide: 'ep:reversi/verify', useClass: ep__ $notes_threadMuting_delete, $notes_timeline, $notes_translate, + $notes_tts, $notes_unrenote, $notes_userListTimeline, $notifications_create, diff --git a/packages/backend/src/server/api/endpoints.ts b/packages/backend/src/server/api/endpoints.ts index 3dfb7fdad4..b6ace034ae 100644 --- a/packages/backend/src/server/api/endpoints.ts +++ b/packages/backend/src/server/api/endpoints.ts @@ -307,6 +307,7 @@ import * as ep___notes_threadMuting_create from './endpoints/notes/thread-muting import * as ep___notes_threadMuting_delete from './endpoints/notes/thread-muting/delete.js'; import * as ep___notes_timeline from './endpoints/notes/timeline.js'; import * as ep___notes_translate from './endpoints/notes/translate.js'; +import * as ep___notes_tts from './endpoints/notes/tts.js'; import * as ep___notes_unrenote from './endpoints/notes/unrenote.js'; import * as ep___notes_userListTimeline from './endpoints/notes/user-list-timeline.js'; import * as ep___notifications_create from './endpoints/notifications/create.js'; @@ -688,6 +689,7 @@ const eps = [ ['notes/thread-muting/delete', ep___notes_threadMuting_delete], ['notes/timeline', ep___notes_timeline], ['notes/translate', ep___notes_translate], + ['notes/tts', ep___notes_tts], ['notes/unrenote', ep___notes_unrenote], ['notes/user-list-timeline', ep___notes_userListTimeline], ['notifications/create', ep___notifications_create], diff --git a/packages/backend/src/server/api/endpoints/admin/meta.ts b/packages/backend/src/server/api/endpoints/admin/meta.ts index 2e7f73da73..85342e5353 100644 --- a/packages/backend/src/server/api/endpoints/admin/meta.ts +++ b/packages/backend/src/server/api/endpoints/admin/meta.ts @@ -118,6 +118,10 @@ export const meta = { type: 'boolean', optional: false, nullable: false, }, + ttsAvailable: { + type: 'boolean', + optional: false, nullable: false, + }, silencedHosts: { type: 'array', optional: true, @@ -556,6 +560,7 @@ export default class extends Endpoint { // eslint- enableEmail: instance.enableEmail, enableServiceWorker: instance.enableServiceWorker, translatorAvailable: instance.deeplAuthKey != null, + ttsAvailable: instance.hfAuthKey != null, cacheRemoteFiles: instance.cacheRemoteFiles, cacheRemoteSensitiveFiles: instance.cacheRemoteSensitiveFiles, pinnedUsers: instance.pinnedUsers, diff --git a/packages/backend/src/server/api/endpoints/admin/update-meta.ts b/packages/backend/src/server/api/endpoints/admin/update-meta.ts index 5efdc9d8c4..c36a63e141 100644 --- a/packages/backend/src/server/api/endpoints/admin/update-meta.ts +++ b/packages/backend/src/server/api/endpoints/admin/update-meta.ts @@ -92,6 +92,7 @@ export const paramDef = { }, deeplAuthKey: { type: 'string', nullable: true }, deeplIsPro: { type: 'boolean' }, + hfAuthKey: { type: 'string', nullable: true }, enableEmail: { type: 'boolean' }, email: { type: 'string', nullable: true }, smtpSecure: { type: 'boolean' }, @@ -506,6 +507,14 @@ export default class extends Endpoint { // eslint- set.deeplIsPro = ps.deeplIsPro; } + if (ps.hfAuthKey !== undefined) { + if (ps.hfAuthKey === '') { + set.hfAuthKey = null; + } else { + set.hfAuthKey = ps.hfAuthKey; + } + } + if (ps.enableIpLogging !== undefined) { set.enableIpLogging = ps.enableIpLogging; } diff --git a/packages/backend/src/server/api/endpoints/notes/tts.ts b/packages/backend/src/server/api/endpoints/notes/tts.ts new file mode 100644 index 0000000000..86b3ad7e99 --- /dev/null +++ b/packages/backend/src/server/api/endpoints/notes/tts.ts @@ -0,0 +1,109 @@ +/* + * SPDX-FileCopyrightText: syuilo and misskey-project + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { Injectable } from '@nestjs/common'; +import { Endpoint } from '@/server/api/endpoint-base.js'; +import { NoteEntityService } from '@/core/entities/NoteEntityService.js'; +import { MetaService } from '@/core/MetaService.js'; +import { HttpRequestService } from '@/core/HttpRequestService.js'; +import { GetterService } from '@/server/api/GetterService.js'; +import { RoleService } from '@/core/RoleService.js'; +import { ApiError } from '../../error.js'; + +export const meta = { + tags: ['notes'], + + requireCredential: true, + kind: 'read:account', + + res: { + type: 'string', + optional: true, nullable: false, + contentMediaType: 'audio/flac', + }, + + errors: { + unavailable: { + message: 'Convert of notes unavailable.', + code: 'UNAVAILABLE', + id: '97a0826c-6393-11ef-a650-67972d710975', + }, + noSuchNote: { + message: 'No such note.', + code: 'NO_SUCH_NOTE', + id: 'bea9b03f-36e0-49c5-a4db-627a029f8971', + }, + cannotConvertInvisibleNote: { + message: 'Cannot convert invisible note.', + code: 'CANNOT_CONVERT_INVISIBLE_NOTE', + id: 'f57caae0-6394-11ef-8e2a-d706932c1030', + }, + }, +} as const; + +export const paramDef = { + type: 'object', + properties: { + noteId: { type: 'string', format: 'misskey:id' }, + }, + required: ['noteId'], +} as const; + +@Injectable() +export default class extends Endpoint { // eslint-disable-line import/no-default-export + constructor( + private noteEntityService: NoteEntityService, + private getterService: GetterService, + private metaService: MetaService, + private httpRequestService: HttpRequestService, + private roleService: RoleService, + ) { + super(meta, paramDef, async (ps, me) => { + const policies = await this.roleService.getUserPolicies(me.id); + if (!policies.canUseTTS) { + throw new ApiError(meta.errors.unavailable); + } + + const note = await this.getterService.getNote(ps.noteId).catch(err => { + if (err.id === '9725d0ce-ba28-4dde-95a7-2cbb2c15de24') throw new ApiError(meta.errors.noSuchNote); + throw err; + }); + + if (!(await this.noteEntityService.isVisibleForMe(note, me.id))) { + throw new ApiError(meta.errors.cannotConvertInvisibleNote); + } + + if (note.text == null) { + return; + } + + const instance = await this.metaService.fetch(); + + if (instance.hfAuthKey == null) { + throw new ApiError(meta.errors.unavailable); + } + + const endpoint = 'https://api-inference.huggingface.co/models/suno/bark'; + + const res = await this.httpRequestService.send(endpoint, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': 'Bearer ' + instance.hfAuthKey, + }, + body: JSON.stringify({ + inputs: note.text, + }), + timeout: 60000, + }); + + if (res.headers.get('content-type') === 'audio/flac') { + return res.body; + } else { + throw new ApiError(meta.errors.unavailable); + } + }); + } +} diff --git a/packages/frontend/src/components/MkNote.vue b/packages/frontend/src/components/MkNote.vue index 4caafe54bf..ffb4c8d0a5 100644 --- a/packages/frontend/src/components/MkNote.vue +++ b/packages/frontend/src/components/MkNote.vue @@ -77,6 +77,14 @@ SPDX-License-Identifier: AGPL-3.0-only +
+ +
+ + {{ 'From ' + appearNote.id }} + +
+
@@ -160,12 +168,13 @@ SPDX-License-Identifier: AGPL-3.0-only