diff --git a/.config/example.yml b/.config/example.yml index 93dea1f254..cabf167fba 100644 --- a/.config/example.yml +++ b/.config/example.yml @@ -122,10 +122,12 @@ id: 'aid' # Proxy for HTTP/HTTPS #proxy: http://127.0.0.1:3128 -#proxyBypassHosts: [ -# 'example.com', -# '192.0.2.8' -#] +proxyBypassHosts: + - api.deepl.com + - api-free.deepl.com + - www.recaptcha.net + - hcaptcha.com + - challenges.cloudflare.com # Proxy for SMTP/SMTPS #proxySmtp: http://127.0.0.1:3128 # use HTTP/1.1 CONNECT diff --git a/CHANGELOG.md b/CHANGELOG.md index 918bb225a1..0a5e566263 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,9 @@ You should also include the user name that made the change. - You may have to `yarn run clean-all`, `sudo corepack enable` and `yarn set version berry` before running `yarn install` if you're still on yarn classic - 新たに動的なPagesを作ることはできなくなりました - 代わりに今後AiScriptを用いてより柔軟に動的なコンテンツを作成できるMisskey Play機能の実装を予定しています。 +- AiScriptが0.12.0にアップデートされました + - 0.12.0の変更点についてはこちら https://github.com/syuilo/aiscript/blob/master/CHANGELOG.md#0120 + - 0.12.0未満のプラグインは読み込むことはできません - iOS15以下のデバイスはサポートされなくなりました - API: カスタム絵文字エンティティに`url`プロパティが含まれなくなりました - 絵文字画像を表示するには、`/emoji/.webp`にリクエストすると画像が返ります。 @@ -27,6 +30,7 @@ You should also include the user name that made the change. - remote: `https://p1.a9z.dev/emoji/syuilo_birth_present@mk.f72u.net.webp` - API: `user`および`note`エンティティに`emojis`プロパティが含まれなくなりました - API: `user`エンティティに`avatarColor`および`bannerColor`プロパティが含まれなくなりました +- API: `instance`エンティティに`latestStatus`、`lastCommunicatedAt`、`latestRequestSentAt`プロパティが含まれなくなりました ### Improvements - Push notification of Antenna note @tamaina @@ -41,6 +45,7 @@ You should also include the user name that made the change. - Server: delete outdated notifications regularly to improve db performance @syuilo - Server: delete outdated hard-mutes regularly to improve db performance @syuilo - Server: delete outdated notes of antenna regularly to improve db performance @syuilo +- Server: improve activitypub deliver performance @syuilo - Client: use tabler-icons instead of fontawesome to better design @syuilo - Client: Add new gabber kick sounds (thanks for noizenecio) - Client: Add link to user RSS feed in profile menu @ssmucny @@ -74,6 +79,7 @@ You should also include the user name that made the change. - Client: Webhookの編集画面で、内容を保存することができない問題を修正 @m-hayabusa - Client: update emoji picker immediately on all input @saschanaz - Client: チャートのツールチップが画面に残ることがあるのを修正 @syuilo +- Client: fix wrong link in tutorial @syuilo ## 12.119.1 (2022/12/03) ### Bugfixes diff --git a/locales/de-DE.yml b/locales/de-DE.yml index 3ecfaa0563..dfac4fb791 100644 --- a/locales/de-DE.yml +++ b/locales/de-DE.yml @@ -1382,6 +1382,7 @@ _profile: changeBanner: "Banner ändern" _exportOrImport: allNotes: "Alle Notizen" + favoritedNotes: "Als Favorit markierte Notizen" followingList: "Gefolgte Benutzer" muteList: "Stummschaltungen" blockingList: "Blockierungen" diff --git a/locales/en-US.yml b/locales/en-US.yml index ecdd8492ee..414bc1df51 100644 --- a/locales/en-US.yml +++ b/locales/en-US.yml @@ -1382,6 +1382,7 @@ _profile: changeBanner: "Change banner" _exportOrImport: allNotes: "All notes" + favoritedNotes: "Favorite notes" followingList: "Followed users" muteList: "Muted users" blockingList: "Blocked users" diff --git a/locales/ja-JP.yml b/locales/ja-JP.yml index a07fb5ff91..d6a5518196 100644 --- a/locales/ja-JP.yml +++ b/locales/ja-JP.yml @@ -167,7 +167,6 @@ annotation: "注釈" federation: "連合" instances: "インスタンス" registeredAt: "初観測" -latestRequestSentAt: "直近のリクエスト送信" latestRequestReceivedAt: "直近のリクエスト受信" latestStatus: "直近のステータス" storageUsage: "ストレージ使用量" @@ -916,6 +915,7 @@ caption: "キャプション" loggedInAsBot: "Botアカウントでログイン中" tools: "ツール" cannotLoad: "読み込めません" +numberOfProfileView: "プロフィール表示回数" _sensitiveMediaDetection: description: "機械学習を使って自動でセンシティブなメディアを検出し、モデレーションに役立てることができます。サーバーの負荷が少し増えます。" diff --git a/locales/ko-KR.yml b/locales/ko-KR.yml index 0b07e76444..fa73d49a34 100644 --- a/locales/ko-KR.yml +++ b/locales/ko-KR.yml @@ -915,6 +915,7 @@ windowRestore: "복구" caption: "캡션" loggedInAsBot: "봇 계정으로 로그인중" tools: "도구" +cannotLoad: "불러오지 못했습니다" _sensitiveMediaDetection: description: "기계학습을 통해 자동으로 민감한 미디어를 탐지하여, 모더레이션에 참고할 수 있도록 합니다. 서버의 부하를 약간 증가시킵니다." sensitivity: "탐지 민감도" diff --git a/locales/th-TH.yml b/locales/th-TH.yml index d2b408dc4c..9dfcf0d2c4 100644 --- a/locales/th-TH.yml +++ b/locales/th-TH.yml @@ -1382,6 +1382,7 @@ _profile: changeBanner: "เปลี่ยนแบนเนอร์" _exportOrImport: allNotes: "โน้ตทั้งหมด" + favoritedNotes: "บันทึกที่ชื่นชอบ" followingList: "กำลังติดตาม" muteList: "ปิดเสียง" blockingList: "บล็อค" diff --git a/locales/zh-CN.yml b/locales/zh-CN.yml index e042fc2a20..80d367a284 100644 --- a/locales/zh-CN.yml +++ b/locales/zh-CN.yml @@ -915,6 +915,7 @@ windowRestore: "还原" caption: "标题" loggedInAsBot: "已登录的Bot" tools: "工具" +cannotLoad: "无法加载" _sensitiveMediaDetection: description: "可以使用机器学习技术自动检测敏感媒体，以便进行审核。服务器负载将略微增加。" sensitivity: "检测敏感度" diff --git a/locales/zh-TW.yml b/locales/zh-TW.yml index d78ecd2025..a97593e59c 100644 --- a/locales/zh-TW.yml +++ b/locales/zh-TW.yml @@ -915,6 +915,8 @@ windowRestore: "復原" caption: "標題" loggedInAsBot: "以機器人帳號登入中" tools: "工具" +cannotLoad: "無法載入" +numberOfProfileView: "個人檔案檢視次數" _sensitiveMediaDetection: description: "您可以使用機器學習自動檢測敏感媒體並將其用於審核。 伺服器的負荷會稍微增加。" sensitivity: "檢測敏感度" @@ -1381,6 +1383,7 @@ _profile: changeBanner: "變更橫幅圖像" _exportOrImport: allNotes: "所有貼文" + favoritedNotes: "「我的最愛」貼文" followingList: "追隨中" muteList: "靜音" blockingList: "封鎖" diff --git a/package.json b/package.json index 3b0319aada..1e3bf82a7b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "misskey", - "version": "13.0.0-beta.14", + "version": "13.0.0-beta.20", "codename": "indigo", "repository": { "type": "git", @@ -57,7 +57,7 @@ "@typescript-eslint/parser": "5.47.1", "cross-env": "7.0.3", "cypress": "12.2.0", - "eslint": "^8.30.0", + "eslint": "^8.31.0", "start-server-and-test": "1.15.2", "typescript": "4.9.4" } diff --git a/packages/backend/migration/1672703171386-remove-latestRequestSentAt.js b/packages/backend/migration/1672703171386-remove-latestRequestSentAt.js new file mode 100644 index 0000000000..c9b28dd7e1 --- /dev/null +++ b/packages/backend/migration/1672703171386-remove-latestRequestSentAt.js @@ -0,0 +1,11 @@ +export class removeLatestRequestSentAt1672703171386 { + name = 'removeLatestRequestSentAt1672703171386' + + async up(queryRunner) { + await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "latestRequestSentAt"`); + } + + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "instance" ADD "latestRequestSentAt" TIMESTAMP WITH TIME ZONE`); + } +} diff --git a/packages/backend/migration/1672704017999-remove-lastCommunicatedAt.js b/packages/backend/migration/1672704017999-remove-lastCommunicatedAt.js new file mode 100644 index 0000000000..38a6769851 --- /dev/null +++ b/packages/backend/migration/1672704017999-remove-lastCommunicatedAt.js @@ -0,0 +1,11 @@ +export class removeLastCommunicatedAt1672704017999 { + name = 'removeLastCommunicatedAt1672704017999' + + async up(queryRunner) { + await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "lastCommunicatedAt"`); + } + + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "instance" ADD "lastCommunicatedAt" TIMESTAMP WITH TIME ZONE NOT NULL`); + } +} diff --git a/packages/backend/migration/1672704136584-remove-latestStatus.js b/packages/backend/migration/1672704136584-remove-latestStatus.js new file mode 100644 index 0000000000..937c2fe8fd --- /dev/null +++ b/packages/backend/migration/1672704136584-remove-latestStatus.js @@ -0,0 +1,11 @@ +export class removeLatestStatus1672704136584 { + name = 'removeLatestStatus1672704136584' + + async up(queryRunner) { + await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "latestStatus"`); + } + + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "instance" ADD "latestStatus" integer`); + } +} diff --git a/packages/backend/package.json b/packages/backend/package.json index b206815641..d139c1db86 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -21,9 +21,9 @@ "@tensorflow/tfjs-node": "4.1.0" }, "dependencies": { - "@bull-board/api": "^4.9.0", - "@bull-board/fastify": "^4.9.0", - "@bull-board/ui": "^4.9.0", + "@bull-board/api": "^4.10.0", + "@bull-board/fastify": "^4.10.0", + "@bull-board/ui": "^4.10.0", "@discordapp/twemoji": "14.0.2", "@fastify/accepts": "4.1.0", "@fastify/cookie": "^8.3.0", @@ -37,12 +37,11 @@ "@nestjs/testing": "9.2.1", "@peertube/http-signature": "1.7.0", "@sinonjs/fake-timers": "10.0.2", - "@syuilo/aiscript": "0.11.1", "accepts": "^1.3.8", "ajv": "8.11.2", "archiver": "5.3.1", "autwh": "0.1.0", - "aws-sdk": "2.1272.0", + "aws-sdk": "2.1286.0", "bcryptjs": "2.4.3", "blurhash": "2.0.4", "bull": "4.10.2", @@ -57,7 +56,7 @@ "date-fns": "2.29.3", "deep-email-validator": "0.1.21", "escape-regexp": "0.0.1", - "fastify": "4.10.2", + "fastify": "4.11.0", "feed": "4.2.2", "file-type": "18.0.0", "fluent-ffmpeg": "2.1.2", @@ -69,7 +68,7 @@ "is-svg": "4.3.2", "js-yaml": "4.1.0", "jsdom": "20.0.3", - "json5": "2.2.2", + "json5": "2.2.3", "json5-loader": "4.0.1", "jsonld": "8.1.0", "jsrsasign": "10.6.1", @@ -78,7 +77,6 @@ "misskey-js": "0.0.14", "ms": "3.0.0-canary.1", "nested-property": "4.0.0", - "node-fetch": "3.3.0", "nodemailer": "6.8.0", "nsfwjs": "2.4.2", "oauth": "^0.10.0", @@ -115,10 +113,11 @@ "tinycolor2": "1.5.1", "tmp": "0.2.1", "tsc-alias": "1.8.2", - "tsconfig-paths": "4.1.1", + "tsconfig-paths": "4.1.2", "twemoji-parser": "14.0.0", "typeorm": "0.3.11", "ulid": "2.3.0", + "undici": "^5.14.0", "unzipper": "0.10.11", "uuid": "9.0.0", "vary": "1.1.2", @@ -141,7 +140,7 @@ "@types/escape-regexp": "0.0.1", "@types/fluent-ffmpeg": "2.1.20", "@types/ioredis": "4.28.10", - "@types/jest": "29.2.4", + "@types/jest": "29.2.5", "@types/js-yaml": "4.0.5", "@types/jsdom": "20.0.1", "@types/jsonld": "1.5.8", @@ -161,7 +160,7 @@ "@types/rename": "1.0.4", "@types/sanitize-html": "2.8.0", "@types/semver": "7.3.13", - "@types/sharp": "0.31.0", + "@types/sharp": "0.31.1", "@types/sinonjs__fake-timers": "8.1.2", "@types/speakeasy": "2.0.7", "@types/syslog-pro": "^1.0.0", @@ -172,15 +171,16 @@ "@types/vary": "1.1.0", "@types/web-push": "3.3.2", "@types/websocket": "1.0.5", - "@types/ws": "8.5.3", + "@types/ws": "8.5.4", "@typescript-eslint/eslint-plugin": "5.47.1", "@typescript-eslint/parser": "5.47.1", "cross-env": "7.0.3", - "eslint": "8.30.0", + "eslint": "8.31.0", "eslint-plugin-import": "2.26.0", "execa": "6.1.0", "jest": "29.3.1", "jest-mock": "^29.3.1", + "node-fetch": "3.3.0", "typescript": "4.9.4" } } diff --git a/packages/backend/src/core/CaptchaService.ts b/packages/backend/src/core/CaptchaService.ts index 0207cf58a0..6b5cb4db32 100644 --- a/packages/backend/src/core/CaptchaService.ts +++ b/packages/backend/src/core/CaptchaService.ts @@ -27,16 +27,19 @@ export class CaptchaService { response, }); - const res = await fetch(url, { - method: 'POST', - body: params, - headers: { - 'User-Agent': this.config.userAgent, + const res = await this.httpRequestService.fetch( + url, + { + method: 'POST', + body: params, + headers: { + 'User-Agent': this.config.userAgent, + }, }, - // TODO - //timeout: 10 * 1000, - agent: (url, bypassProxy) => this.httpRequestService.getAgentByUrl(url, bypassProxy), - }).catch(err => { + { + noOkError: true, + } + ).catch(err => { throw `${err.message ?? err}`; }); diff --git a/packages/backend/src/core/DownloadService.ts b/packages/backend/src/core/DownloadService.ts index 587fbefe64..afbe1f03d3 100644 --- a/packages/backend/src/core/DownloadService.ts +++ b/packages/backend/src/core/DownloadService.ts @@ -8,11 +8,12 @@ import got, * as Got from 'got'; import chalk from 'chalk'; import { DI } from '@/di-symbols.js'; import type { Config } from '@/config.js'; -import { HttpRequestService } from '@/core/HttpRequestService.js'; +import { HttpRequestService, UndiciFetcher } from '@/core/HttpRequestService.js'; import { createTemp } from '@/misc/create-temp.js'; import { StatusError } from '@/misc/status-error.js'; import { LoggerService } from '@/core/LoggerService.js'; import type Logger from '@/logger.js'; +import { buildConnector } from 'undici'; const pipeline = util.promisify(stream.pipeline); import { bindThis } from '@/decorators.js'; @@ -20,6 +21,7 @@ import { bindThis } from '@/decorators.js'; @Injectable() export class DownloadService { private logger: Logger; + private undiciFetcher: UndiciFetcher; constructor( @Inject(DI.config) @@ -29,69 +31,58 @@ export class DownloadService { private loggerService: LoggerService, ) { this.logger = this.loggerService.getLogger('download'); + + this.undiciFetcher = new UndiciFetcher(this.httpRequestService.getStandardUndiciFetcherOption( + { + connect: process.env.NODE_ENV === 'development' ? + this.httpRequestService.clientDefaults.connect + : + this.httpRequestService.getConnectorWithIpCheck( + buildConnector({ + ...this.httpRequestService.clientDefaults.connect, + }), + (ip) => !this.isPrivateIp(ip) + ), + bodyTimeout: 30 * 1000, + }, + { + connect: this.httpRequestService.clientDefaults.connect, + } + ), this.logger); } @bindThis - public gotUrl(url: string): Got.Request { + public fetchUrl(url: string): any { this.logger.info(`Downloading ${chalk.cyan(url)} ...`); const timeout = 30 * 1000; const operationTimeout = 60 * 1000; const maxSize = this.config.maxFileSize ?? 262144000; - - const req = got.stream(url, { - headers: { - 'User-Agent': this.config.userAgent, - }, - timeout: { - lookup: timeout, - connect: timeout, - secureConnect: timeout, - socket: timeout, // read timeout - response: timeout, - send: timeout, - request: operationTimeout, // whole operation timeout - }, - agent: { - http: this.httpRequestService.httpAgent, - https: this.httpRequestService.httpsAgent, - }, - http2: false, // default - retry: { - limit: 0, - }, - }).on('response', (res: Got.Response) => { - if ((process.env.NODE_ENV === 'production' || process.env.NODE_ENV === 'test') && !this.config.proxy && res.ip) { - if (this.isPrivateIp(res.ip)) { - this.logger.warn(`Blocked address: ${res.ip}`); - req.destroy(); - } - } - - const contentLength = res.headers['content-length']; - if (contentLength != null) { - const size = Number(contentLength); - if (size > maxSize) { - this.logger.warn(`maxSize exceeded (${size} > ${maxSize}) on response`); - req.destroy(); - } - } - }).on('downloadProgress', (progress: Got.Progress) => { - if (progress.transferred > maxSize) { - this.logger.warn(`maxSize exceeded (${progress.transferred} > ${maxSize}) on downloadProgress`); - req.destroy(); - } - }); - return req; + const response = await this.undiciFetcher.fetch( + url, + { + method: 'GET', + headers: { + 'User-Agent': this.config.userAgent, + }, + } + ); + + if (response.body === null) { + throw new StatusError('No body', 400, 'No body'); + } + + this.logger.succ(`Download finished: ${chalk.cyan(url)}`); + + return response; } @bindThis - public async pipeRequestToFile(req: Got.Request, path: string): Promise { - const copied = req.pipe(new stream.PassThrough()); + public async pipeRequestToFile(response: any, path: string): Promise { try { this.logger.info(`Saving File to ${chalk.cyanBright(path)} from downloading ...`); - await pipeline(copied, fs.createWriteStream(path)); + await pipeline(stream.Readable.fromWeb(response.body), fs.createWriteStream(path)); } catch (e) { if (e instanceof Got.HTTPError) { throw new StatusError(`${e.response.statusCode} ${e.response.statusMessage}`, e.response.statusCode, e.response.statusMessage); @@ -125,7 +116,7 @@ export class DownloadService { cleanup(); } } - + @bindThis private isPrivateIp(ip: string): boolean { for (const net of this.config.allowedPrivateNetworks ?? []) { @@ -135,6 +126,6 @@ export class DownloadService { } } - return PrivateIp(ip); + return PrivateIp(ip) ?? false; } } diff --git a/packages/backend/src/core/FederatedInstanceService.ts b/packages/backend/src/core/FederatedInstanceService.ts index 97745a1168..d517117da6 100644 --- a/packages/backend/src/core/FederatedInstanceService.ts +++ b/packages/backend/src/core/FederatedInstanceService.ts @@ -22,7 +22,7 @@ export class FederatedInstanceService { } @bindThis - public async registerOrFetchInstanceDoc(host: string): Promise { + public async fetch(host: string): Promise { host = this.utilityService.toPuny(host); const cached = this.cache.get(host); @@ -35,7 +35,6 @@ export class FederatedInstanceService { id: this.idService.genId(), host, caughtAt: new Date(), - lastCommunicatedAt: new Date(), }).then(x => this.instancesRepository.findOneByOrFail(x.identifiers[0])); this.cache.set(host, i); @@ -45,4 +44,17 @@ export class FederatedInstanceService { return index; } } + + @bindThis + public async updateCachePartial(host: string, data: Partial): Promise { + host = this.utilityService.toPuny(host); + + const cached = this.cache.get(host); + if (cached == null) return; + + this.cache.set(host, { + ...cached, + ...data, + }); + } } diff --git a/packages/backend/src/core/FetchInstanceMetadataService.ts b/packages/backend/src/core/FetchInstanceMetadataService.ts index 7eea45200e..cb9d099a22 100644 --- a/packages/backend/src/core/FetchInstanceMetadataService.ts +++ b/packages/backend/src/core/FetchInstanceMetadataService.ts @@ -1,7 +1,6 @@ import { URL } from 'node:url'; import { Inject, Injectable } from '@nestjs/common'; import { JSDOM } from 'jsdom'; -import fetch from 'node-fetch'; import tinycolor from 'tinycolor2'; import type { Instance } from '@/models/entities/Instance.js'; import type { InstancesRepository } from '@/models/index.js'; @@ -191,11 +190,7 @@ export class FetchInstanceMetadataService { const faviconUrl = url + '/favicon.ico'; - const favicon = await fetch(faviconUrl, { - // TODO - //timeout: 10000, - agent: url => this.httpRequestService.getAgentByUrl(url), - }); + const favicon = await this.httpRequestService.fetch(faviconUrl, {}, { noOkError: true }); if (favicon.ok) { return faviconUrl; diff --git a/packages/backend/src/core/HttpRequestService.ts b/packages/backend/src/core/HttpRequestService.ts index 49b28ae523..632e8f98ae 100644 --- a/packages/backend/src/core/HttpRequestService.ts +++ b/packages/backend/src/core/HttpRequestService.ts @@ -1,67 +1,255 @@ import * as http from 'node:http'; import * as https from 'node:https'; import CacheableLookup from 'cacheable-lookup'; -import fetch from 'node-fetch'; import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent'; import { Inject, Injectable } from '@nestjs/common'; import { DI } from '@/di-symbols.js'; import type { Config } from '@/config.js'; import { StatusError } from '@/misc/status-error.js'; import { bindThis } from '@/decorators.js'; -import type { Response } from 'node-fetch'; -import type { URL } from 'node:url'; +import * as undici from 'undici'; +import { LookupFunction } from 'node:net'; +import { LoggerService } from '@/core/LoggerService.js'; +import type Logger from '@/logger.js'; + +// true to allow, false to deny +export type IpChecker = (ip: string) => boolean; + +/* + * Child class to create and save Agent for fetch. + * You should construct this when you want + * to change timeout, size limit, socket connect function, etc. + */ +export class UndiciFetcher { + /** + * Get http non-proxy agent (undici) + */ + public nonProxiedAgent: undici.Agent; + + /** + * Get http proxy or non-proxy agent (undici) + */ + public agent: undici.ProxyAgent | undici.Agent; + + private proxyBypassHosts: string[]; + private userAgent: string | undefined; + + private logger: Logger | undefined; + + constructor( + args: { + agentOptions: undici.Agent.Options; + proxy?: { + uri: string; + options?: undici.Agent.Options; // Override of agentOptions + }, + proxyBypassHosts?: string[]; + userAgent?: string; + }, + logger?: Logger, + ) { + this.logger = logger; + this.logger?.debug('UndiciFetcher constructor', args); + + this.proxyBypassHosts = args.proxyBypassHosts ?? []; + this.userAgent = args.userAgent; + + this.nonProxiedAgent = new undici.Agent({ + ...args.agentOptions, + connect: (process.env.NODE_ENV !== 'production' && typeof args.agentOptions.connect !== 'function') + ? (options, cb) => { + // Custom connector for debug + undici.buildConnector(args.agentOptions.connect as undici.buildConnector.BuildOptions)(options, (err, socket) => { + this.logger?.debug('Socket connector called', socket); + if (err) { + this.logger?.debug(`Socket error`, err); + cb(new Error(`Error while socket connecting\n${err}`), null); + return; + } + this.logger?.debug(`Socket connected: port ${socket.localPort} => remote ${socket.remoteAddress}`); + cb(null, socket); + }); + } : args.agentOptions.connect, + }); + + this.agent = args.proxy + ? new undici.ProxyAgent({ + ...args.agentOptions, + ...args.proxy.options, + + uri: args.proxy.uri, + + connect: (process.env.NODE_ENV !== 'production' && typeof (args.proxy?.options?.connect ?? args.agentOptions.connect) !== 'function') + ? (options, cb) => { + // Custom connector for debug + undici.buildConnector((args.proxy?.options?.connect ?? args.agentOptions.connect) as undici.buildConnector.BuildOptions)(options, (err, socket) => { + this.logger?.debug('Socket connector called (secure)', socket); + if (err) { + this.logger?.debug(`Socket error`, err); + cb(new Error(`Error while socket connecting\n${err}`), null); + return; + } + this.logger?.debug(`Socket connected (secure): port ${socket.localPort} => remote ${socket.remoteAddress}`); + cb(null, socket); + }); + } : (args.proxy?.options?.connect ?? args.agentOptions.connect), + }) + : this.nonProxiedAgent; + } + + /** + * Get agent by URL + * @param url URL + * @param bypassProxy Allways bypass proxy + */ + @bindThis + public getAgentByUrl(url: URL, bypassProxy = false): undici.Agent | undici.ProxyAgent { + if (bypassProxy || this.proxyBypassHosts.includes(url.hostname)) { + return this.nonProxiedAgent; + } else { + return this.agent; + } + } + + @bindThis + public async fetch( + url: string | URL, + options: undici.RequestInit = {}, + privateOptions: { noOkError?: boolean; bypassProxy?: boolean; } = { noOkError: false, bypassProxy: false } + ): Promise { + const res = await undici.fetch(url, { + dispatcher: this.getAgentByUrl(new URL(url), privateOptions.bypassProxy), + ...options, + }).catch((err) => { + this.logger?.error('fetch error', err); + throw new StatusError('Resource Unreachable', 500, 'Resource Unreachable'); + }); + if (!res.ok && !privateOptions.noOkError) { + throw new StatusError(`${res.status} ${res.statusText}`, res.status, res.statusText); + } + return res; + } + + @bindThis + public async getJson(url: string, accept = 'application/json, */*', headers?: Record): Promise { + const res = await this.fetch( + url, + { + headers: Object.assign({ + 'User-Agent': this.userAgent, + Accept: accept, + }, headers ?? {}), + } + ); + + return await res.json() as T; + } + + @bindThis + public async getHtml(url: string, accept = 'text/html, */*', headers?: Record): Promise { + const res = await this.fetch( + url, + { + headers: Object.assign({ + 'User-Agent': this.userAgent, + Accept: accept, + }, headers ?? {}), + } + ); + + return await res.text(); + } +} @Injectable() export class HttpRequestService { - /** - * Get http non-proxy agent - */ + public defaultFetcher: UndiciFetcher; + public fetch: UndiciFetcher['fetch']; + public getHtml: UndiciFetcher['getHtml']; + public defaultJsonFetcher: UndiciFetcher; + public getJson: UndiciFetcher['getJson']; + + //#region for old http/https, only used in S3Service + // http non-proxy agent private http: http.Agent; - /** - * Get https non-proxy agent - */ + // https non-proxy agent private https: https.Agent; - /** - * Get http proxy or non-proxy agent - */ + // http proxy or non-proxy agent public httpAgent: http.Agent; - /** - * Get https proxy or non-proxy agent - */ + // https proxy or non-proxy agent public httpsAgent: https.Agent; + //#endregion + + public readonly dnsCache: CacheableLookup; + public readonly clientDefaults: undici.Agent.Options; + private maxSockets: number; + + private logger: Logger; constructor( @Inject(DI.config) private config: Config, + private loggerService: LoggerService, ) { - const cache = new CacheableLookup({ + this.logger = this.loggerService.getLogger('http-request'); + + this.dnsCache = new CacheableLookup({ maxTtl: 3600, // 1hours errorTtl: 30, // 30secs lookup: false, // nativeのdns.lookupにfallbackしない }); - + + this.clientDefaults = { + keepAliveTimeout: 30 * 1000, + keepAliveMaxTimeout: 10 * 60 * 1000, + keepAliveTimeoutThreshold: 1 * 1000, + strictContentLength: true, + headersTimeout: 10 * 1000, + bodyTimeout: 10 * 1000, + maxHeaderSize: 16364, // default + maxResponseSize: 10 * 1024 * 1024, + maxRedirections: 3, + connect: { + timeout: 10 * 1000, // コネクションが確立するまでのタイムアウト + maxCachedSessions: 300, // TLSセッションのキャッシュ数 https://github.com/nodejs/undici/blob/v5.14.0/lib/core/connect.js#L80 + lookup: this.dnsCache.lookup as LookupFunction, // https://github.com/nodejs/undici/blob/v5.14.0/lib/core/connect.js#L98 + }, + } + + this.maxSockets = Math.max(256, this.config.deliverJobConcurrency ?? 128); + + this.defaultFetcher = new UndiciFetcher(this.getStandardUndiciFetcherOption(), this.logger); + + this.fetch = this.defaultFetcher.fetch; + this.getHtml = this.defaultFetcher.getHtml; + + this.defaultJsonFetcher = new UndiciFetcher(this.getStandardUndiciFetcherOption({ + maxResponseSize: 1024 * 256, + }), this.logger); + + this.getJson = this.defaultJsonFetcher.getJson; + + //#region for old http/https, only used in S3Service this.http = new http.Agent({ keepAlive: true, keepAliveMsecs: 30 * 1000, - lookup: cache.lookup, + lookup: this.dnsCache.lookup, } as http.AgentOptions); this.https = new https.Agent({ keepAlive: true, keepAliveMsecs: 30 * 1000, - lookup: cache.lookup, + lookup: this.dnsCache.lookup, } as https.AgentOptions); - - const maxSockets = Math.max(256, config.deliverJobConcurrency ?? 128); - + this.httpAgent = config.proxy ? new HttpProxyAgent({ keepAlive: true, keepAliveMsecs: 30 * 1000, - maxSockets, + maxSockets: this.maxSockets, maxFreeSockets: 256, scheduling: 'lifo', proxy: config.proxy, @@ -72,21 +260,46 @@ export class HttpRequestService { ? new HttpsProxyAgent({ keepAlive: true, keepAliveMsecs: 30 * 1000, - maxSockets, + maxSockets: this.maxSockets, maxFreeSockets: 256, scheduling: 'lifo', proxy: config.proxy, }) : this.https; + //#endregion } /** - * Get agent by URL + * Get http agent by URL * @param url URL * @param bypassProxy Allways bypass proxy */ @bindThis - public getAgentByUrl(url: URL, bypassProxy = false): http.Agent | https.Agent { + public getStandardUndiciFetcherOption(opts: undici.Agent.Options = {}, proxyOpts: undici.Agent.Options = {}) { + return { + agentOptions: { + ...this.clientDefaults, + ...opts, + }, + ...(this.config.proxy ? { + proxy: { + uri: this.config.proxy, + options: { + connections: this.maxSockets, + ...proxyOpts, + } + } + } : {}), + } + } + + /** + * Get http agent by URL + * @param url URL + * @param bypassProxy Allways bypass proxy + */ + @bindThis + public getHttpAgentByUrl(url: URL, bypassProxy = false): http.Agent | https.Agent { if (bypassProxy || (this.config.proxyBypassHosts || []).includes(url.hostname)) { return url.protocol === 'http:' ? this.http : this.https; } else { @@ -94,67 +307,37 @@ export class HttpRequestService { } } + /** + * check ip + */ @bindThis - public async getJson(url: string, accept = 'application/json, */*', timeout = 10000, headers?: Record): Promise { - const res = await this.getResponse({ - url, - method: 'GET', - headers: Object.assign({ - 'User-Agent': this.config.userAgent, - Accept: accept, - }, headers ?? {}), - timeout, - size: 1024 * 256, - }); + public getConnectorWithIpCheck(connector: undici.buildConnector.connector, checkIp: IpChecker): undici.buildConnector.connectorAsync { + return (options, cb) => { + connector(options, (err, socket) => { + this.logger.debug('Socket connector (with ip checker) called', socket); + if (err) { + this.logger.error(`Socket error`, err) + cb(new Error(`Error while socket connecting\n${err}`), null); + return; + } - return await res.json(); - } + if (socket.remoteAddress == undefined) { + this.logger.error(`Socket error: remoteAddress is undefined`); + cb(new Error('remoteAddress is undefined (maybe socket destroyed)'), null); + return; + } - @bindThis - public async getHtml(url: string, accept = 'text/html, */*', timeout = 10000, headers?: Record): Promise { - const res = await this.getResponse({ - url, - method: 'GET', - headers: Object.assign({ - 'User-Agent': this.config.userAgent, - Accept: accept, - }, headers ?? {}), - timeout, - }); + // allow + if (checkIp(socket.remoteAddress)) { + this.logger.debug(`Socket connected (ip ok): ${socket.localPort} => ${socket.remoteAddress}`); + cb(null, socket); + return; + } - return await res.text(); - } - - @bindThis - public async getResponse(args: { - url: string, - method: string, - body?: string, - headers: Record, - timeout?: number, - size?: number, - }): Promise { - const timeout = args.timeout ?? 10 * 1000; - - const controller = new AbortController(); - setTimeout(() => { - controller.abort(); - }, timeout * 6); - - const res = await fetch(args.url, { - method: args.method, - headers: args.headers, - body: args.body, - timeout, - size: args.size ?? 10 * 1024 * 1024, - agent: (url) => this.getAgentByUrl(url), - signal: controller.signal, - }); - - if (!res.ok) { - throw new StatusError(`${res.status} ${res.statusText}`, res.status, res.statusText); - } - - return res; + this.logger.error('IP is not allowed', socket); + cb(new StatusError('IP is not allowed', 403, 'IP is not allowed'), null); + socket.destroy(); + }); + }; } } diff --git a/packages/backend/src/core/NoteCreateService.ts b/packages/backend/src/core/NoteCreateService.ts index cc6c213446..6038840406 100644 --- a/packages/backend/src/core/NoteCreateService.ts +++ b/packages/backend/src/core/NoteCreateService.ts @@ -428,7 +428,7 @@ export class NoteCreateService { // Register host if (this.userEntityService.isRemoteUser(user)) { - this.federatedInstanceService.registerOrFetchInstanceDoc(user.host).then(i => { + this.federatedInstanceService.fetch(user.host).then(i => { this.instancesRepository.increment({ id: i.id }, 'notesCount', 1); this.instanceChart.updateNote(i.host, note, true); }); diff --git a/packages/backend/src/core/NoteDeleteService.ts b/packages/backend/src/core/NoteDeleteService.ts index 1313fcbd3d..b1f16b6e8a 100644 --- a/packages/backend/src/core/NoteDeleteService.ts +++ b/packages/backend/src/core/NoteDeleteService.ts @@ -100,7 +100,7 @@ export class NoteDeleteService { this.perUserNotesChart.update(user, note, false); if (this.userEntityService.isRemoteUser(user)) { - this.federatedInstanceService.registerOrFetchInstanceDoc(user.host).then(i => { + this.federatedInstanceService.fetch(user.host).then(i => { this.instancesRepository.decrement({ id: i.id }, 'notesCount', 1); this.instanceChart.updateNote(i.host, note, false); }); diff --git a/packages/backend/src/core/S3Service.ts b/packages/backend/src/core/S3Service.ts index 0ce69aaa74..930188ce6e 100644 --- a/packages/backend/src/core/S3Service.ts +++ b/packages/backend/src/core/S3Service.ts @@ -33,7 +33,7 @@ export class S3Service { ? false : meta.objectStorageS3ForcePathStyle, httpOptions: { - agent: this.httpRequestService.getAgentByUrl(new URL(u), !meta.objectStorageUseProxy), + agent: this.httpRequestService.getHttpAgentByUrl(new URL(u), !meta.objectStorageUseProxy), }, }); } diff --git a/packages/backend/src/core/UserFollowingService.ts b/packages/backend/src/core/UserFollowingService.ts index 074aae86c8..52834c375e 100644 --- a/packages/backend/src/core/UserFollowingService.ts +++ b/packages/backend/src/core/UserFollowingService.ts @@ -205,12 +205,12 @@ export class UserFollowingService { //#region Update instance stats if (this.userEntityService.isRemoteUser(follower) && this.userEntityService.isLocalUser(followee)) { - this.federatedInstanceService.registerOrFetchInstanceDoc(follower.host).then(i => { + this.federatedInstanceService.fetch(follower.host).then(i => { this.instancesRepository.increment({ id: i.id }, 'followingCount', 1); this.instanceChart.updateFollowing(i.host, true); }); } else if (this.userEntityService.isLocalUser(follower) && this.userEntityService.isRemoteUser(followee)) { - this.federatedInstanceService.registerOrFetchInstanceDoc(followee.host).then(i => { + this.federatedInstanceService.fetch(followee.host).then(i => { this.instancesRepository.increment({ id: i.id }, 'followersCount', 1); this.instanceChart.updateFollowers(i.host, true); }); @@ -323,12 +323,12 @@ export class UserFollowingService { //#region Update instance stats if (this.userEntityService.isRemoteUser(follower) && this.userEntityService.isLocalUser(followee)) { - this.federatedInstanceService.registerOrFetchInstanceDoc(follower.host).then(i => { + this.federatedInstanceService.fetch(follower.host).then(i => { this.instancesRepository.decrement({ id: i.id }, 'followingCount', 1); this.instanceChart.updateFollowing(i.host, false); }); } else if (this.userEntityService.isLocalUser(follower) && this.userEntityService.isRemoteUser(followee)) { - this.federatedInstanceService.registerOrFetchInstanceDoc(followee.host).then(i => { + this.federatedInstanceService.fetch(followee.host).then(i => { this.instancesRepository.decrement({ id: i.id }, 'followersCount', 1); this.instanceChart.updateFollowers(i.host, false); }); diff --git a/packages/backend/src/core/WebfingerService.ts b/packages/backend/src/core/WebfingerService.ts index 4c91ab8438..69df2d0c1b 100644 --- a/packages/backend/src/core/WebfingerService.ts +++ b/packages/backend/src/core/WebfingerService.ts @@ -30,7 +30,7 @@ export class WebfingerService { public async webfinger(query: string): Promise { const url = this.genUrl(query); - return await this.httpRequestService.getJson(url, 'application/jrd+json, application/json') as IWebFinger; + return await this.httpRequestService.getJson(url, 'application/jrd+json, application/json'); } @bindThis diff --git a/packages/backend/src/core/activitypub/ApRequestService.ts b/packages/backend/src/core/activitypub/ApRequestService.ts index d1edd579fa..c686a7bfd8 100644 --- a/packages/backend/src/core/activitypub/ApRequestService.ts +++ b/packages/backend/src/core/activitypub/ApRequestService.ts @@ -5,7 +5,7 @@ import { DI } from '@/di-symbols.js'; import type { Config } from '@/config.js'; import type { User } from '@/models/entities/User.js'; import { UserKeypairStoreService } from '@/core/UserKeypairStoreService.js'; -import { HttpRequestService } from '@/core/HttpRequestService.js'; +import { HttpRequestService, UndiciFetcher } from '@/core/HttpRequestService.js'; import { bindThis } from '@/decorators.js'; type Request = { @@ -28,6 +28,8 @@ type PrivateKey = { @Injectable() export class ApRequestService { + private undiciFetcher: UndiciFetcher; + constructor( @Inject(DI.config) private config: Config, @@ -35,6 +37,9 @@ export class ApRequestService { private userKeypairStoreService: UserKeypairStoreService, private httpRequestService: HttpRequestService, ) { + this.undiciFetcher = new UndiciFetcher(this.httpRequestService.getStandardUndiciFetcherOption({ + maxRedirections: 0, + })); } @bindThis @@ -152,12 +157,14 @@ export class ApRequestService { }, }); - await this.httpRequestService.getResponse({ + await this.undiciFetcher.fetch( url, - method: req.request.method, - headers: req.request.headers, - body, - }); + { + method: req.request.method, + headers: req.request.headers, + body, + } + ); } /** @@ -180,11 +187,13 @@ export class ApRequestService { }, }); - const res = await this.httpRequestService.getResponse({ + const res = await this.httpRequestService.fetch( url, - method: req.request.method, - headers: req.request.headers, - }); + { + method: req.request.method, + headers: req.request.headers, + } + ); return await res.json(); } diff --git a/packages/backend/src/core/activitypub/ApResolverService.ts b/packages/backend/src/core/activitypub/ApResolverService.ts index e96c84f148..2f8c97d375 100644 --- a/packages/backend/src/core/activitypub/ApResolverService.ts +++ b/packages/backend/src/core/activitypub/ApResolverService.ts @@ -4,7 +4,7 @@ import { InstanceActorService } from '@/core/InstanceActorService.js'; import type { NotesRepository, PollsRepository, NoteReactionsRepository, UsersRepository } from '@/models/index.js'; import type { Config } from '@/config.js'; import { MetaService } from '@/core/MetaService.js'; -import { HttpRequestService } from '@/core/HttpRequestService.js'; +import { HttpRequestService, UndiciFetcher } from '@/core/HttpRequestService.js'; import { DI } from '@/di-symbols.js'; import { UtilityService } from '@/core/UtilityService.js'; import { bindThis } from '@/decorators.js'; @@ -17,6 +17,7 @@ import type { IObject, ICollection, IOrderedCollection } from './type.js'; export class Resolver { private history: Set; private user?: ILocalUser; + private undiciFetcher: UndiciFetcher; constructor( private config: Config, @@ -34,6 +35,9 @@ export class Resolver { private recursionLimit = 100, ) { this.history = new Set(); + this.undiciFetcher = new UndiciFetcher(this.httpRequestService.getStandardUndiciFetcherOption({ + maxRedirections: 0, + })); } @bindThis @@ -96,8 +100,8 @@ export class Resolver { } const object = (this.user - ? await this.apRequestService.signedGet(value, this.user) - : await this.httpRequestService.getJson(value, 'application/activity+json, application/ld+json')) as IObject; + ? await this.apRequestService.signedGet(value, this.user) as IObject + : await this.undiciFetcher.getJson(value, 'application/activity+json, application/ld+json')); if (object == null || ( Array.isArray(object['@context']) ? diff --git a/packages/backend/src/core/activitypub/LdSignatureService.ts b/packages/backend/src/core/activitypub/LdSignatureService.ts index b71320ed0b..4e4b7dce2d 100644 --- a/packages/backend/src/core/activitypub/LdSignatureService.ts +++ b/packages/backend/src/core/activitypub/LdSignatureService.ts @@ -1,6 +1,5 @@ import * as crypto from 'node:crypto'; import { Inject, Injectable } from '@nestjs/common'; -import fetch from 'node-fetch'; import { HttpRequestService } from '@/core/HttpRequestService.js'; import { bindThis } from '@/decorators.js'; import { CONTEXTS } from './misc/contexts.js'; @@ -116,14 +115,19 @@ class LdSignature { @bindThis private async fetchDocument(url: string) { - const json = await fetch(url, { - headers: { - Accept: 'application/ld+json, application/json', + const json = await this.httpRequestService.fetch( + url, + { + headers: { + Accept: 'application/ld+json, application/json', + }, + // TODO + //timeout: this.loderTimeout, }, - // TODO - //timeout: this.loderTimeout, - agent: u => u.protocol === 'http:' ? this.httpRequestService.httpAgent : this.httpRequestService.httpsAgent, - }).then(res => { + { + noOkError: true, + } + ).then(res => { if (!res.ok) { throw `${res.status} ${res.statusText}`; } else { diff --git a/packages/backend/src/core/activitypub/models/ApPersonService.ts b/packages/backend/src/core/activitypub/models/ApPersonService.ts index d5faf37df2..e08f33c906 100644 --- a/packages/backend/src/core/activitypub/models/ApPersonService.ts +++ b/packages/backend/src/core/activitypub/models/ApPersonService.ts @@ -348,7 +348,7 @@ export class ApPersonService implements OnModuleInit { } // Register host - this.federatedInstanceService.registerOrFetchInstanceDoc(host).then(i => { + this.federatedInstanceService.fetch(host).then(i => { this.instancesRepository.increment({ id: i.id }, 'usersCount', 1); this.instanceChart.newUser(i.host); this.fetchInstanceMetadataService.fetchInstanceMetadata(i); diff --git a/packages/backend/src/core/chart/charts/federation.ts b/packages/backend/src/core/chart/charts/federation.ts index d9234e8028..b8012809f7 100644 --- a/packages/backend/src/core/chart/charts/federation.ts +++ b/packages/backend/src/core/chart/charts/federation.ts @@ -86,7 +86,7 @@ export default class FederationChart extends Chart { .where(`instance.host IN (${ subInstancesQuery.getQuery() })`) .andWhere(meta.blockedHosts.length === 0 ? '1=1' : 'instance.host NOT IN (:...blocked)', { blocked: meta.blockedHosts }) .andWhere('instance.isSuspended = false') - .andWhere('instance.lastCommunicatedAt > :gt', { gt: new Date(Date.now() - (1000 * 60 * 60 * 24 * 30)) }) + .andWhere('instance.isNotResponding = false') .getRawOne() .then(x => parseInt(x.count, 10)), this.instancesRepository.createQueryBuilder('instance') @@ -94,7 +94,7 @@ export default class FederationChart extends Chart { .where(`instance.host IN (${ pubInstancesQuery.getQuery() })`) .andWhere(meta.blockedHosts.length === 0 ? '1=1' : 'instance.host NOT IN (:...blocked)', { blocked: meta.blockedHosts }) .andWhere('instance.isSuspended = false') - .andWhere('instance.lastCommunicatedAt > :gt', { gt: new Date(Date.now() - (1000 * 60 * 60 * 24 * 30)) }) + .andWhere('instance.isNotResponding = false') .getRawOne() .then(x => parseInt(x.count, 10)), ]); diff --git a/packages/backend/src/core/entities/InstanceEntityService.ts b/packages/backend/src/core/entities/InstanceEntityService.ts index 5a7ceb89a3..81d02bb331 100644 --- a/packages/backend/src/core/entities/InstanceEntityService.ts +++ b/packages/backend/src/core/entities/InstanceEntityService.ts @@ -7,8 +7,8 @@ import type { } from '@/models/entities/Blocking.js'; import type { User } from '@/models/entities/User.js'; import type { Instance } from '@/models/entities/Instance.js'; import { MetaService } from '@/core/MetaService.js'; -import { UserEntityService } from './UserEntityService.js'; import { bindThis } from '@/decorators.js'; +import { UserEntityService } from './UserEntityService.js'; @Injectable() export class InstanceEntityService { @@ -33,8 +33,6 @@ export class InstanceEntityService { notesCount: instance.notesCount, followingCount: instance.followingCount, followersCount: instance.followersCount, - latestRequestSentAt: instance.latestRequestSentAt ? instance.latestRequestSentAt.toISOString() : null, - lastCommunicatedAt: instance.lastCommunicatedAt.toISOString(), isNotResponding: instance.isNotResponding, isSuspended: instance.isSuspended, isBlocked: meta.blockedHosts.includes(instance.host), diff --git a/packages/backend/src/models/entities/Instance.ts b/packages/backend/src/models/entities/Instance.ts index 7ea9234384..8092f67c86 100644 --- a/packages/backend/src/models/entities/Instance.ts +++ b/packages/backend/src/models/entities/Instance.ts @@ -59,22 +59,6 @@ export class Instance { }) public followersCount: number; - /** - * 直近のリクエスト送信日時 - */ - @Column('timestamp with time zone', { - nullable: true, - }) - public latestRequestSentAt: Date | null; - - /** - * 直近のリクエスト送信時のHTTPステータスコード - */ - @Column('integer', { - nullable: true, - }) - public latestStatus: number | null; - /** * 直近のリクエスト受信日時 */ @@ -83,12 +67,6 @@ export class Instance { }) public latestRequestReceivedAt: Date | null; - /** - * このインスタンスと最後にやり取りした日時 - */ - @Column('timestamp with time zone') - public lastCommunicatedAt: Date; - /** * このインスタンスと不通かどうか */ diff --git a/packages/backend/src/models/schema/federation-instance.ts b/packages/backend/src/models/schema/federation-instance.ts index c57b3fec19..f3f93f3097 100644 --- a/packages/backend/src/models/schema/federation-instance.ts +++ b/packages/backend/src/models/schema/federation-instance.ts @@ -32,16 +32,6 @@ export const packedFederationInstanceSchema = { type: 'number', optional: false, nullable: false, }, - latestRequestSentAt: { - type: 'string', - optional: false, nullable: true, - format: 'date-time', - }, - lastCommunicatedAt: { - type: 'string', - optional: false, nullable: false, - format: 'date-time', - }, isNotResponding: { type: 'boolean', optional: false, nullable: false, diff --git a/packages/backend/src/queue/processors/DeliverProcessorService.ts b/packages/backend/src/queue/processors/DeliverProcessorService.ts index 58969d550e..c5e4a66517 100644 --- a/packages/backend/src/queue/processors/DeliverProcessorService.ts +++ b/packages/backend/src/queue/processors/DeliverProcessorService.ts @@ -15,10 +15,10 @@ import ApRequestChart from '@/core/chart/charts/ap-request.js'; import FederationChart from '@/core/chart/charts/federation.js'; import { StatusError } from '@/misc/status-error.js'; import { UtilityService } from '@/core/UtilityService.js'; +import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; import type Bull from 'bull'; import type { DeliverJobData } from '../types.js'; -import { bindThis } from '@/decorators.js'; @Injectable() export class DeliverProcessorService { @@ -48,7 +48,6 @@ export class DeliverProcessorService { ) { this.logger = this.queueLoggerService.logger.createSubLogger('deliver'); this.suspendedHostsCache = new Cache(1000 * 60 * 60); - this.latest = null; } @bindThis @@ -76,20 +75,18 @@ export class DeliverProcessorService { } try { - if (this.latest !== (this.latest = JSON.stringify(job.data.content, null, 2))) { - this.logger.debug(`delivering ${this.latest}`); - } - await this.apRequestService.signedPost(job.data.user, job.data.to, job.data.content); // Update stats - this.federatedInstanceService.registerOrFetchInstanceDoc(host).then(i => { - this.instancesRepository.update(i.id, { - latestRequestSentAt: new Date(), - latestStatus: 200, - lastCommunicatedAt: new Date(), - isNotResponding: false, - }); + this.federatedInstanceService.fetch(host).then(i => { + if (i.isNotResponding) { + this.instancesRepository.update(i.id, { + isNotResponding: false, + }); + this.federatedInstanceService.updateCachePartial(host, { + isNotResponding: false, + }); + } this.fetchInstanceMetadataService.fetchInstanceMetadata(i); @@ -100,13 +97,16 @@ export class DeliverProcessorService { return 'Success'; } catch (res) { - // Update stats - this.federatedInstanceService.registerOrFetchInstanceDoc(host).then(i => { - this.instancesRepository.update(i.id, { - latestRequestSentAt: new Date(), - latestStatus: res instanceof StatusError ? res.statusCode : null, - isNotResponding: true, - }); + // Update stats + this.federatedInstanceService.fetch(host).then(i => { + if (!i.isNotResponding) { + this.instancesRepository.update(i.id, { + isNotResponding: true, + }); + this.federatedInstanceService.updateCachePartial(host, { + isNotResponding: true, + }); + } this.instanceChart.requestSent(i.host, false); this.apRequestChart.deliverFail(); @@ -114,17 +114,17 @@ export class DeliverProcessorService { }); if (res instanceof StatusError) { - // 4xx + // 4xx if (res.isClientError) { - // HTTPステータスコード4xxはクライアントエラーであり、それはつまり - // 何回再送しても成功することはないということなのでエラーにはしないでおく + // HTTPステータスコード4xxはクライアントエラーであり、それはつまり + // 何回再送しても成功することはないということなのでエラーにはしないでおく return `${res.statusCode} ${res.statusMessage}`; } // 5xx etc. throw `${res.statusCode} ${res.statusMessage}`; } else { - // DNS error, socket error, timeout ... + // DNS error, socket error, timeout ... throw res; } } diff --git a/packages/backend/src/queue/processors/InboxProcessorService.ts b/packages/backend/src/queue/processors/InboxProcessorService.ts index c032122caf..d033637849 100644 --- a/packages/backend/src/queue/processors/InboxProcessorService.ts +++ b/packages/backend/src/queue/processors/InboxProcessorService.ts @@ -176,10 +176,12 @@ export class InboxProcessorService { } // Update stats - this.federatedInstanceService.registerOrFetchInstanceDoc(authUser.user.host).then(i => { + this.federatedInstanceService.fetch(authUser.user.host).then(i => { this.instancesRepository.update(i.id, { latestRequestReceivedAt: new Date(), - lastCommunicatedAt: new Date(), + isNotResponding: false, + }); + this.federatedInstanceService.updateCachePartial(host, { isNotResponding: false, }); diff --git a/packages/backend/src/queue/processors/WebhookDeliverProcessorService.ts b/packages/backend/src/queue/processors/WebhookDeliverProcessorService.ts index 183ef07477..f0543a5ed1 100644 --- a/packages/backend/src/queue/processors/WebhookDeliverProcessorService.ts +++ b/packages/backend/src/queue/processors/WebhookDeliverProcessorService.ts @@ -33,24 +33,26 @@ export class WebhookDeliverProcessorService { try { this.logger.debug(`delivering ${job.data.webhookId}`); - const res = await this.httpRequestService.getResponse({ - url: job.data.to, - method: 'POST', - headers: { - 'User-Agent': 'Misskey-Hooks', - 'X-Misskey-Host': this.config.host, - 'X-Misskey-Hook-Id': job.data.webhookId, - 'X-Misskey-Hook-Secret': job.data.secret, - }, - body: JSON.stringify({ - hookId: job.data.webhookId, - userId: job.data.userId, - eventId: job.data.eventId, - createdAt: job.data.createdAt, - type: job.data.type, - body: job.data.content, - }), - }); + const res = await this.httpRequestService.fetch( + job.data.to, + { + method: 'POST', + headers: { + 'User-Agent': 'Misskey-Hooks', + 'X-Misskey-Host': this.config.host, + 'X-Misskey-Hook-Id': job.data.webhookId, + 'X-Misskey-Hook-Secret': job.data.secret, + }, + body: JSON.stringify({ + hookId: job.data.webhookId, + userId: job.data.userId, + eventId: job.data.eventId, + createdAt: job.data.createdAt, + type: job.data.type, + body: job.data.content, + }), + } + ); this.webhooksRepository.update({ id: job.data.webhookId }, { latestSentAt: new Date(), diff --git a/packages/backend/src/server/api/endpoints/federation/instances.ts b/packages/backend/src/server/api/endpoints/federation/instances.ts index 81276a7ab0..180887285a 100644 --- a/packages/backend/src/server/api/endpoints/federation/instances.ts +++ b/packages/backend/src/server/api/endpoints/federation/instances.ts @@ -64,8 +64,8 @@ export default class extends Endpoint { case '-followers': query.orderBy('instance.followersCount', 'ASC'); break; case '+caughtAt': query.orderBy('instance.caughtAt', 'DESC'); break; case '-caughtAt': query.orderBy('instance.caughtAt', 'ASC'); break; - case '+lastCommunicatedAt': query.orderBy('instance.lastCommunicatedAt', 'DESC'); break; - case '-lastCommunicatedAt': query.orderBy('instance.lastCommunicatedAt', 'ASC'); break; + case '+latestRequestReceivedAt': query.orderBy('instance.latestRequestReceivedAt', 'DESC'); break; + case '-latestRequestReceivedAt': query.orderBy('instance.latestRequestReceivedAt', 'ASC'); break; default: query.orderBy('instance.id', 'DESC'); break; } diff --git a/packages/backend/src/server/api/endpoints/fetch-rss.ts b/packages/backend/src/server/api/endpoints/fetch-rss.ts index 58fa01ac48..558ec887db 100644 --- a/packages/backend/src/server/api/endpoints/fetch-rss.ts +++ b/packages/backend/src/server/api/endpoints/fetch-rss.ts @@ -33,15 +33,17 @@ export default class extends Endpoint { private httpRequestService: HttpRequestService, ) { super(meta, paramDef, async (ps, me) => { - const res = await this.httpRequestService.getResponse({ - url: ps.url, - method: 'GET', - headers: Object.assign({ - 'User-Agent': config.userAgent, - Accept: 'application/rss+xml, */*', - }), - timeout: 5000, - }); + const res = await this.httpRequestService.fetch( + ps.url, + { + method: 'GET', + headers: Object.assign({ + 'User-Agent': config.userAgent, + Accept: 'application/rss+xml, */*', + }), + // timeout: 5000, + } + ); const text = await res.text(); diff --git a/packages/backend/src/server/api/endpoints/notes/translate.ts b/packages/backend/src/server/api/endpoints/notes/translate.ts index ec16965998..71c50ebba5 100644 --- a/packages/backend/src/server/api/endpoints/notes/translate.ts +++ b/packages/backend/src/server/api/endpoints/notes/translate.ts @@ -1,5 +1,4 @@ import { URLSearchParams } from 'node:url'; -import fetch from 'node-fetch'; import { Inject, Injectable } from '@nestjs/common'; import type { NotesRepository } from '@/models/index.js'; import { Endpoint } from '@/server/api/endpoint-base.js'; @@ -84,25 +83,28 @@ export default class extends Endpoint { const endpoint = instance.deeplIsPro ? 'https://api.deepl.com/v2/translate' : 'https://api-free.deepl.com/v2/translate'; - const res = await fetch(endpoint, { - method: 'POST', - headers: { - 'Content-Type': 'application/x-www-form-urlencoded', - 'User-Agent': config.userAgent, - Accept: 'application/json, */*', + const res = await this.httpRequestService.fetch( + endpoint, + { + method: 'POST', + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + 'User-Agent': config.userAgent, + Accept: 'application/json, */*', + }, + body: params.toString(), }, - body: params, - // TODO - //timeout: 10000, - agent: (url) => this.httpRequestService.getAgentByUrl(url), - }); + { + noOkError: false, + } + ); const json = (await res.json()) as { - translations: { - detected_source_language: string; - text: string; - }[]; - }; + translations: { + detected_source_language: string; + text: string; + }[]; + }; return { sourceLang: json.translations[0].detected_source_language, diff --git a/packages/backend/src/server/api/endpoints/users/show.ts b/packages/backend/src/server/api/endpoints/users/show.ts index 48a6bbf9bc..fcdaeae1c9 100644 --- a/packages/backend/src/server/api/endpoints/users/show.ts +++ b/packages/backend/src/server/api/endpoints/users/show.ts @@ -139,10 +139,12 @@ export default class extends Endpoint { throw new ApiError(meta.errors.noSuchUser); } - if (me == null && ip != null) { - this.perUserPvChart.commitByVisitor(user, ip); - } else if (me && me.id !== user.id) { - this.perUserPvChart.commitByUser(user, me.id); + if (user.host == null) { + if (me == null && ip != null) { + this.perUserPvChart.commitByVisitor(user, ip); + } else if (me && me.id !== user.id) { + this.perUserPvChart.commitByUser(user, me.id); + } } return await this.userEntityService.pack(user, me, { diff --git a/packages/backend/src/server/api/integration/DiscordServerService.ts b/packages/backend/src/server/api/integration/DiscordServerService.ts index 805056da8b..0ac2733817 100644 --- a/packages/backend/src/server/api/integration/DiscordServerService.ts +++ b/packages/backend/src/server/api/integration/DiscordServerService.ts @@ -181,7 +181,7 @@ export class DiscordServerService { } })); - const { id, username, discriminator } = (await this.httpRequestService.getJson('https://discord.com/api/users/@me', '*/*', 10 * 1000, { + const { id, username, discriminator } = (await this.httpRequestService.getJson('https://discord.com/api/users/@me', '*/*', { 'Authorization': `Bearer ${accessToken}`, })) as Record; @@ -249,7 +249,7 @@ export class DiscordServerService { } })); - const { id, username, discriminator } = (await this.httpRequestService.getJson('https://discord.com/api/users/@me', '*/*', 10 * 1000, { + const { id, username, discriminator } = (await this.httpRequestService.getJson('https://discord.com/api/users/@me', '*/*', { 'Authorization': `Bearer ${accessToken}`, })) as Record; if (typeof id !== 'string' || typeof username !== 'string' || typeof discriminator !== 'string') { diff --git a/packages/backend/src/server/api/integration/GithubServerService.ts b/packages/backend/src/server/api/integration/GithubServerService.ts index 6f38c262a1..a8c745d2dc 100644 --- a/packages/backend/src/server/api/integration/GithubServerService.ts +++ b/packages/backend/src/server/api/integration/GithubServerService.ts @@ -174,7 +174,7 @@ export class GithubServerService { } })); - const { login, id } = (await this.httpRequestService.getJson('https://api.github.com/user', 'application/vnd.github.v3+json', 10 * 1000, { + const { login, id } = (await this.httpRequestService.getJson('https://api.github.com/user', 'application/vnd.github.v3+json', { 'Authorization': `bearer ${accessToken}`, })) as Record; if (typeof login !== 'string' || typeof id !== 'string') { @@ -223,7 +223,7 @@ export class GithubServerService { } })); - const { login, id } = (await this.httpRequestService.getJson('https://api.github.com/user', 'application/vnd.github.v3+json', 10 * 1000, { + const { login, id } = (await this.httpRequestService.getJson('https://api.github.com/user', 'application/vnd.github.v3+json', { 'Authorization': `bearer ${accessToken}`, })) as Record; diff --git a/packages/backend/src/server/web/UrlPreviewService.ts b/packages/backend/src/server/web/UrlPreviewService.ts index baef8fa993..802b404ce6 100644 --- a/packages/backend/src/server/web/UrlPreviewService.ts +++ b/packages/backend/src/server/web/UrlPreviewService.ts @@ -63,9 +63,8 @@ export class UrlPreviewService { this.logger.info(meta.summalyProxy ? `(Proxy) Getting preview of ${url}@${lang} ...` : `Getting preview of ${url}@${lang} ...`); - try { - const summary = meta.summalyProxy ? await this.httpRequestService.getJson(`${meta.summalyProxy}?${query({ + const summary = meta.summalyProxy ? await this.httpRequestService.getJson>(`${meta.summalyProxy}?${query({ url: url, lang: lang ?? 'ja-JP', })}`) : await summaly.default(url, { diff --git a/packages/frontend/package.json b/packages/frontend/package.json index f92ee44224..2506e8e9d3 100644 --- a/packages/frontend/package.json +++ b/packages/frontend/package.json @@ -11,14 +11,14 @@ "@rollup/plugin-alias": "4.0.2", "@rollup/plugin-json": "6.0.0", "@rollup/pluginutils": "5.0.2", - "@syuilo/aiscript": "0.11.1", + "@syuilo/aiscript": "0.12.0", "@tabler/icons": "^1.118.0", "@vitejs/plugin-vue": "4.0.0", "@vue/compiler-sfc": "3.2.45", "autobind-decorator": "2.4.0", "autosize": "5.0.2", "blurhash": "2.0.4", - "broadcast-channel": "4.18.1", + "broadcast-channel": "4.19.1", "browser-image-resizer": "git+https://github.com/misskey-dev/browser-image-resizer#v2.2.1-misskey.3", "chart.js": "4.1.1", "chartjs-adapter-date-fns": "3.0.0", @@ -34,7 +34,7 @@ "idb-keyval": "6.2.0", "insert-text-at-cursor": "0.3.0", "is-file-animated": "1.0.2", - "json5": "2.2.2", + "json5": "2.2.3", "katex": "0.15.6", "matter-js": "0.18.0", "mfm-js": "0.23.0", @@ -57,7 +57,7 @@ "throttle-debounce": "5.0.0", "tinycolor2": "1.5.1", "tsc-alias": "1.8.2", - "tsconfig-paths": "4.1.1", + "tsconfig-paths": "4.1.2", "twemoji-parser": "14.0.0", "typescript": "4.9.4", "uuid": "9.0.0", @@ -76,22 +76,22 @@ "@types/matter-js": "0.18.2", "@types/punycode": "2.1.0", "@types/sanitize-html": "^2.8.0", - "@types/seedrandom": "3.0.3", + "@types/seedrandom": "3.0.4", "@types/throttle-debounce": "5.0.0", "@types/tinycolor2": "1.4.3", "@types/uuid": "9.0.0", "@types/websocket": "1.0.5", - "@types/ws": "8.5.3", + "@types/ws": "8.5.4", "@typescript-eslint/eslint-plugin": "5.47.1", "@typescript-eslint/parser": "5.47.1", "@vue/runtime-core": "3.2.45", "cross-env": "7.0.3", "cypress": "12.2.0", - "eslint": "8.30.0", + "eslint": "8.31.0", "eslint-plugin-import": "2.26.0", "eslint-plugin-vue": "9.8.0", "start-server-and-test": "1.15.2", "vue-eslint-parser": "^9.1.0", - "vue-tsc": "^1.0.18" + "vue-tsc": "^1.0.19" } } diff --git a/packages/frontend/src/components/MkAbuseReportWindow.vue b/packages/frontend/src/components/MkAbuseReportWindow.vue index 039f77c859..ab90ed357a 100644 --- a/packages/frontend/src/components/MkAbuseReportWindow.vue +++ b/packages/frontend/src/components/MkAbuseReportWindow.vue @@ -23,7 +23,7 @@ + + diff --git a/packages/frontend/src/components/MkContextMenu.vue b/packages/frontend/src/components/MkContextMenu.vue index 6470f8b972..9d9abc5d09 100644 --- a/packages/frontend/src/components/MkContextMenu.vue +++ b/packages/frontend/src/components/MkContextMenu.vue @@ -22,7 +22,7 @@ const emit = defineEmits<{ (ev: 'closed'): void; }>(); -let rootEl = $ref(); +let rootEl = $shallowRef(); let zIndex = $ref(os.claimZIndex('high')); diff --git a/packages/frontend/src/components/MkCropperDialog.vue b/packages/frontend/src/components/MkCropperDialog.vue index ae18160dea..f00fef12f1 100644 --- a/packages/frontend/src/components/MkCropperDialog.vue +++ b/packages/frontend/src/components/MkCropperDialog.vue @@ -50,8 +50,8 @@ const props = defineProps<{ }>(); const imgUrl = getProxiedImageUrl(props.file.url); -let dialogEl = $ref>(); -let imgEl = $ref(); +let dialogEl = $shallowRef>(); +let imgEl = $shallowRef(); let cropper: Cropper | null = null; let loading = $ref(true); diff --git a/packages/frontend/src/components/MkDialog.vue b/packages/frontend/src/components/MkDialog.vue index 374ecd8abf..18c9f9203c 100644 --- a/packages/frontend/src/components/MkDialog.vue +++ b/packages/frontend/src/components/MkDialog.vue @@ -39,7 +39,7 @@ diff --git a/packages/frontend/src/components/global/MkStickyContainer.vue b/packages/frontend/src/components/global/MkStickyContainer.vue index 44f4f065a6..a3fee91a36 100644 --- a/packages/frontend/src/components/global/MkStickyContainer.vue +++ b/packages/frontend/src/components/global/MkStickyContainer.vue @@ -18,9 +18,9 @@ const CURRENT_STICKY_TOP = 'CURRENT_STICKY_TOP'; - - diff --git a/packages/frontend/src/pages/admin/overview.active-users.vue b/packages/frontend/src/pages/admin/overview.active-users.vue index ea8c74f3a2..14b09f34e9 100644 --- a/packages/frontend/src/pages/admin/overview.active-users.vue +++ b/packages/frontend/src/pages/admin/overview.active-users.vue @@ -9,52 +9,20 @@ diff --git a/packages/frontend/src/pages/user/activity.pv.vue b/packages/frontend/src/pages/user/activity.pv.vue new file mode 100644 index 0000000000..7715b66673 --- /dev/null +++ b/packages/frontend/src/pages/user/activity.pv.vue @@ -0,0 +1,188 @@ + + + + + diff --git a/packages/frontend/src/pages/user/activity.vue b/packages/frontend/src/pages/user/activity.vue new file mode 100644 index 0000000000..f9dce3a9e8 --- /dev/null +++ b/packages/frontend/src/pages/user/activity.vue @@ -0,0 +1,29 @@ + + + + + diff --git a/packages/frontend/src/pages/user/home.vue b/packages/frontend/src/pages/user/home.vue index 45334a998b..4a92074d93 100644 --- a/packages/frontend/src/pages/user/home.vue +++ b/packages/frontend/src/pages/user/home.vue @@ -203,12 +203,12 @@ onUnmounted(() => { > .main { position: relative; - overflow: hidden; + overflow: clip; > .banner-container { position: relative; height: 250px; - overflow: hidden; + overflow: clip; background-size: cover; background-position: center; diff --git a/packages/frontend/src/pages/user/index.activity.vue b/packages/frontend/src/pages/user/index.activity.vue index 523072d2e6..0cc1524663 100644 --- a/packages/frontend/src/pages/user/index.activity.vue +++ b/packages/frontend/src/pages/user/index.activity.vue @@ -33,10 +33,16 @@ let chartSrc = $ref('per-user-notes'); function showMenu(ev: MouseEvent) { os.popupMenu([{ text: i18n.ts.notes, - active: true, + active: chartSrc === 'per-user-notes', action: () => { chartSrc = 'per-user-notes'; }, + }, { + text: i18n.ts.numberOfProfileView, + active: chartSrc === 'per-user-pv', + action: () => { + chartSrc = 'per-user-pv'; + }, }, /*, { text: i18n.ts.following, action: () => { diff --git a/packages/frontend/src/pages/user/index.vue b/packages/frontend/src/pages/user/index.vue index f40cd0b8d6..7abae1ea30 100644 --- a/packages/frontend/src/pages/user/index.vue +++ b/packages/frontend/src/pages/user/index.vue @@ -5,6 +5,7 @@
+ @@ -32,6 +33,7 @@ import { i18n } from '@/i18n'; import { $i } from '@/account'; const XHome = defineAsyncComponent(() => import('./home.vue')); +const XActivity = defineAsyncComponent(() => import('./activity.vue')); const XReactions = defineAsyncComponent(() => import('./reactions.vue')); const XClips = defineAsyncComponent(() => import('./clips.vue')); const XPages = defineAsyncComponent(() => import('./pages.vue')); @@ -70,6 +72,10 @@ const headerTabs = $computed(() => user ? [{ key: 'home', title: i18n.ts.overview, icon: 'ti ti-home', +}, { + key: 'activity', + title: i18n.ts.activity, + icon: 'ti ti-chart-line', }, ...($i && ($i.id === user.id)) || user.publicReactions ? [{ key: 'reactions', title: i18n.ts.reaction, @@ -86,7 +92,7 @@ const headerTabs = $computed(() => user ? [{ key: 'gallery', title: i18n.ts.gallery, icon: 'ti ti-icons', -}] : null); +}] : []); definePageMetadata(computed(() => user ? { icon: 'ti ti-user', diff --git a/packages/frontend/src/pizzax.ts b/packages/frontend/src/pizzax.ts index 642e1f4f7f..7ff09f75fb 100644 --- a/packages/frontend/src/pizzax.ts +++ b/packages/frontend/src/pizzax.ts @@ -23,8 +23,13 @@ export class Storage { // TODO: これが実装されたらreadonlyにしたい: https://github.com/microsoft/TypeScript/issues/37487 public readonly state: { [K in keyof T]: T[K]['default'] }; public readonly reactiveState: { [K in keyof T]: Ref }; + public readonly ready: Promise; + private markAsReady: () => void = () => {}; constructor(key: string, def: T) { + this.ready = new Promise((res) => { + this.markAsReady = res; + }); this.key = key; this.keyForLocalStorage = 'pizzax::' + key; this.def = def; @@ -72,6 +77,7 @@ export class Storage { } } localStorage.setItem(this.keyForLocalStorage + '::cache::' + $i.id, JSON.stringify(cache)); + this.markAsReady(); }); }, 1); // streamingのuser storage updateイベントを監視して更新 @@ -87,6 +93,8 @@ export class Storage { localStorage.setItem(this.keyForLocalStorage + '::cache::' + $i.id, JSON.stringify(cache)); } }); + } else { + this.markAsReady(); } } diff --git a/packages/frontend/src/plugin.ts b/packages/frontend/src/plugin.ts index 3a00cd0455..c19fe2b08d 100644 --- a/packages/frontend/src/plugin.ts +++ b/packages/frontend/src/plugin.ts @@ -1,16 +1,17 @@ -import { AiScript, utils, values } from '@syuilo/aiscript'; -import { deserialize } from '@syuilo/aiscript/built/serializer'; -import { jsToVal } from '@syuilo/aiscript/built/interpreter/util'; +import { Interpreter, Parser, utils, values } from '@syuilo/aiscript'; import { createAiScriptEnv } from '@/scripts/aiscript/api'; import { inputText } from '@/os'; import { noteActions, notePostInterruptors, noteViewInterruptors, postFormActions, userActions } from '@/store'; -const pluginContexts = new Map(); +const parser = new Parser(); +const pluginContexts = new Map(); export function install(plugin) { + // 後方互換性のため + if (plugin.src == null) return; console.info('Plugin installed:', plugin.name, 'v' + plugin.version); - const aiscript = new AiScript(createPluginEnv({ + const aiscript = new Interpreter(createPluginEnv({ plugin: plugin, storageKey: 'plugins:' + plugin.id, }), { @@ -32,13 +33,13 @@ export function install(plugin) { initPlugin({ plugin, aiscript }); - aiscript.exec(deserialize(plugin.ast)); + aiscript.exec(parser.parse(plugin.src)); } function createPluginEnv(opts) { const config = new Map(); for (const [k, v] of Object.entries(opts.plugin.config || {})) { - config.set(k, jsToVal(typeof opts.plugin.configData[k] !== 'undefined' ? opts.plugin.configData[k] : v.default)); + config.set(k, utils.jsToVal(typeof opts.plugin.configData[k] !== 'undefined' ? opts.plugin.configData[k] : v.default)); } return { diff --git a/packages/frontend/src/router.ts b/packages/frontend/src/router.ts index 111b15e0a6..9001f0f37f 100644 --- a/packages/frontend/src/router.ts +++ b/packages/frontend/src/router.ts @@ -207,6 +207,7 @@ export const routes = [{ }, { path: '/explore', component: page(() => import('./pages/explore.vue')), + hash: 'initialTab', }, { path: '/search', component: page(() => import('./pages/search.vue')), diff --git a/packages/frontend/src/scripts/chart-legend.ts b/packages/frontend/src/scripts/chart-legend.ts new file mode 100644 index 0000000000..6a5370cc87 --- /dev/null +++ b/packages/frontend/src/scripts/chart-legend.ts @@ -0,0 +1,12 @@ +import { Plugin } from 'chart.js'; +import MkChartLegend from '@/components/MkChartLegend.vue'; + +export const chartLegend = (legend: InstanceType) => ({ + id: 'htmlLegend', + afterUpdate(chart, args, options) { + // Reuse the built-in legendItems generator + const items = chart.options.plugins.legend.labels.generateLabels(chart); + + legend.update(chart, items); + }, +}) as Plugin; diff --git a/packages/frontend/src/scripts/chart-vline.ts b/packages/frontend/src/scripts/chart-vline.ts index 8e3c4436b2..f321443834 100644 --- a/packages/frontend/src/scripts/chart-vline.ts +++ b/packages/frontend/src/scripts/chart-vline.ts @@ -1,10 +1,12 @@ +import { Plugin } from 'chart.js'; + export const chartVLine = (vLineColor: string) => ({ id: 'vLine', beforeDraw(chart, args, options) { if (chart.tooltip?._active?.length) { - const activePoint = chart.tooltip._active[0]; const ctx = chart.ctx; - const x = activePoint.element.x; + const xs = chart.tooltip._active.map(a => a.element.x); + const x = xs.reduce((a, b) => a + b, 0) / xs.length; const topY = chart.scales.y.top; const bottomY = chart.scales.y.bottom; @@ -18,4 +20,4 @@ export const chartVLine = (vLineColor: string) => ({ ctx.restore(); } }, -}); +}) as Plugin; diff --git a/packages/frontend/src/scripts/hpml/evaluator.ts b/packages/frontend/src/scripts/hpml/evaluator.ts index 196b3142a1..d4090ea15c 100644 --- a/packages/frontend/src/scripts/hpml/evaluator.ts +++ b/packages/frontend/src/scripts/hpml/evaluator.ts @@ -1,13 +1,11 @@ import autobind from 'autobind-decorator'; -import { PageVar, envVarsDef, Fn, HpmlScope, HpmlError } from '.'; -import { version } from '@/config'; -import { AiScript, utils, values } from '@syuilo/aiscript'; -import { createAiScriptEnv } from '../aiscript/api'; +import { markRaw, ref, Ref, unref } from 'vue'; import { collectPageVars } from '../collect-page-vars'; import { initHpmlLib, initAiLib } from './lib'; -import * as os from '@/os'; -import { markRaw, ref, Ref, unref } from 'vue'; import { Expr, isLiteralValue, Variable } from './expr'; +import { PageVar, envVarsDef, Fn, HpmlScope, HpmlError } from '.'; +import { version } from '@/config'; +import * as os from '@/os'; /** * Hpml evaluator @@ -16,7 +14,6 @@ export class Hpml { private variables: Variable[]; private pageVars: PageVar[]; private envVars: Record; - public aiscript?: AiScript; public pageVarUpdatedCallback?: values.VFn; public canvases: Record = {}; public vars: Ref> = ref({}); @@ -24,7 +21,6 @@ export class Hpml { private opts: { randomSeed: string; visitor?: any; url?: string; - enableAiScript: boolean; }; constructor(page: Hpml['page'], opts: Hpml['opts']) { @@ -33,31 +29,6 @@ export class Hpml { this.pageVars = collectPageVars(this.page.content); this.opts = opts; - if (this.opts.enableAiScript) { - this.aiscript = markRaw(new AiScript({ ...createAiScriptEnv({ - storageKey: 'pages:' + this.page.id, - }), ...initAiLib(this) }, { - in: (q) => { - return new Promise(ok => { - os.inputText({ - title: q, - }).then(({ canceled, result: a }) => { - ok(a); - }); - }); - }, - out: (value) => { - console.log(value); - }, - log: (type, params) => { - }, - })); - - this.aiscript.scope.opts.onUpdated = (name, value) => { - this.eval(); - }; - } - const date = new Date(); this.envVars = { @@ -74,7 +45,7 @@ export class Hpml { IS_CAT: opts.visitor ? opts.visitor.isCat : false, SEED: opts.randomSeed ? opts.randomSeed : '', YMD: `${date.getFullYear()}/${date.getMonth() + 1}/${date.getDate()}`, - AISCRIPT_DISABLED: !this.opts.enableAiScript, + AISCRIPT_DISABLED: true, NULL: null, }; @@ -99,13 +70,6 @@ export class Hpml { }); } - @autobind - public callAiScript(fn: string) { - try { - if (this.aiscript) this.aiscript.execFn(this.aiscript.scope.get(fn), []); - } catch (err) {} - } - @autobind public registerCanvas(id: string, canvas: any) { this.canvases[id] = canvas; @@ -116,9 +80,6 @@ export class Hpml { const pageVar = this.pageVars.find(v => v.name === name); if (pageVar !== undefined) { pageVar.value = value; - if (this.pageVarUpdatedCallback) { - if (this.aiscript) this.aiscript.execFn(this.pageVarUpdatedCallback, [values.STR(name), utils.jsToVal(value)]); - } } else { throw new HpmlError(`No such page var '${name}'`); } @@ -180,18 +141,6 @@ export class Hpml { return scope.getState(expr.value); } - if (expr.type === 'aiScriptVar') { - if (this.aiscript) { - try { - return utils.valToJs(this.aiscript.scope.get(expr.value)); - } catch (err) { - return null; - } - } else { - return null; - } - } - // Define user function if (expr.type === 'fn') { return { diff --git a/packages/frontend/src/scripts/hpml/lib.ts b/packages/frontend/src/scripts/hpml/lib.ts index b684876a7f..02d663b31b 100644 --- a/packages/frontend/src/scripts/hpml/lib.ts +++ b/packages/frontend/src/scripts/hpml/lib.ts @@ -1,9 +1,8 @@ import tinycolor from 'tinycolor2'; -import { Hpml } from './evaluator'; -import { values, utils } from '@syuilo/aiscript'; -import { Fn, HpmlScope } from '.'; -import { Expr } from './expr'; import seedrandom from 'seedrandom'; +import { Hpml } from './evaluator'; +import { Expr } from './expr'; +import { Fn, HpmlScope } from '.'; /* TODO: https://www.chartjs.org/docs/latest/configuration/canvas-background.html#color // https://stackoverflow.com/questions/38493564/chart-area-background-color-chartjs diff --git a/packages/frontend/src/scripts/init-chart.ts b/packages/frontend/src/scripts/init-chart.ts new file mode 100644 index 0000000000..fc18869009 --- /dev/null +++ b/packages/frontend/src/scripts/init-chart.ts @@ -0,0 +1,53 @@ +import { + Chart, + ArcElement, + LineElement, + BarElement, + PointElement, + BarController, + LineController, + DoughnutController, + CategoryScale, + LinearScale, + TimeScale, + Legend, + Title, + Tooltip, + SubTitle, + Filler, +} from 'chart.js'; +import gradient from 'chartjs-plugin-gradient'; +import zoomPlugin from 'chartjs-plugin-zoom'; +import { MatrixController, MatrixElement } from 'chartjs-chart-matrix'; +import { defaultStore } from '@/store'; +import 'chartjs-adapter-date-fns'; + +export function initChart() { + Chart.register( + ArcElement, + LineElement, + BarElement, + PointElement, + BarController, + LineController, + DoughnutController, + CategoryScale, + LinearScale, + TimeScale, + Legend, + Title, + Tooltip, + SubTitle, + Filler, + MatrixController, MatrixElement, + zoomPlugin, + gradient, + ); + + // フォントカラー + Chart.defaults.color = getComputedStyle(document.documentElement).getPropertyValue('--fg'); + + Chart.defaults.borderColor = defaultStore.state.darkMode ? 'rgba(255, 255, 255, 0.1)' : 'rgba(0, 0, 0, 0.1)'; + + Chart.defaults.animation = false; +} diff --git a/packages/frontend/src/scripts/touch.ts b/packages/frontend/src/scripts/touch.ts index 5251bc2e27..4afd0e5dd4 100644 --- a/packages/frontend/src/scripts/touch.ts +++ b/packages/frontend/src/scripts/touch.ts @@ -1,23 +1,13 @@ +import { deviceKind } from '@/scripts/device-kind'; + const isTouchSupported = 'maxTouchPoints' in navigator && navigator.maxTouchPoints > 0; -export let isTouchUsing = false; +export let isTouchUsing = deviceKind === 'tablet' || deviceKind === 'smartphone'; -export let isScreenTouching = false; - -if (isTouchSupported) { +if (isTouchSupported && !isTouchUsing) { window.addEventListener('touchstart', () => { // maxTouchPointsなどでの判定だけだと、「タッチ機能付きディスプレイを使っているがマウスでしか操作しない」場合にも // タッチで使っていると判定されてしまうため、実際に一度でもタッチされたらtrueにする isTouchUsing = true; - - isScreenTouching = true; - }, { passive: true }); - - window.addEventListener('touchend', () => { - // 子要素のtouchstartイベントでstopPropagation()が呼ばれると親要素に伝搬されずタッチされたと判定されないため、 - // touchendイベントでもtouchstartイベントと同様にtrueにする - isTouchUsing = true; - - isScreenTouching = false; }, { passive: true }); } diff --git a/packages/frontend/src/store.ts b/packages/frontend/src/store.ts index 8e3e6b36da..4b1f47c2bc 100644 --- a/packages/frontend/src/store.ts +++ b/packages/frontend/src/store.ts @@ -170,10 +170,6 @@ export const defaultStore = markRaw(new Storage('base', { where: 'device', default: false, }, - disablePagesScript: { - where: 'device', - default: false, - }, emojiStyle: { where: 'device', default: 'twemoji', // twemoji / fluentEmoji / native diff --git a/packages/frontend/src/style.scss b/packages/frontend/src/style.scss index 51ea1bd343..f4369884f7 100644 --- a/packages/frontend/src/style.scss +++ b/packages/frontend/src/style.scss @@ -246,32 +246,6 @@ hr { } } -._inputs { - display: flex; - margin: 32px 0; - - &:first-child { - margin-top: 8px; - } - - &:last-child { - margin-bottom: 8px; - } - - > * { - flex: 1; - margin: 0 !important; - - &:not(:first-child) { - margin-left: 8px !important; - } - - &:not(:last-child) { - margin-right: 8px !important; - } - } -} - ._panel { background: var(--panel); border-radius: var(--radius); @@ -472,14 +446,6 @@ hr { } } -._keyValue { - display: flex; - - > * { - flex: 1; - } -} - ._link { color: var(--link); } diff --git a/packages/frontend/src/ui/_common_/statusbar-federation.vue b/packages/frontend/src/ui/_common_/statusbar-federation.vue index 7c3de32ac9..70d683d755 100644 --- a/packages/frontend/src/ui/_common_/statusbar-federation.vue +++ b/packages/frontend/src/ui/_common_/statusbar-federation.vue @@ -44,7 +44,7 @@ let key = $ref(0); const tick = () => { os.api('federation/instances', { - sort: '+lastCommunicatedAt', + sort: '+latestRequestReceivedAt', limit: 30, }).then(res => { instances.value = res; diff --git a/packages/frontend/src/ui/classic.vue b/packages/frontend/src/ui/classic.vue index 532829ee2e..280e69e7dd 100644 --- a/packages/frontend/src/ui/classic.vue +++ b/packages/frontend/src/ui/classic.vue @@ -7,7 +7,7 @@
- +
@@ -17,7 +17,7 @@
- +
@@ -64,7 +64,7 @@ let fullView = $ref(false); let globalHeaderHeight = $ref(0); const wallpaper = localStorage.getItem('wallpaper') != null; const showMenuOnTop = $computed(() => defaultStore.state.menuDisplay === 'top'); -let live2d = $ref(); +let live2d = $shallowRef(); let widgetsLeft = $ref(); let widgetsRight = $ref(); @@ -76,7 +76,7 @@ provideMetadataReceiver((info) => { } }); provide('shouldHeaderThin', showMenuOnTop); -provide('shouldSpacerMin', true); +provide('forceSpacerMin', true); function attachSticky(el) { const sticky = new StickySidebar(el, defaultStore.state.menuDisplay === 'top' ? 0 : 16, defaultStore.state.menuDisplay === 'top' ? 60 : 0); // TODO: ヘッダーの高さを60pxと決め打ちしているのを直す @@ -129,18 +129,20 @@ if (window.innerWidth < 1024) { document.documentElement.style.overflowY = 'scroll'; -if (defaultStore.state.widgets.length === 0) { - defaultStore.set('widgets', [{ - name: 'calendar', - id: 'a', place: null, data: {}, - }, { - name: 'notifications', - id: 'b', place: null, data: {}, - }, { - name: 'trends', - id: 'c', place: null, data: {}, - }]); -} +defaultStore.ready.then(() => { + if (defaultStore.state.widgets.length === 0) { + defaultStore.set('widgets', [{ + name: 'calendar', + id: 'a', place: null, data: {}, + }, { + name: 'notifications', + id: 'b', place: null, data: {}, + }, { + name: 'trends', + id: 'c', place: null, data: {}, + }]); + } +}); onMounted(() => { window.addEventListener('resize', () => { diff --git a/packages/frontend/src/ui/deck.vue b/packages/frontend/src/ui/deck.vue index 6efb2ce290..b0306ab832 100644 --- a/packages/frontend/src/ui/deck.vue +++ b/packages/frontend/src/ui/deck.vue @@ -125,7 +125,7 @@ function showSettings() { os.pageWindow('/settings/deck'); } -let columnsEl = $ref(); +let columnsEl = $shallowRef(); const addColumn = async (ev) => { const columns = [ diff --git a/packages/frontend/src/ui/deck/antenna-column.vue b/packages/frontend/src/ui/deck/antenna-column.vue index ba14530662..53d744676c 100644 --- a/packages/frontend/src/ui/deck/antenna-column.vue +++ b/packages/frontend/src/ui/deck/antenna-column.vue @@ -26,7 +26,7 @@ const emit = defineEmits<{ (ev: 'parent-focus', direction: 'up' | 'down' | 'left' | 'right'): void; }>(); -let timeline = $ref>(); +let timeline = $shallowRef>(); onMounted(() => { if (props.column.antennaId == null) { diff --git a/packages/frontend/src/ui/deck/column.vue b/packages/frontend/src/ui/deck/column.vue index 4e6677fe4a..775bdf6c1e 100644 --- a/packages/frontend/src/ui/deck/column.vue +++ b/packages/frontend/src/ui/deck/column.vue @@ -40,7 +40,7 @@ import { MenuItem } from '@/types/menu'; provide('shouldHeaderThin', true); provide('shouldOmitHeaderTitle', true); -provide('shouldSpacerMin', true); +provide('forceSpacerMin', true); const props = withDefaults(defineProps<{ column: Column; @@ -59,7 +59,7 @@ const emit = defineEmits<{ (ev: 'change-active-state', v: boolean): void; }>(); -let body = $ref(); +let body = $shallowRef(); let dragging = $ref(false); watch($$(dragging), v => os.deckGlobalEvents.emit(v ? 'column.dragStart' : 'column.dragEnd')); @@ -251,7 +251,7 @@ function onDrop(ev) { --deckColumnHeaderHeight: 40px; height: 100%; - overflow: hidden; + overflow: clip; contain: strict; &.draghover { diff --git a/packages/frontend/src/ui/deck/list-column.vue b/packages/frontend/src/ui/deck/list-column.vue index d9f3f7b4e7..e31446ebb2 100644 --- a/packages/frontend/src/ui/deck/list-column.vue +++ b/packages/frontend/src/ui/deck/list-column.vue @@ -26,7 +26,7 @@ const emit = defineEmits<{ (ev: 'parent-focus', direction: 'up' | 'down' | 'left' | 'right'): void; }>(); -let timeline = $ref>(); +let timeline = $shallowRef>(); if (props.column.listId == null) { setList(); diff --git a/packages/frontend/src/ui/universal.vue b/packages/frontend/src/ui/universal.vue index 7922ab0086..9e1fee5b6b 100644 --- a/packages/frontend/src/ui/universal.vue +++ b/packages/frontend/src/ui/universal.vue @@ -19,11 +19,11 @@
- - - - - + + + + +
@@ -86,7 +86,7 @@ window.addEventListener('resize', () => { }); let pageMetadata = $ref>(); -const widgetsEl = $ref(); +const widgetsEl = $shallowRef(); const widgetsShowing = $ref(false); provide('router', mainRouter); @@ -113,18 +113,20 @@ mainRouter.on('change', () => { document.documentElement.style.overflowY = 'scroll'; -if (defaultStore.state.widgets.length === 0) { - defaultStore.set('widgets', [{ - name: 'calendar', - id: 'a', place: 'right', data: {}, - }, { - name: 'notifications', - id: 'b', place: 'right', data: {}, - }, { - name: 'trends', - id: 'c', place: 'right', data: {}, - }]); -} +defaultStore.ready.then(() => { + if (defaultStore.state.widgets.length === 0) { + defaultStore.set('widgets', [{ + name: 'calendar', + id: 'a', place: 'right', data: {}, + }, { + name: 'notifications', + id: 'b', place: 'right', data: {}, + }, { + name: 'trends', + id: 'c', place: 'right', data: {}, + }]); + } +}); onMounted(() => { if (!isDesktop.value) { @@ -285,8 +287,10 @@ const wallpaper = localStorage.getItem('wallpaper') != null; z-index: 1000; bottom: 0; left: 0; - padding: 16px 16px calc(env(safe-area-inset-bottom, 0px) + 16px) 16px; - display: flex; + padding: 12px 12px max(12px, env(safe-area-inset-bottom, 0px)) 12px; + display: grid; + grid-template-columns: 1fr 1fr 1fr 1fr 1fr; + grid-gap: 8px; width: 100%; box-sizing: border-box; -webkit-backdrop-filter: var(--blur, blur(32px)); @@ -296,26 +300,15 @@ const wallpaper = localStorage.getItem('wallpaper') != null; > .button { position: relative; - flex: 1; padding: 0; + aspect-ratio: 1; + width: 100%; + max-width: 60px; margin: auto; - height: 64px; - border-radius: 8px; + border-radius: 100%; background: var(--panel); color: var(--fg); - &:not(:last-child) { - margin-right: 12px; - } - - @media (max-width: 400px) { - height: 60px; - - &:not(:last-child) { - margin-right: 8px; - } - } - &:hover { background: var(--X2); } @@ -329,25 +322,22 @@ const wallpaper = localStorage.getItem('wallpaper') != null; animation: blink 1s infinite; } - &:first-child { - margin-left: 0; - } - - &:last-child { - margin-right: 0; - } - - > * { - font-size: 20px; + > .icon { + font-size: 18px; } &:disabled { cursor: default; - > * { + > .icon { opacity: 0.5; } } + + &.post { + background: linear-gradient(90deg, var(--buttonGradateA), var(--buttonGradateB)); + color: var(--fgOnAccent); + } } } diff --git a/packages/frontend/src/ui/universal.widgets.vue b/packages/frontend/src/ui/universal.widgets.vue index 35de23ebfa..d4210f6988 100644 --- a/packages/frontend/src/ui/universal.widgets.vue +++ b/packages/frontend/src/ui/universal.widgets.vue @@ -31,7 +31,7 @@ const emit = defineEmits<{ (ev: 'mounted', el?: Element): void; }>(); -let rootEl = $ref(); +let rootEl = $shallowRef(); const widgets = $computed(() => { if (props.place === null) return defaultStore.reactiveState.widgets.value; diff --git a/packages/frontend/src/widgets/aichan.vue b/packages/frontend/src/widgets/aichan.vue index 828490fd9c..ab5b375ae4 100644 --- a/packages/frontend/src/widgets/aichan.vue +++ b/packages/frontend/src/widgets/aichan.vue @@ -5,7 +5,7 @@