From d4fb201d05fc4e0da85f21eece7cc6baf30b6d05 Mon Sep 17 00:00:00 2001 From: syuilo Date: Wed, 25 Jan 2023 12:00:04 +0900 Subject: [PATCH] =?UTF-8?q?fix(server):=20node-fetch=E3=81=8A=E3=82=88?= =?UTF-8?q?=E3=81=B3got=E3=82=92=E4=BD=BF=E3=81=86=E4=BB=A5=E5=89=8D?= =?UTF-8?q?=E3=81=AE=E5=AE=9F=E8=A3=85=E3=81=AB=E6=88=BB=E3=81=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit see #9710 --- packages/backend/package.json | 3 +- packages/backend/src/core/CaptchaService.ts | 17 +- packages/backend/src/core/DownloadService.ts | 76 ++-- .../src/core/FetchInstanceMetadataService.ts | 5 +- .../backend/src/core/HttpRequestService.ts | 381 ++++-------------- packages/backend/src/core/S3Service.ts | 2 +- .../src/core/activitypub/ApRequestService.ts | 35 +- .../src/core/activitypub/ApResolverService.ts | 10 +- .../core/activitypub/LdSignatureService.ts | 19 +- .../WebhookDeliverProcessorService.ts | 39 +- .../src/server/api/endpoints/fetch-rss.ts | 17 +- .../server/api/endpoints/notes/translate.ts | 22 +- pnpm-lock.yaml | 4 +- 13 files changed, 203 insertions(+), 427 deletions(-) diff --git a/packages/backend/package.json b/packages/backend/package.json index 6cac949ea7..586b95f2e5 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -63,7 +63,7 @@ "file-type": "18.2.0", "fluent-ffmpeg": "2.1.2", "form-data": "^4.0.0", - "got": "12.5.3", + "got": "^12.5.3", "hpagent": "1.2.0", "ioredis": "4.28.5", "ip-cidr": "3.0.11", @@ -119,7 +119,6 @@ "typeorm": "0.3.11", "typescript": "4.9.4", "ulid": "2.3.0", - "undici": "^5.16.0", "unzipper": "0.10.11", "uuid": "9.0.0", "vary": "1.1.2", diff --git a/packages/backend/src/core/CaptchaService.ts b/packages/backend/src/core/CaptchaService.ts index 1e98914052..c8428a26b0 100644 --- a/packages/backend/src/core/CaptchaService.ts +++ b/packages/backend/src/core/CaptchaService.ts @@ -21,18 +21,13 @@ export class CaptchaService { response, }); - const res = await this.httpRequestService.fetch( - url, - { - method: 'POST', - body: params, + const res = await this.httpRequestService.send(url, { + method: 'POST', + body: JSON.stringify(params), + headers: { + 'Content-Type': 'application/json', }, - { - noOkError: true, - } - ).catch(err => { - throw `${err.message ?? err}`; - }); + }, { throwErrorWhenResponseNotOk: false }); if (!res.ok) { throw `${res.status}`; diff --git a/packages/backend/src/core/DownloadService.ts b/packages/backend/src/core/DownloadService.ts index c5b2bcaef4..a971e06fd8 100644 --- a/packages/backend/src/core/DownloadService.ts +++ b/packages/backend/src/core/DownloadService.ts @@ -5,10 +5,10 @@ import { Inject, Injectable } from '@nestjs/common'; import IPCIDR from 'ip-cidr'; import PrivateIp from 'private-ip'; import chalk from 'chalk'; -import { buildConnector } from 'undici'; +import got, * as Got from 'got'; import { DI } from '@/di-symbols.js'; import type { Config } from '@/config.js'; -import { HttpRequestService, UndiciFetcher } from '@/core/HttpRequestService.js'; +import { HttpRequestService } from '@/core/HttpRequestService.js'; import { createTemp } from '@/misc/create-temp.js'; import { StatusError } from '@/misc/status-error.js'; import { LoggerService } from '@/core/LoggerService.js'; @@ -20,7 +20,6 @@ import { bindThis } from '@/decorators.js'; @Injectable() export class DownloadService { private logger: Logger; - private undiciFetcher: UndiciFetcher; constructor( @Inject(DI.config) @@ -30,21 +29,6 @@ export class DownloadService { private loggerService: LoggerService, ) { this.logger = this.loggerService.getLogger('download'); - - this.undiciFetcher = this.httpRequestService.createFetcher({ - connect: process.env.NODE_ENV === 'development' ? - this.httpRequestService.clientDefaults.connect - : - this.httpRequestService.getConnectorWithIpCheck( - buildConnector({ - ...this.httpRequestService.clientDefaults.connect, - }), - (ip) => !this.isPrivateIp(ip), - ), - bodyTimeout: 30 * 1000, - }, { - connect: this.httpRequestService.clientDefaults.connect, - }, this.logger); } @bindThis @@ -55,14 +39,60 @@ export class DownloadService { const operationTimeout = 60 * 1000; const maxSize = this.config.maxFileSize ?? 262144000; - const response = await this.undiciFetcher.fetch(url); + const req = got.stream(url, { + headers: { + 'User-Agent': this.config.userAgent, + }, + timeout: { + lookup: timeout, + connect: timeout, + secureConnect: timeout, + socket: timeout, // read timeout + response: timeout, + send: timeout, + request: operationTimeout, // whole operation timeout + }, + agent: { + http: this.httpRequestService.httpAgent, + https: this.httpRequestService.httpsAgent, + }, + http2: false, // default + retry: { + limit: 0, + }, + }).on('response', (res: Got.Response) => { + if ((process.env.NODE_ENV === 'production' || process.env.NODE_ENV === 'test') && !this.config.proxy && res.ip) { + if (this.isPrivateIp(res.ip)) { + this.logger.warn(`Blocked address: ${res.ip}`); + req.destroy(); + } + } - if (response.body === null) { - throw new StatusError('No body', 400, 'No body'); + const contentLength = res.headers['content-length']; + if (contentLength != null) { + const size = Number(contentLength); + if (size > maxSize) { + this.logger.warn(`maxSize exceeded (${size} > ${maxSize}) on response`); + req.destroy(); + } + } + }).on('downloadProgress', (progress: Got.Progress) => { + if (progress.transferred > maxSize) { + this.logger.warn(`maxSize exceeded (${progress.transferred} > ${maxSize}) on downloadProgress`); + req.destroy(); + } + }); + + try { + await pipeline(req, fs.createWriteStream(path)); + } catch (e) { + if (e instanceof Got.HTTPError) { + throw new StatusError(`${e.response.statusCode} ${e.response.statusMessage}`, e.response.statusCode, e.response.statusMessage); + } else { + throw e; + } } - await pipeline(stream.Readable.fromWeb(response.body), fs.createWriteStream(path)); - this.logger.succ(`Download finished: ${chalk.cyan(url)}`); } diff --git a/packages/backend/src/core/FetchInstanceMetadataService.ts b/packages/backend/src/core/FetchInstanceMetadataService.ts index cb9d099a22..35f30deeb4 100644 --- a/packages/backend/src/core/FetchInstanceMetadataService.ts +++ b/packages/backend/src/core/FetchInstanceMetadataService.ts @@ -2,6 +2,7 @@ import { URL } from 'node:url'; import { Inject, Injectable } from '@nestjs/common'; import { JSDOM } from 'jsdom'; import tinycolor from 'tinycolor2'; +import fetch from 'node-fetch'; import type { Instance } from '@/models/entities/Instance.js'; import type { InstancesRepository } from '@/models/index.js'; import { AppLockService } from '@/core/AppLockService.js'; @@ -190,7 +191,9 @@ export class FetchInstanceMetadataService { const faviconUrl = url + '/favicon.ico'; - const favicon = await this.httpRequestService.fetch(faviconUrl, {}, { noOkError: true }); + const favicon = await this.httpRequestService.send(faviconUrl, { + method: 'HEAD', + }, { throwErrorWhenResponseNotOk: false }); if (favicon.ok) { return faviconUrl; diff --git a/packages/backend/src/core/HttpRequestService.ts b/packages/backend/src/core/HttpRequestService.ts index cd859d0023..baf74acfa6 100644 --- a/packages/backend/src/core/HttpRequestService.ts +++ b/packages/backend/src/core/HttpRequestService.ts @@ -1,284 +1,67 @@ import * as http from 'node:http'; import * as https from 'node:https'; -import { LookupFunction } from 'node:net'; import CacheableLookup from 'cacheable-lookup'; +import fetch from 'node-fetch'; import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent'; import { Inject, Injectable } from '@nestjs/common'; -import * as undici from 'undici'; import { DI } from '@/di-symbols.js'; import type { Config } from '@/config.js'; import { StatusError } from '@/misc/status-error.js'; import { bindThis } from '@/decorators.js'; -import { LoggerService } from '@/core/LoggerService.js'; -import type Logger from '@/logger.js'; - -// true to allow, false to deny -export type IpChecker = (ip: string) => boolean; - -/* - * Child class to create and save Agent for fetch. - * You should construct this when you want - * to change timeout, size limit, socket connect function, etc. - */ -export class UndiciFetcher { - /** - * Get http non-proxy agent (undici) - */ - public nonProxiedAgent: undici.Agent; - - /** - * Get http proxy or non-proxy agent (undici) - */ - public agent: undici.ProxyAgent | undici.Agent; - - private proxyBypassHosts: string[]; - private userAgent: string | undefined; - - private logger: Logger | undefined; - - constructor( - args: { - agentOptions: undici.Agent.Options; - proxy?: { - uri: string; - options?: undici.Agent.Options; // Override of agentOptions - }, - proxyBypassHosts?: string[]; - userAgent?: string; - }, - logger?: Logger, - ) { - this.logger = logger; - this.logger?.debug('UndiciFetcher constructor', args); - - this.proxyBypassHosts = args.proxyBypassHosts ?? []; - this.userAgent = args.userAgent; - - this.nonProxiedAgent = new undici.Agent({ - ...args.agentOptions, - connect: (process.env.NODE_ENV !== 'production' && typeof args.agentOptions.connect !== 'function') - ? (options, cb) => { - // Custom connector for debug - undici.buildConnector(args.agentOptions.connect as undici.buildConnector.BuildOptions)(options, (err, socket) => { - this.logger?.debug('Socket connector called', socket); - if (err) { - this.logger?.debug('Socket error', err); - cb(new Error(`Error while socket connecting\n${err}`), null); - return; - } - this.logger?.debug(`Socket connected: port ${socket.localPort} => remote ${socket.remoteAddress}`); - cb(null, socket); - }); - } : args.agentOptions.connect, - }); - - this.agent = args.proxy - ? new undici.ProxyAgent({ - ...args.agentOptions, - ...args.proxy.options, - - uri: args.proxy.uri, - - connect: (process.env.NODE_ENV !== 'production' && typeof (args.proxy.options?.connect ?? args.agentOptions.connect) !== 'function') - ? (options, cb) => { - // Custom connector for debug - undici.buildConnector((args.proxy?.options?.connect ?? args.agentOptions.connect) as undici.buildConnector.BuildOptions)(options, (err, socket) => { - this.logger?.debug('Socket connector called (secure)', socket); - if (err) { - this.logger?.debug('Socket error', err); - cb(new Error(`Error while socket connecting\n${err}`), null); - return; - } - this.logger?.debug(`Socket connected (secure): port ${socket.localPort} => remote ${socket.remoteAddress}`); - cb(null, socket); - }); - } : (args.proxy.options?.connect ?? args.agentOptions.connect), - }) - : this.nonProxiedAgent; - } - - /** - * Get agent by URL - * @param url URL - * @param bypassProxy Allways bypass proxy - */ - @bindThis - public getAgentByUrl(url: URL, bypassProxy = false): undici.Agent | undici.ProxyAgent { - if (bypassProxy || this.proxyBypassHosts.includes(url.hostname)) { - return this.nonProxiedAgent; - } else { - return this.agent; - } - } - - @bindThis - public async fetch( - url: string | URL, - options: undici.RequestInit = {}, - privateOptions: { noOkError?: boolean; bypassProxy?: boolean; } = { noOkError: false, bypassProxy: false }, - ): Promise { - const res = await undici.fetch(url, { - dispatcher: this.getAgentByUrl(new URL(url), privateOptions.bypassProxy), - ...options, - headers: { - 'User-Agent': this.userAgent ?? '', - ...(options.headers ?? {}), - }, - }).catch((err) => { - this.logger?.error(`fetch error to ${typeof url === 'string' ? url : url.href}`, err); - throw new StatusError('Resource Unreachable', 500, 'Resource Unreachable'); - }); - if (!res.ok && !privateOptions.noOkError) { - throw new StatusError(`${res.status} ${res.statusText}`, res.status, res.statusText); - } - return res; - } - - @bindThis - public async request( - url: string | URL, - options: { dispatcher?: undici.Dispatcher } & Omit & Partial> = {}, - privateOptions: { noOkError?: boolean; bypassProxy?: boolean; } = { noOkError: false, bypassProxy: false }, - ): Promise { - const res = await undici.request(url, { - dispatcher: this.getAgentByUrl(new URL(url), privateOptions.bypassProxy), - ...options, - headers: { - 'user-agent': this.userAgent ?? '', - ...(options.headers ?? {}), - }, - }).catch((err) => { - this.logger?.error(`fetch error to ${typeof url === 'string' ? url : url.href}`, err); - throw new StatusError('Resource Unreachable', 500, 'Resource Unreachable'); - }); - - if (res.statusCode >= 400) { - throw new StatusError(`${res.statusCode}`, res.statusCode, ''); - } - - return res; - } - - @bindThis - public async getJson(url: string, accept = 'application/json, */*', headers?: Record): Promise { - const { body } = await this.request( - url, - { - headers: Object.assign({ - Accept: accept, - }, headers ?? {}), - }, - ); - - return await body.json() as T; - } - - @bindThis - public async getHtml(url: string, accept = 'text/html, */*', headers?: Record): Promise { - const { body } = await this.request( - url, - { - headers: Object.assign({ - Accept: accept, - }, headers ?? {}), - }, - ); - - return await body.text(); - } -} +import type { Response } from 'node-fetch'; +import type { URL } from 'node:url'; @Injectable() export class HttpRequestService { - public defaultFetcher: UndiciFetcher; - public fetch: UndiciFetcher['fetch']; - public request: UndiciFetcher['request']; - public getHtml: UndiciFetcher['getHtml']; - public defaultJsonFetcher: UndiciFetcher; - public getJson: UndiciFetcher['getJson']; - - //#region for old http/https, only used in S3Service - // http non-proxy agent + /** + * Get http non-proxy agent + */ private http: http.Agent; - // https non-proxy agent + /** + * Get https non-proxy agent + */ private https: https.Agent; - // http proxy or non-proxy agent + /** + * Get http proxy or non-proxy agent + */ public httpAgent: http.Agent; - // https proxy or non-proxy agent + /** + * Get https proxy or non-proxy agent + */ public httpsAgent: https.Agent; - //#endregion - - public readonly dnsCache: CacheableLookup; - public readonly clientDefaults: undici.Agent.Options; - private maxSockets: number; - - private logger: Logger; constructor( @Inject(DI.config) private config: Config, - private loggerService: LoggerService, ) { - this.logger = this.loggerService.getLogger('http-request'); - - this.dnsCache = new CacheableLookup({ + const cache = new CacheableLookup({ maxTtl: 3600, // 1hours errorTtl: 30, // 30secs lookup: false, // nativeのdns.lookupにfallbackしない }); - - this.clientDefaults = { - keepAliveTimeout: 30 * 1000, - keepAliveMaxTimeout: 10 * 60 * 1000, - keepAliveTimeoutThreshold: 1 * 1000, - strictContentLength: true, - headersTimeout: 10 * 1000, - bodyTimeout: 10 * 1000, - maxHeaderSize: 16364, // default - maxResponseSize: 10 * 1024 * 1024, - maxRedirections: 3, - connect: { - timeout: 10 * 1000, // コネクションが確立するまでのタイムアウト - maxCachedSessions: 300, // TLSセッションのキャッシュ数 https://github.com/nodejs/undici/blob/v5.14.0/lib/core/connect.js#L80 - lookup: this.dnsCache.lookup as LookupFunction, // https://github.com/nodejs/undici/blob/v5.14.0/lib/core/connect.js#L98 - }, - }; - - this.maxSockets = Math.max(64, ((this.config.deliverJobConcurrency ?? 128) / (this.config.clusterLimit ?? 1))); - - this.defaultFetcher = this.createFetcher({}, {}, this.logger); - - this.fetch = this.defaultFetcher.fetch; - this.request = this.defaultFetcher.request; - this.getHtml = this.defaultFetcher.getHtml; - - this.defaultJsonFetcher = this.createFetcher({ - maxResponseSize: 1024 * 256, - }, {}, this.logger); - - this.getJson = this.defaultJsonFetcher.getJson; - - //#region for old http/https, only used in S3Service + this.http = new http.Agent({ keepAlive: true, keepAliveMsecs: 30 * 1000, - lookup: this.dnsCache.lookup, + lookup: cache.lookup, } as http.AgentOptions); this.https = new https.Agent({ keepAlive: true, keepAliveMsecs: 30 * 1000, - lookup: this.dnsCache.lookup, + lookup: cache.lookup, } as https.AgentOptions); - + + const maxSockets = Math.max(256, config.deliverJobConcurrency ?? 128); + this.httpAgent = config.proxy ? new HttpProxyAgent({ keepAlive: true, keepAliveMsecs: 30 * 1000, - maxSockets: this.maxSockets, + maxSockets, maxFreeSockets: 256, scheduling: 'lifo', proxy: config.proxy, @@ -289,47 +72,21 @@ export class HttpRequestService { ? new HttpsProxyAgent({ keepAlive: true, keepAliveMsecs: 30 * 1000, - maxSockets: this.maxSockets, + maxSockets, maxFreeSockets: 256, scheduling: 'lifo', proxy: config.proxy, }) : this.https; - //#endregion - } - - @bindThis - private getStandardUndiciFetcherOption(opts: undici.Agent.Options = {}, proxyOpts: undici.Agent.Options = {}) { - return { - agentOptions: { - ...this.clientDefaults, - ...opts, - }, - ...(this.config.proxy ? { - proxy: { - uri: this.config.proxy, - options: { - connections: this.maxSockets, - ...proxyOpts, - }, - }, - } : {}), - userAgent: this.config.userAgent, - }; - } - - @bindThis - public createFetcher(opts: undici.Agent.Options = {}, proxyOpts: undici.Agent.Options = {}, logger: Logger) { - return new UndiciFetcher(this.getStandardUndiciFetcherOption(opts, proxyOpts), logger); } /** - * Get http agent by URL + * Get agent by URL * @param url URL * @param bypassProxy Allways bypass proxy */ @bindThis - public getHttpAgentByUrl(url: URL, bypassProxy = false): http.Agent | https.Agent { + public getAgentByUrl(url: URL, bypassProxy = false): http.Agent | https.Agent { if (bypassProxy || (this.config.proxyBypassHosts || []).includes(url.hostname)) { return url.protocol === 'http:' ? this.http : this.https; } else { @@ -337,37 +94,67 @@ export class HttpRequestService { } } - /** - * check ip - */ @bindThis - public getConnectorWithIpCheck(connector: undici.buildConnector.connector, checkIp: IpChecker): undici.buildConnector.connectorAsync { - return (options, cb) => { - connector(options, (err, socket) => { - this.logger.debug('Socket connector (with ip checker) called', socket); - if (err) { - this.logger.error('Socket error', err); - cb(new Error(`Error while socket connecting\n${err}`), null); - return; - } + public async getJson(url: string, accept = 'application/json, */*', headers?: Record): Promise { + const res = await this.send(url, { + method: 'GET', + headers: Object.assign({ + 'User-Agent': this.config.userAgent, + Accept: accept, + }, headers ?? {}), + timeout: 5000, + size: 1024 * 256, + }); - if (socket.remoteAddress == undefined) { - this.logger.error('Socket error: remoteAddress is undefined'); - cb(new Error('remoteAddress is undefined (maybe socket destroyed)'), null); - return; - } + return await res.json(); + } - // allow - if (checkIp(socket.remoteAddress)) { - this.logger.debug(`Socket connected (ip ok): ${socket.localPort} => ${socket.remoteAddress}`); - cb(null, socket); - return; - } + @bindThis + public async getHtml(url: string, accept = 'text/html, */*', headers?: Record): Promise { + const res = await this.send(url, { + method: 'GET', + headers: Object.assign({ + 'User-Agent': this.config.userAgent, + Accept: accept, + }, headers ?? {}), + timeout: 5000, + }); - this.logger.error('IP is not allowed', socket); - cb(new StatusError('IP is not allowed', 403, 'IP is not allowed'), null); - socket.destroy(); - }); - }; + return await res.text(); + } + + @bindThis + public async send(url: string, args: { + method?: string, + body?: string, + headers?: Record, + timeout?: number, + size?: number, + } = {}, extra: { + throwErrorWhenResponseNotOk: boolean; + } = { + throwErrorWhenResponseNotOk: true, + }): Promise { + const timeout = args.timeout ?? 5000; + + const controller = new AbortController(); + setTimeout(() => { + controller.abort(); + }, timeout); + + const res = await fetch(url, { + method: args.method ?? 'GET', + headers: args.headers, + body: args.body, + size: args.size ?? 10 * 1024 * 1024, + agent: (url) => this.getAgentByUrl(url), + signal: controller.signal, + }); + + if (!res.ok && extra.throwErrorWhenResponseNotOk) { + throw new StatusError(`${res.status} ${res.statusText}`, res.status, res.statusText); + } + + return res; } } diff --git a/packages/backend/src/core/S3Service.ts b/packages/backend/src/core/S3Service.ts index 930188ce6e..0ce69aaa74 100644 --- a/packages/backend/src/core/S3Service.ts +++ b/packages/backend/src/core/S3Service.ts @@ -33,7 +33,7 @@ export class S3Service { ? false : meta.objectStorageS3ForcePathStyle, httpOptions: { - agent: this.httpRequestService.getHttpAgentByUrl(new URL(u), !meta.objectStorageUseProxy), + agent: this.httpRequestService.getAgentByUrl(new URL(u), !meta.objectStorageUseProxy), }, }); } diff --git a/packages/backend/src/core/activitypub/ApRequestService.ts b/packages/backend/src/core/activitypub/ApRequestService.ts index db87475c4c..b949f34841 100644 --- a/packages/backend/src/core/activitypub/ApRequestService.ts +++ b/packages/backend/src/core/activitypub/ApRequestService.ts @@ -5,16 +5,14 @@ import { DI } from '@/di-symbols.js'; import type { Config } from '@/config.js'; import type { User } from '@/models/entities/User.js'; import { UserKeypairStoreService } from '@/core/UserKeypairStoreService.js'; -import { HttpRequestService, UndiciFetcher } from '@/core/HttpRequestService.js'; +import { HttpRequestService } from '@/core/HttpRequestService.js'; import { LoggerService } from '@/core/LoggerService.js'; import { bindThis } from '@/decorators.js'; import type Logger from '@/logger.js'; -import type { Dispatcher } from 'undici'; -import { DevNull } from '@/misc/dev-null.js'; type Request = { url: string; - method: Dispatcher.HttpMethod; + method: string; headers: Record; }; @@ -32,7 +30,6 @@ type PrivateKey = { @Injectable() export class ApRequestService { - private undiciFetcher: UndiciFetcher; private logger: Logger; constructor( @@ -44,9 +41,6 @@ export class ApRequestService { private loggerService: LoggerService, ) { this.logger = this.loggerService.getLogger('ap-request'); // なぜか TypeError: Cannot read properties of undefined (reading 'getLogger') と言われる - this.undiciFetcher = this.httpRequestService.createFetcher({ - maxRedirections: 0, - }, {}, this.logger); } @bindThis @@ -165,15 +159,11 @@ export class ApRequestService { }, }); - const response = await this.undiciFetcher.request( - url, - { - method: req.request.method, - headers: req.request.headers, - body, - }, - ); - response.body.pipe(new DevNull()); + await this.httpRequestService.send(url, { + method: req.request.method, + headers: req.request.headers, + body, + }); } /** @@ -195,13 +185,10 @@ export class ApRequestService { }, }); - const res = await this.httpRequestService.fetch( - url, - { - method: req.request.method, - headers: req.request.headers, - }, - ); + const res = await this.httpRequestService.send(url, { + method: req.request.method, + headers: req.request.headers, + }); return await res.json(); } diff --git a/packages/backend/src/core/activitypub/ApResolverService.ts b/packages/backend/src/core/activitypub/ApResolverService.ts index ca7760af81..8ead29a5f2 100644 --- a/packages/backend/src/core/activitypub/ApResolverService.ts +++ b/packages/backend/src/core/activitypub/ApResolverService.ts @@ -4,7 +4,7 @@ import { InstanceActorService } from '@/core/InstanceActorService.js'; import type { NotesRepository, PollsRepository, NoteReactionsRepository, UsersRepository } from '@/models/index.js'; import type { Config } from '@/config.js'; import { MetaService } from '@/core/MetaService.js'; -import { HttpRequestService, UndiciFetcher } from '@/core/HttpRequestService.js'; +import { HttpRequestService } from '@/core/HttpRequestService.js'; import { DI } from '@/di-symbols.js'; import { UtilityService } from '@/core/UtilityService.js'; import { bindThis } from '@/decorators.js'; @@ -19,7 +19,6 @@ import type { IObject, ICollection, IOrderedCollection } from './type.js'; export class Resolver { private history: Set; private user?: ILocalUser; - private undiciFetcher: UndiciFetcher; private logger: Logger; constructor( @@ -39,10 +38,7 @@ export class Resolver { private recursionLimit = 100, ) { this.history = new Set(); - this.logger = this.loggerService?.getLogger('ap-resolve'); // なぜか TypeError: Cannot read properties of undefined (reading 'getLogger') と言われる - this.undiciFetcher = this.httpRequestService.createFetcher({ - maxRedirections: 0, - }, {}, this.logger); + this.logger = this.loggerService.getLogger('ap-resolve'); // なぜか TypeError: Cannot read properties of undefined (reading 'getLogger') と言われる } @bindThis @@ -106,7 +102,7 @@ export class Resolver { const object = (this.user ? await this.apRequestService.signedGet(value, this.user) as IObject - : await this.undiciFetcher.getJson(value, 'application/activity+json, application/ld+json')); + : await this.httpRequestService.getJson(value, 'application/activity+json, application/ld+json')) as IObject; if (object == null || ( Array.isArray(object['@context']) ? diff --git a/packages/backend/src/core/activitypub/LdSignatureService.ts b/packages/backend/src/core/activitypub/LdSignatureService.ts index 4e4b7dce2d..5a1e01574d 100644 --- a/packages/backend/src/core/activitypub/LdSignatureService.ts +++ b/packages/backend/src/core/activitypub/LdSignatureService.ts @@ -9,7 +9,7 @@ import { CONTEXTS } from './misc/contexts.js'; class LdSignature { public debug = false; public preLoad = true; - public loderTimeout = 10 * 1000; + public loderTimeout = 5000; constructor( private httpRequestService: HttpRequestService, @@ -115,19 +115,12 @@ class LdSignature { @bindThis private async fetchDocument(url: string) { - const json = await this.httpRequestService.fetch( - url, - { - headers: { - Accept: 'application/ld+json, application/json', - }, - // TODO - //timeout: this.loderTimeout, + const json = await this.httpRequestService.send(url, { + headers: { + Accept: 'application/ld+json, application/json', }, - { - noOkError: true, - } - ).then(res => { + timeout: this.loderTimeout, + }, { throwErrorWhenResponseNotOk: false }).then(res => { if (!res.ok) { throw `${res.status} ${res.statusText}`; } else { diff --git a/packages/backend/src/queue/processors/WebhookDeliverProcessorService.ts b/packages/backend/src/queue/processors/WebhookDeliverProcessorService.ts index f0543a5ed1..57210b25d2 100644 --- a/packages/backend/src/queue/processors/WebhookDeliverProcessorService.ts +++ b/packages/backend/src/queue/processors/WebhookDeliverProcessorService.ts @@ -6,10 +6,10 @@ import type { Config } from '@/config.js'; import type Logger from '@/logger.js'; import { HttpRequestService } from '@/core/HttpRequestService.js'; import { StatusError } from '@/misc/status-error.js'; +import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; import type Bull from 'bull'; import type { WebhookDeliverJobData } from '../types.js'; -import { bindThis } from '@/decorators.js'; @Injectable() export class WebhookDeliverProcessorService { @@ -33,26 +33,23 @@ export class WebhookDeliverProcessorService { try { this.logger.debug(`delivering ${job.data.webhookId}`); - const res = await this.httpRequestService.fetch( - job.data.to, - { - method: 'POST', - headers: { - 'User-Agent': 'Misskey-Hooks', - 'X-Misskey-Host': this.config.host, - 'X-Misskey-Hook-Id': job.data.webhookId, - 'X-Misskey-Hook-Secret': job.data.secret, - }, - body: JSON.stringify({ - hookId: job.data.webhookId, - userId: job.data.userId, - eventId: job.data.eventId, - createdAt: job.data.createdAt, - type: job.data.type, - body: job.data.content, - }), - } - ); + const res = await this.httpRequestService.send(job.data.to, { + method: 'POST', + headers: { + 'User-Agent': 'Misskey-Hooks', + 'X-Misskey-Host': this.config.host, + 'X-Misskey-Hook-Id': job.data.webhookId, + 'X-Misskey-Hook-Secret': job.data.secret, + }, + body: JSON.stringify({ + hookId: job.data.webhookId, + userId: job.data.userId, + eventId: job.data.eventId, + createdAt: job.data.createdAt, + type: job.data.type, + body: job.data.content, + }), + }); this.webhooksRepository.update({ id: job.data.webhookId }, { latestSentAt: new Date(), diff --git a/packages/backend/src/server/api/endpoints/fetch-rss.ts b/packages/backend/src/server/api/endpoints/fetch-rss.ts index ae6a87513d..5849d3111f 100644 --- a/packages/backend/src/server/api/endpoints/fetch-rss.ts +++ b/packages/backend/src/server/api/endpoints/fetch-rss.ts @@ -33,16 +33,13 @@ export default class extends Endpoint { private httpRequestService: HttpRequestService, ) { super(meta, paramDef, async (ps, me) => { - const res = await this.httpRequestService.fetch( - ps.url, - { - method: 'GET', - headers: { - Accept: 'application/rss+xml, */*', - }, - // timeout: 5000, - } - ); + const res = await this.httpRequestService.send(ps.url, { + method: 'GET', + headers: { + Accept: 'application/rss+xml, */*', + }, + timeout: 5000, + }); const text = await res.text(); diff --git a/packages/backend/src/server/api/endpoints/notes/translate.ts b/packages/backend/src/server/api/endpoints/notes/translate.ts index ab19771676..66655234a1 100644 --- a/packages/backend/src/server/api/endpoints/notes/translate.ts +++ b/packages/backend/src/server/api/endpoints/notes/translate.ts @@ -7,8 +7,8 @@ import { DI } from '@/di-symbols.js'; import { NoteEntityService } from '@/core/entities/NoteEntityService.js'; import { MetaService } from '@/core/MetaService.js'; import { HttpRequestService } from '@/core/HttpRequestService.js'; -import { ApiError } from '../../error.js'; import { GetterService } from '@/server/api/GetterService.js'; +import { ApiError } from '../../error.js'; export const meta = { tags: ['notes'], @@ -83,20 +83,14 @@ export default class extends Endpoint { const endpoint = instance.deeplIsPro ? 'https://api.deepl.com/v2/translate' : 'https://api-free.deepl.com/v2/translate'; - const res = await this.httpRequestService.fetch( - endpoint, - { - method: 'POST', - headers: { - 'Content-Type': 'application/x-www-form-urlencoded', - Accept: 'application/json, */*', - }, - body: params.toString(), + const res = await this.httpRequestService.send(endpoint, { + method: 'POST', + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + Accept: 'application/json, */*', }, - { - noOkError: false, - } - ); + body: params.toString(), + }); const json = (await res.json()) as { translations: { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3fdaf1449a..bfe98f9a84 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -142,7 +142,7 @@ importers: file-type: 18.2.0 fluent-ffmpeg: 2.1.2 form-data: ^4.0.0 - got: 12.5.3 + got: ^12.5.3 hpagent: 1.2.0 ioredis: 4.28.5 ip-cidr: 3.0.11 @@ -201,7 +201,6 @@ importers: typeorm: 0.3.11 typescript: 4.9.4 ulid: 2.3.0 - undici: ^5.16.0 unzipper: 0.10.11 uuid: 9.0.0 vary: 1.1.2 @@ -306,7 +305,6 @@ importers: typeorm: 0.3.11_ioredis@4.28.5+pg@8.8.0 typescript: 4.9.4 ulid: 2.3.0 - undici: 5.16.0 unzipper: 0.10.11 uuid: 9.0.0 vary: 1.1.2