diff --git a/packages/backend/src/core/CaptchaService.ts b/packages/backend/src/core/CaptchaService.ts index bcbd47736f..e2dc68a8b0 100644 --- a/packages/backend/src/core/CaptchaService.ts +++ b/packages/backend/src/core/CaptchaService.ts @@ -4,7 +4,6 @@ import type { UsersRepository } from '@/models/index.js'; import type { Config } from '@/config.js'; import { HttpRequestService } from '@/core/HttpRequestService.js'; import { bindThis } from '@/decorators.js'; -import { fetch } from 'undici'; type CaptchaResponse = { success: boolean; @@ -28,16 +27,20 @@ export class CaptchaService { response, }); - const res = await fetch(url, { - method: 'POST', - body: params, - headers: { - 'User-Agent': this.config.userAgent, + const res = await this.httpRequestService.fetch( + url, + { + method: 'POST', + body: params, + headers: { + 'User-Agent': this.config.userAgent, + }, }, - // TODO - //timeout: 10 * 1000, - dispatcher: this.httpRequestService.getAgentByUrl(new URL(url)), - }).catch(err => { + { + noOkError: true, + bypassProxy: true, + } + ).catch(err => { throw `${err.message ?? err}`; }); diff --git a/packages/backend/src/core/DownloadService.ts b/packages/backend/src/core/DownloadService.ts index af31d6f72a..65ff5b0897 100644 --- a/packages/backend/src/core/DownloadService.ts +++ b/packages/backend/src/core/DownloadService.ts @@ -8,11 +8,12 @@ import got, * as Got from 'got'; import chalk from 'chalk'; import { DI } from '@/di-symbols.js'; import type { Config } from '@/config.js'; -import { HttpRequestService } from '@/core/HttpRequestService.js'; +import { HttpRequestService, UndiciFetcher } from '@/core/HttpRequestService.js'; import { createTemp } from '@/misc/create-temp.js'; import { StatusError } from '@/misc/status-error.js'; import { LoggerService } from '@/core/LoggerService.js'; import type Logger from '@/logger.js'; +import { buildConnector } from 'undici'; const pipeline = util.promisify(stream.pipeline); import { bindThis } from '@/decorators.js'; @@ -20,6 +21,7 @@ import { bindThis } from '@/decorators.js'; @Injectable() export class DownloadService { private logger: Logger; + private undiciFetcher: UndiciFetcher:; constructor( @Inject(DI.config) @@ -29,6 +31,16 @@ export class DownloadService { private loggerService: LoggerService, ) { this.logger = this.loggerService.getLogger('download'); + + this.undiciFetcher = new UndiciFetcher(this.httpRequestService.getStandardUndiciFetcherConstructorOption({ + connect: this.httpRequestService.getConnectorWithIpCheck( + buildConnector({ + ...this.httpRequestService.clientDefaults.connect, + }), + this.isPrivateIp + ), + bodyTimeout: 30 * 1000, + })); } @bindThis @@ -38,22 +50,16 @@ export class DownloadService { const timeout = 30 * 1000; const operationTimeout = 60 * 1000; const maxSize = this.config.maxFileSize ?? 262144000; - - const response = await this.httpRequestService.fetch({ - method: 'GET', + + const response = await this.undiciFetcher.fetch( url, - headers: { - 'User-Agent': this.config.userAgent, - }, - timeout, - size: maxSize, - ipCheckers: - (process.env.NODE_ENV === 'production' || process.env.NODE_ENV === 'test') && - !this.config.proxy ? - [{ type: 'black', fn: this.isPrivateIp }] : - undefined, - // http2: false, // default - }); + { + method: 'GET', + headers: { + 'User-Agent': this.config.userAgent, + }, + } + ); if (response.body === null) { throw new StatusError('No body', 400, 'No body'); diff --git a/packages/backend/src/core/FetchInstanceMetadataService.ts b/packages/backend/src/core/FetchInstanceMetadataService.ts index bf42c6529a..17a363b415 100644 --- a/packages/backend/src/core/FetchInstanceMetadataService.ts +++ b/packages/backend/src/core/FetchInstanceMetadataService.ts @@ -190,9 +190,7 @@ export class FetchInstanceMetadataService { const faviconUrl = url + '/favicon.ico'; - const favicon = await this.httpRequestService.fetch({ - url: faviconUrl, - }); + const favicon = await this.httpRequestService.fetch(faviconUrl, {}, { noOkError: true, bypassProxy: false }); if (favicon.ok) { return faviconUrl; diff --git a/packages/backend/src/core/HttpRequestService.ts b/packages/backend/src/core/HttpRequestService.ts index c94135d84a..c8ccbaf35f 100644 --- a/packages/backend/src/core/HttpRequestService.ts +++ b/packages/backend/src/core/HttpRequestService.ts @@ -9,23 +9,123 @@ import { StatusError } from '@/misc/status-error.js'; import { bindThis } from '@/decorators.js'; import * as undici from 'undici'; import { LookupFunction } from 'node:net'; -import { TransformStream } from 'node:stream/web'; -import * as dns from 'node:dns'; +// true to allow, false to deny export type IpChecker = (ip: string) => boolean; -@Injectable() -export class HttpRequestService { +/* + * Child class to create and save Agent for fetch. + * You should construct this when you want + * to change timeout, size limit, socket connect function, etc. + */ +export class UndiciFetcher { /** * Get http non-proxy agent (undici) */ - private nonProxiedAgent: undici.Agent; - + public nonProxiedAgent: undici.Agent; + /** * Get http proxy or non-proxy agent (undici) */ public agent: undici.ProxyAgent | undici.Agent; + private proxyBypassHosts: string[]; + private userAgent: string | undefined; + + constructor(args: { + agentOptions: undici.Agent.Options; + proxy?: { + uri: string; + options?: undici.Agent.Options; // Override of agentOptions + }, + proxyBypassHosts?: string[]; + userAgent?: string; + }) { + this.proxyBypassHosts = args.proxyBypassHosts ?? []; + this.userAgent = args.userAgent; + + this.nonProxiedAgent = new undici.Agent({ + ...args.agentOptions, + }); + this.agent = args.proxy + ? new undici.ProxyAgent({ + ...args.agentOptions, + ...args.proxy.options, + + uri: args.proxy.uri, + }) + : this.nonProxiedAgent; + } + + /** + * Get agent by URL + * @param url URL + * @param bypassProxy Allways bypass proxy + */ + @bindThis + public getAgentByUrl(url: URL, bypassProxy = false): undici.Agent | undici.ProxyAgent { + if (bypassProxy || this.proxyBypassHosts.includes(url.hostname)) { + return this.nonProxiedAgent; + } else { + return this.agent; + } + } + + @bindThis + public async fetch( + url: string | URL, + options: undici.RequestInit = {}, + privateOptions: { noOkError?: boolean; bypassProxy?: boolean; } = { noOkError: false, bypassProxy: false } + ): Promise { + const res = await undici.fetch(url, { + dispatcher: this.getAgentByUrl(new URL(url), privateOptions.bypassProxy), + ...options, + }) + if (!res.ok && !privateOptions.noOkError) { + throw new StatusError(`${res.status} ${res.statusText}`, res.status, res.statusText); + } + return res; + } + + @bindThis + public async getJson(url: string, accept = 'application/json, */*', headers?: Record): Promise { + const res = await this.fetch( + url, + { + headers: Object.assign({ + 'User-Agent': this.userAgent, + Accept: accept, + }, headers ?? {}), + } + ); + + return await res.json() as T; + } + + @bindThis + public async getHtml(url: string, accept = 'text/html, */*', headers?: Record): Promise { + const res = await this.fetch( + url, + { + headers: Object.assign({ + 'User-Agent': this.userAgent, + Accept: accept, + }, headers ?? {}), + } + ); + + return await res.text(); + } +} + +@Injectable() +export class HttpRequestService { + public defaultFetcher: UndiciFetcher; + public fetch: UndiciFetcher['fetch']; + public getHtml: UndiciFetcher['getHtml']; + public defaultJsonFetcher: UndiciFetcher; + public getJson: UndiciFetcher['getJson']; + //#region for old http/https, only used in S3Service // http non-proxy agent private http: http.Agent; @@ -42,6 +142,7 @@ export class HttpRequestService { public readonly dnsCache: CacheableLookup; public readonly clientDefaults: undici.Agent.Options; + private maxSockets: number; constructor( @Inject(DI.config) @@ -54,20 +155,35 @@ export class HttpRequestService { }); this.clientDefaults = { - keepAliveTimeout: 4 * 1000, + keepAliveTimeout: 30 * 1000, keepAliveMaxTimeout: 10 * 60 * 1000, keepAliveTimeoutThreshold: 1 * 1000, strictContentLength: true, + headersTimeout: 10 * 1000, + bodyTimeout: 10 * 1000, + maxHeaderSize: 16364, // default + maxResponseSize: 10 * 1024 * 1024, connect: { - maxCachedSessions: 100, // TLSセッションのキャッシュ数 https://github.com/nodejs/undici/blob/v5.14.0/lib/core/connect.js#L80 + timeout: 10 * 1000, // コネクションが確立するまでのタイムアウト + maxCachedSessions: 300, // TLSセッションのキャッシュ数 https://github.com/nodejs/undici/blob/v5.14.0/lib/core/connect.js#L80 lookup: this.dnsCache.lookup as LookupFunction, // https://github.com/nodejs/undici/blob/v5.14.0/lib/core/connect.js#L98 }, } - this.nonProxiedAgent = new undici.Agent({ - ...this.clientDefaults, - }); + this.maxSockets = Math.max(256, this.config.deliverJobConcurrency ?? 128); + this.defaultFetcher = new UndiciFetcher(this.getStandardUndiciFetcherConstructorOption()); + + this.fetch = this.defaultFetcher.fetch; + this.getHtml = this.defaultFetcher.getHtml; + + this.defaultJsonFetcher = new UndiciFetcher(this.getStandardUndiciFetcherConstructorOption({ + maxResponseSize: 1024 * 256, + })); + + this.getJson = this.defaultJsonFetcher.getJson; + + //#region for old http/https, only used in S3Service this.http = new http.Agent({ keepAlive: true, keepAliveMsecs: 30 * 1000, @@ -80,23 +196,11 @@ export class HttpRequestService { lookup: this.dnsCache.lookup, } as https.AgentOptions); - const maxSockets = Math.max(256, config.deliverJobConcurrency ?? 128); - - this.agent = config.proxy - ? new undici.ProxyAgent({ - ...this.clientDefaults, - connections: maxSockets, - - uri: config.proxy, - }) - : this.nonProxiedAgent; - - //#region for old http/https, only used in S3Service this.httpAgent = config.proxy ? new HttpProxyAgent({ keepAlive: true, keepAliveMsecs: 30 * 1000, - maxSockets, + maxSockets: this.maxSockets, maxFreeSockets: 256, scheduling: 'lifo', proxy: config.proxy, @@ -107,26 +211,35 @@ export class HttpRequestService { ? new HttpsProxyAgent({ keepAlive: true, keepAliveMsecs: 30 * 1000, - maxSockets, + maxSockets: this.maxSockets, maxFreeSockets: 256, scheduling: 'lifo', proxy: config.proxy, }) : this.https; + //#endregion } - //#endregion /** - * Get agent by URL + * Get http agent by URL * @param url URL * @param bypassProxy Allways bypass proxy */ @bindThis - public getAgentByUrl(url: URL, bypassProxy = false): undici.Agent | undici.ProxyAgent { - if (bypassProxy || (this.config.proxyBypassHosts || []).includes(url.hostname)) { - return this.nonProxiedAgent; - } else { - return this.agent; + public getStandardUndiciFetcherConstructorOption(opts: undici.Agent.Options = {}) { + return { + agentOptions: { + ...this.clientDefaults, + ...opts, + }, + ...(this.config.proxy ? { + proxy: { + uri: this.config.proxy, + options: { + connections: this.maxSockets, + } + } + } : {}), } } @@ -143,141 +256,33 @@ export class HttpRequestService { return url.protocol === 'http:' ? this.httpAgent : this.httpsAgent; } } + /** * check ip */ @bindThis - public checkIp(url: URL, fn: IpChecker): Promise { - const lookup = this.dnsCache.lookup as LookupFunction || dns.lookup; - - return new Promise((resolve, reject) => { - lookup(url.hostname, {}, (err, ip) => { + public getConnectorWithIpCheck(connector: undici.buildConnector.connector, checkIp: IpChecker): undici.buildConnector.connectorAsync { + return (options, cb) => { + connector(options, (err, socket) => { if (err) { - resolve(false); - } else { - resolve(fn(ip)); + cb(err, null); + return; } + + if (socket.remoteAddress == undefined) { + cb(new Error('remoteAddress is undefined (maybe socket destroyed)'), null); + return; + } + + // allow + if (checkIp(socket.remoteAddress)) { + cb(null, socket); + return; + } + + socket.destroy(); + cb(new StatusError('IP is not allowed', 403, 'IP is not allowed'), null); }); - }); - } - - @bindThis - public async getJson(url: string, accept = 'application/json, */*', timeout = 10000, headers?: Record): Promise { - const res = await this.fetch({ - url, - headers: Object.assign({ - 'User-Agent': this.config.userAgent, - Accept: accept, - }, headers ?? {}), - timeout, - size: 1024 * 256, - }); - - return await res.json() as T; - } - - @bindThis - public async getHtml(url: string, accept = 'text/html, */*', timeout = 10000, headers?: Record): Promise { - const res = await this.fetch({ - url, - headers: Object.assign({ - 'User-Agent': this.config.userAgent, - Accept: accept, - }, headers ?? {}), - timeout, - }); - - return await res.text(); - } - - @bindThis - public async fetch(args: { - url: string, - method?: string, - body?: string, - headers?: Record, - timeout?: number, - size?: number, - redirect?: RequestRedirect | undefined, - dispatcher?: undici.Dispatcher, - ipCheckers?: { - type: 'black' | 'white', - fn: IpChecker, - }[], - noOkError?: boolean, - }): Promise { - const url = new URL(args.url); - - if (args.ipCheckers) { - for (const check of args.ipCheckers) { - const result = await this.checkIp(url, check.fn); - if ( - (check.type === 'black' && result === true) || - (check.type === 'white' && result === false) - ) { - throw new StatusError('IP is not allowed', 403, 'IP is not allowed'); - } - } - } - - const timeout = args.timeout ?? 10 * 1000; - - const controller = new AbortController(); - setTimeout(() => { - controller.abort(); - }, timeout * 6); - - const res = await Promise.race([ - undici.fetch(args.url, { - method: args.method ?? 'GET', - headers: args.headers, - body: args.body, - redirect: args.redirect, - dispatcher: args.dispatcher ?? this.getAgentByUrl(url), - keepalive: true, - signal: controller.signal, - }), - new Promise((res) => setTimeout(() => res(null), timeout)) - ]); - - if (res == null) { - throw new StatusError(`Gateway Timeout`, 504, 'Gateway Timeout'); - } - - if (!res.ok && !args.noOkError) { - throw new StatusError(`${res.status} ${res.statusText}`, res.status, res.statusText); - } - - return ({ - ...res, - body: this.fetchLimiter(res, args.size), - }); - } - - /** - * Fetch body limiter - * @param res undici.Response - * @param size number of Max size (Bytes) (default: 10MiB) - * @returns ReadableStream (provided by node:stream/web) - */ - @bindThis - private fetchLimiter(res: undici.Response, size: number = 10 * 1024 * 1024) { - if (res.body == null) return null; - - let total = 0; - return res.body.pipeThrough(new TransformStream({ - start() {}, - transform(chunk, controller) { - // TypeScirptグローバルの定義はUnit8ArrayだがundiciはReadableStreamを渡してくるので一応変換 - const uint8 = new Uint8Array(chunk); - total += uint8.length; - if (total > size) { - controller.error(new StatusError(`Payload Too Large`, 413, 'Payload Too Large')); - } else { - controller.enqueue(uint8); - } - }, - flush() {}, - })); + }; } } diff --git a/packages/backend/src/core/activitypub/ApRequestService.ts b/packages/backend/src/core/activitypub/ApRequestService.ts index b09bae7c26..7ac5eaf7a0 100644 --- a/packages/backend/src/core/activitypub/ApRequestService.ts +++ b/packages/backend/src/core/activitypub/ApRequestService.ts @@ -180,11 +180,13 @@ export class ApRequestService { }, }); - const res = await this.httpRequestService.fetch({ + const res = await this.httpRequestService.fetch( url, - method: req.request.method, - headers: req.request.headers, - }); + { + method: req.request.method, + headers: req.request.headers, + } + ); return await res.json(); } diff --git a/packages/backend/src/core/activitypub/LdSignatureService.ts b/packages/backend/src/core/activitypub/LdSignatureService.ts index f736543087..c6764e7ee9 100644 --- a/packages/backend/src/core/activitypub/LdSignatureService.ts +++ b/packages/backend/src/core/activitypub/LdSignatureService.ts @@ -115,15 +115,20 @@ class LdSignature { @bindThis private async fetchDocument(url: string) { - const json = await this.httpRequestService.fetch({ + const json = await this.httpRequestService.fetch( url, - headers: { - Accept: 'application/ld+json, application/json', + { + headers: { + Accept: 'application/ld+json, application/json', + }, + // TODO + //timeout: this.loderTimeout, }, - noOkError: true, - // TODO - //timeout: this.loderTimeout, - }).then(res => { + { + noOkError: true, + bypassProxy: false, + } + ).then(res => { if (!res.ok) { throw `${res.status} ${res.statusText}`; } else { diff --git a/packages/backend/src/queue/processors/WebhookDeliverProcessorService.ts b/packages/backend/src/queue/processors/WebhookDeliverProcessorService.ts index 57ebd87522..f0543a5ed1 100644 --- a/packages/backend/src/queue/processors/WebhookDeliverProcessorService.ts +++ b/packages/backend/src/queue/processors/WebhookDeliverProcessorService.ts @@ -33,24 +33,26 @@ export class WebhookDeliverProcessorService { try { this.logger.debug(`delivering ${job.data.webhookId}`); - const res = await this.httpRequestService.fetch({ - url: job.data.to, - method: 'POST', - headers: { - 'User-Agent': 'Misskey-Hooks', - 'X-Misskey-Host': this.config.host, - 'X-Misskey-Hook-Id': job.data.webhookId, - 'X-Misskey-Hook-Secret': job.data.secret, - }, - body: JSON.stringify({ - hookId: job.data.webhookId, - userId: job.data.userId, - eventId: job.data.eventId, - createdAt: job.data.createdAt, - type: job.data.type, - body: job.data.content, - }), - }); + const res = await this.httpRequestService.fetch( + job.data.to, + { + method: 'POST', + headers: { + 'User-Agent': 'Misskey-Hooks', + 'X-Misskey-Host': this.config.host, + 'X-Misskey-Hook-Id': job.data.webhookId, + 'X-Misskey-Hook-Secret': job.data.secret, + }, + body: JSON.stringify({ + hookId: job.data.webhookId, + userId: job.data.userId, + eventId: job.data.eventId, + createdAt: job.data.createdAt, + type: job.data.type, + body: job.data.content, + }), + } + ); this.webhooksRepository.update({ id: job.data.webhookId }, { latestSentAt: new Date(), diff --git a/packages/backend/src/server/api/endpoints/fetch-rss.ts b/packages/backend/src/server/api/endpoints/fetch-rss.ts index 537f562231..558ec887db 100644 --- a/packages/backend/src/server/api/endpoints/fetch-rss.ts +++ b/packages/backend/src/server/api/endpoints/fetch-rss.ts @@ -33,15 +33,17 @@ export default class extends Endpoint { private httpRequestService: HttpRequestService, ) { super(meta, paramDef, async (ps, me) => { - const res = await this.httpRequestService.fetch({ - url: ps.url, - method: 'GET', - headers: Object.assign({ - 'User-Agent': config.userAgent, - Accept: 'application/rss+xml, */*', - }), - timeout: 5000, - }); + const res = await this.httpRequestService.fetch( + ps.url, + { + method: 'GET', + headers: Object.assign({ + 'User-Agent': config.userAgent, + Accept: 'application/rss+xml, */*', + }), + // timeout: 5000, + } + ); const text = await res.text(); diff --git a/packages/backend/src/server/api/endpoints/notes/translate.ts b/packages/backend/src/server/api/endpoints/notes/translate.ts index a436bfd428..070625c6a5 100644 --- a/packages/backend/src/server/api/endpoints/notes/translate.ts +++ b/packages/backend/src/server/api/endpoints/notes/translate.ts @@ -83,25 +83,29 @@ export default class extends Endpoint { const endpoint = instance.deeplIsPro ? 'https://api.deepl.com/v2/translate' : 'https://api-free.deepl.com/v2/translate'; - const res = await this.httpRequestService.fetch({ - url: endpoint, - method: 'POST', - headers: { - 'Content-Type': 'application/x-www-form-urlencoded', - 'User-Agent': config.userAgent, - Accept: 'application/json, */*', + const res = await this.httpRequestService.fetch( + endpoint, + { + method: 'POST', + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + 'User-Agent': config.userAgent, + Accept: 'application/json, */*', + }, + body: params.toString(), }, - body: params.toString(), - // TODO - //timeout: 10000, - }); + { + noOkError: false, + bypassProxy: true, + } + ); const json = (await res.json()) as { - translations: { - detected_source_language: string; - text: string; - }[]; - }; + translations: { + detected_source_language: string; + text: string; + }[]; + }; return { sourceLang: json.translations[0].detected_source_language, diff --git a/packages/backend/src/server/api/integration/DiscordServerService.ts b/packages/backend/src/server/api/integration/DiscordServerService.ts index 805056da8b..0ac2733817 100644 --- a/packages/backend/src/server/api/integration/DiscordServerService.ts +++ b/packages/backend/src/server/api/integration/DiscordServerService.ts @@ -181,7 +181,7 @@ export class DiscordServerService { } })); - const { id, username, discriminator } = (await this.httpRequestService.getJson('https://discord.com/api/users/@me', '*/*', 10 * 1000, { + const { id, username, discriminator } = (await this.httpRequestService.getJson('https://discord.com/api/users/@me', '*/*', { 'Authorization': `Bearer ${accessToken}`, })) as Record; @@ -249,7 +249,7 @@ export class DiscordServerService { } })); - const { id, username, discriminator } = (await this.httpRequestService.getJson('https://discord.com/api/users/@me', '*/*', 10 * 1000, { + const { id, username, discriminator } = (await this.httpRequestService.getJson('https://discord.com/api/users/@me', '*/*', { 'Authorization': `Bearer ${accessToken}`, })) as Record; if (typeof id !== 'string' || typeof username !== 'string' || typeof discriminator !== 'string') { diff --git a/packages/backend/src/server/api/integration/GithubServerService.ts b/packages/backend/src/server/api/integration/GithubServerService.ts index 6f38c262a1..a8c745d2dc 100644 --- a/packages/backend/src/server/api/integration/GithubServerService.ts +++ b/packages/backend/src/server/api/integration/GithubServerService.ts @@ -174,7 +174,7 @@ export class GithubServerService { } })); - const { login, id } = (await this.httpRequestService.getJson('https://api.github.com/user', 'application/vnd.github.v3+json', 10 * 1000, { + const { login, id } = (await this.httpRequestService.getJson('https://api.github.com/user', 'application/vnd.github.v3+json', { 'Authorization': `bearer ${accessToken}`, })) as Record; if (typeof login !== 'string' || typeof id !== 'string') { @@ -223,7 +223,7 @@ export class GithubServerService { } })); - const { login, id } = (await this.httpRequestService.getJson('https://api.github.com/user', 'application/vnd.github.v3+json', 10 * 1000, { + const { login, id } = (await this.httpRequestService.getJson('https://api.github.com/user', 'application/vnd.github.v3+json', { 'Authorization': `bearer ${accessToken}`, })) as Record; diff --git a/packages/backend/src/server/web/UrlPreviewService.ts b/packages/backend/src/server/web/UrlPreviewService.ts index 1545f0a9f9..116e9242e2 100644 --- a/packages/backend/src/server/web/UrlPreviewService.ts +++ b/packages/backend/src/server/web/UrlPreviewService.ts @@ -63,7 +63,6 @@ export class UrlPreviewService { this.logger.info(meta.summalyProxy ? `(Proxy) Getting preview of ${url}@${lang} ...` : `Getting preview of ${url}@${lang} ...`); - try { const summary = meta.summalyProxy ? await this.httpRequestService.getJson>(`${meta.summalyProxy}?${query({ url: url,