From 49da90289a2ecd43c48cc961871c316dfc0656be Mon Sep 17 00:00:00 2001 From: tamaina Date: Mon, 2 Jan 2023 17:29:05 +0000 Subject: [PATCH] Implement? HttpFetchService --- packages/backend/package.json | 1 + packages/backend/src/core/HttpFetchService.ts | 178 ++++++++++++++++++ yarn.lock | 19 +- 3 files changed, 189 insertions(+), 9 deletions(-) create mode 100644 packages/backend/src/core/HttpFetchService.ts diff --git a/packages/backend/package.json b/packages/backend/package.json index e4e3274c0b..21cce024ad 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -119,6 +119,7 @@ "twemoji-parser": "14.0.0", "typeorm": "0.3.11", "ulid": "2.3.0", + "undici": "^5.14.0", "unzipper": "0.10.11", "uuid": "9.0.0", "vary": "1.1.2", diff --git a/packages/backend/src/core/HttpFetchService.ts b/packages/backend/src/core/HttpFetchService.ts new file mode 100644 index 0000000000..3f43bbe071 --- /dev/null +++ b/packages/backend/src/core/HttpFetchService.ts @@ -0,0 +1,178 @@ +import CacheableLookup from 'cacheable-lookup'; +import { Inject, Injectable } from '@nestjs/common'; +import { DI } from '@/di-symbols.js'; +import type { Config } from '@/config.js'; +import { StatusError } from '@/misc/status-error.js'; +import { bindThis } from '@/decorators.js'; +import * as undici from 'undici'; +import { LookupFunction } from 'node:net'; +import { TransformStream } from 'node:stream/web'; + +@Injectable() +export class HttpRequestService { + /** + * Get http non-proxy agent + */ + private agent: undici.Agent; + + /** + * Get http proxy or non-proxy agent + */ + public proxiedAgent: undici.ProxyAgent | undici.Agent; + + public readonly clientDefaults: undici.Agent.Options; + + constructor( + @Inject(DI.config) + private config: Config, + ) { + const cache = new CacheableLookup({ + maxTtl: 3600, // 1hours + errorTtl: 30, // 30secs + lookup: false, // nativeのdns.lookupにfallbackしない + }); + + this.clientDefaults = { + keepAliveTimeout: 4 * 1000, + keepAliveMaxTimeout: 10 * 60 * 1000, + keepAliveTimeoutThreshold: 1 * 1000, + strictContentLength: true, + connect: { + maxCachedSessions: 100, // TLSセッションのキャッシュ数 https://github.com/nodejs/undici/blob/v5.14.0/lib/core/connect.js#L80 + lookup: cache.lookup as LookupFunction, // https://github.com/nodejs/undici/blob/v5.14.0/lib/core/connect.js#L98 + }, + } + + this.agent = new undici.Agent({ + ...this.clientDefaults, + }); + + const maxSockets = Math.max(256, config.deliverJobConcurrency ?? 128); + + this.proxiedAgent = config.proxy + ? new undici.ProxyAgent({ + ...this.clientDefaults, + connections: maxSockets, + + uri: config.proxy, + }) + : this.agent; + } + + /** + * Get agent by URL + * @param url URL + * @param bypassProxy Allways bypass proxy + */ + @bindThis + public getAgentByUrl(url: URL, bypassProxy = false): undici.Agent | undici.ProxyAgent { + if (bypassProxy || (this.config.proxyBypassHosts || []).includes(url.hostname)) { + return this.agent; + } else { + return this.proxiedAgent; + } + } + + @bindThis + public async getJson(url: string, accept = 'application/json, */*', timeout = 10000, headers?: Record): Promise { + const res = await this.getResponse({ + url, + method: 'GET', + headers: Object.assign({ + 'User-Agent': this.config.userAgent, + Accept: accept, + }, headers ?? {}), + timeout, + size: 1024 * 256, + }); + + return await res.json(); + } + + @bindThis + public async getHtml(url: string, accept = 'text/html, */*', timeout = 10000, headers?: Record): Promise { + const res = await this.getResponse({ + url, + method: 'GET', + headers: Object.assign({ + 'User-Agent': this.config.userAgent, + Accept: accept, + }, headers ?? {}), + timeout, + }); + + return await res.text(); + } + + @bindThis + public async getResponse(args: { + url: string, + method: string, + body?: string, + headers: Record, + timeout?: number, + size?: number, + redirect?: RequestRedirect | undefined, + dispatcher?: undici.Dispatcher, + }): Promise { + const timeout = args.timeout ?? 10 * 1000; + + const controller = new AbortController(); + setTimeout(() => { + controller.abort(); + }, timeout * 6); + + const res = await Promise.race([ + undici.fetch(args.url, { + method: args.method, + headers: args.headers, + body: args.body, + redirect: args.redirect, + dispatcher: args.dispatcher ?? this.getAgentByUrl(new URL(args.url)), + keepalive: true, + signal: controller.signal, + }), + new Promise((res) => setTimeout(() => res(null))) + ]); + + if (res == null) { + throw new StatusError(`Request Timeout`, 408, 'Request Timeout'); + } + + if (!res.ok) { + throw new StatusError(`${res.status} ${res.statusText}`, res.status, res.statusText); + } + + return ({ + ...res, + body: this.fetchLimiter(res, args.size), + }); + } + + /** + * Fetch body limiter + * @param res undici.Response + * @param size number of Max size (Bytes) (default: 10MiB) + * @returns ReadableStream (provided by node:stream/web) + */ + @bindThis + private fetchLimiter(res: undici.Response, size: number = 10 * 1024 * 1024) { + if (res.body == null) return null; + + let total = 0; + return res.body.pipeThrough(new TransformStream({ + start() {}, + transform(chunk, controller) { + // TypeScirptグローバルの定義はUnit8ArrayだがundiciはReadableStreamを渡してくるので一応変換 + const uint8 = new Uint8Array(chunk); + total += uint8.length; + if (total > size) { + controller.error(new StatusError(`Payload Too Large`, 413, 'Payload Too Large')); + } else { + controller.enqueue(uint8); + } + }, + flush() {}, + })); + } +} diff --git a/yarn.lock b/yarn.lock index e5958bafe5..54f9f9c0bf 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4271,6 +4271,7 @@ __metadata: typeorm: 0.3.11 typescript: 4.9.4 ulid: 2.3.0 + undici: ^5.14.0 unzipper: 0.10.11 uuid: 9.0.0 vary: 1.1.2 @@ -16686,6 +16687,15 @@ __metadata: languageName: node linkType: hard +"undici@npm:^5.14.0, undici@npm:^5.5.1": + version: 5.14.0 + resolution: "undici@npm:5.14.0" + dependencies: + busboy: ^1.6.0 + checksum: 7a076e44d84b25844b4eb657034437b8b9bb91f17d347de474fdea1d4263ce7ae9406db79cd30de5642519277b4893f43073258bcc8fed420b295da3fdd11b26 + languageName: node + linkType: hard + "undici@npm:^5.2.0": version: 5.13.0 resolution: "undici@npm:5.13.0" @@ -16695,15 +16705,6 @@ __metadata: languageName: node linkType: hard -"undici@npm:^5.5.1": - version: 5.14.0 - resolution: "undici@npm:5.14.0" - dependencies: - busboy: ^1.6.0 - checksum: 7a076e44d84b25844b4eb657034437b8b9bb91f17d347de474fdea1d4263ce7ae9406db79cd30de5642519277b4893f43073258bcc8fed420b295da3fdd11b26 - languageName: node - linkType: hard - "union-value@npm:^1.0.0": version: 1.0.1 resolution: "union-value@npm:1.0.1"