ファイルのダウンロードがタイムアウトしなくなっているのを修正など (#6233)
* Refactor download / file-info * body read timeout on download Fix syuilo#6232
This commit is contained in:
parent
9fcf94b197
commit
ca66acac2b
|
@ -102,6 +102,7 @@
|
||||||
"@types/websocket": "1.0.0",
|
"@types/websocket": "1.0.0",
|
||||||
"@types/ws": "7.2.3",
|
"@types/ws": "7.2.3",
|
||||||
"@typescript-eslint/parser": "2.26.0",
|
"@typescript-eslint/parser": "2.26.0",
|
||||||
|
"abort-controller": "3.0.0",
|
||||||
"animejs": "3.1.0",
|
"animejs": "3.1.0",
|
||||||
"apexcharts": "3.17.1",
|
"apexcharts": "3.17.1",
|
||||||
"autobind-decorator": "2.4.0",
|
"autobind-decorator": "2.4.0",
|
||||||
|
|
|
@ -1,55 +1,39 @@
|
||||||
import * as fs from 'fs';
|
import * as fs from 'fs';
|
||||||
|
import * as stream from 'stream';
|
||||||
|
import * as util from 'util';
|
||||||
import fetch from 'node-fetch';
|
import fetch from 'node-fetch';
|
||||||
import { httpAgent, httpsAgent } from './fetch';
|
import { httpAgent, httpsAgent } from './fetch';
|
||||||
|
import { AbortController } from 'abort-controller';
|
||||||
import config from '../config';
|
import config from '../config';
|
||||||
import * as chalk from 'chalk';
|
import * as chalk from 'chalk';
|
||||||
import Logger from '../services/logger';
|
import Logger from '../services/logger';
|
||||||
|
|
||||||
|
const pipeline = util.promisify(stream.pipeline);
|
||||||
|
|
||||||
export async function downloadUrl(url: string, path: string) {
|
export async function downloadUrl(url: string, path: string) {
|
||||||
const logger = new Logger('download');
|
const logger = new Logger('download');
|
||||||
|
|
||||||
logger.info(`Downloading ${chalk.cyan(url)} ...`);
|
logger.info(`Downloading ${chalk.cyan(url)} ...`);
|
||||||
|
const controller = new AbortController();
|
||||||
|
setTimeout(() => {
|
||||||
|
controller.abort();
|
||||||
|
}, 11 * 1000);
|
||||||
|
|
||||||
const response = await fetch(new URL(url).href, {
|
const response = await fetch(new URL(url).href, {
|
||||||
headers: {
|
headers: {
|
||||||
'User-Agent': config.userAgent
|
'User-Agent': config.userAgent
|
||||||
},
|
},
|
||||||
timeout: 10 * 1000,
|
timeout: 10 * 1000,
|
||||||
|
signal: controller.signal,
|
||||||
agent: u => u.protocol == 'http:' ? httpAgent : httpsAgent,
|
agent: u => u.protocol == 'http:' ? httpAgent : httpsAgent,
|
||||||
}).then(response => {
|
|
||||||
if (!response.ok) {
|
|
||||||
logger.error(`Got ${response.status} (${url})`);
|
|
||||||
throw response.status;
|
|
||||||
} else {
|
|
||||||
return response;
|
|
||||||
}
|
|
||||||
});
|
});
|
||||||
|
|
||||||
await new Promise((res, rej) => {
|
if (!response.ok) {
|
||||||
const writable = fs.createWriteStream(path);
|
logger.error(`Got ${response.status} (${url})`);
|
||||||
|
throw response.status;
|
||||||
|
}
|
||||||
|
|
||||||
response.body.on('error', (error: any) => {
|
await pipeline(response.body, fs.createWriteStream(path));
|
||||||
logger.error(`Failed to start download: ${chalk.cyan(url)}: ${error}`, {
|
|
||||||
url: url,
|
|
||||||
e: error
|
|
||||||
});
|
|
||||||
writable.close();
|
|
||||||
rej(error);
|
|
||||||
});
|
|
||||||
|
|
||||||
writable.on('finish', () => {
|
logger.succ(`Download finished: ${chalk.cyan(url)}`);
|
||||||
logger.succ(`Download finished: ${chalk.cyan(url)}`);
|
|
||||||
res();
|
|
||||||
});
|
|
||||||
|
|
||||||
writable.on('error', error => {
|
|
||||||
logger.error(`Download failed: ${chalk.cyan(url)}: ${error}`, {
|
|
||||||
url: url,
|
|
||||||
e: error
|
|
||||||
});
|
|
||||||
rej(error);
|
|
||||||
});
|
|
||||||
|
|
||||||
response.body.pipe(writable);
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,10 +1,14 @@
|
||||||
import * as fs from 'fs';
|
import * as fs from 'fs';
|
||||||
import * as crypto from 'crypto';
|
import * as crypto from 'crypto';
|
||||||
|
import * as stream from 'stream';
|
||||||
|
import * as util from 'util';
|
||||||
import * as fileType from 'file-type';
|
import * as fileType from 'file-type';
|
||||||
import isSvg from 'is-svg';
|
import isSvg from 'is-svg';
|
||||||
import * as probeImageSize from 'probe-image-size';
|
import * as probeImageSize from 'probe-image-size';
|
||||||
import * as sharp from 'sharp';
|
import * as sharp from 'sharp';
|
||||||
|
|
||||||
|
const pipeline = util.promisify(stream.pipeline);
|
||||||
|
|
||||||
export type FileInfo = {
|
export type FileInfo = {
|
||||||
size: number;
|
size: number;
|
||||||
md5: string;
|
md5: string;
|
||||||
|
@ -138,32 +142,17 @@ export async function checkSvg(path: string) {
|
||||||
* Get file size
|
* Get file size
|
||||||
*/
|
*/
|
||||||
export async function getFileSize(path: string): Promise<number> {
|
export async function getFileSize(path: string): Promise<number> {
|
||||||
return new Promise<number>((res, rej) => {
|
const getStat = util.promisify(fs.stat);
|
||||||
fs.stat(path, (err, stats) => {
|
return (await getStat(path)).size;
|
||||||
if (err) return rej(err);
|
|
||||||
res(stats.size);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Calculate MD5 hash
|
* Calculate MD5 hash
|
||||||
*/
|
*/
|
||||||
async function calcHash(path: string): Promise<string> {
|
async function calcHash(path: string): Promise<string> {
|
||||||
return new Promise<string>((res, rej) => {
|
const hash = crypto.createHash('md5').setEncoding('hex');
|
||||||
const readable = fs.createReadStream(path);
|
await pipeline(fs.createReadStream(path), hash);
|
||||||
const hash = crypto.createHash('md5');
|
return hash.read();
|
||||||
const chunks: Buffer[] = [];
|
|
||||||
readable
|
|
||||||
.on('error', rej)
|
|
||||||
.pipe(hash)
|
|
||||||
.on('error', rej)
|
|
||||||
.on('data', chunk => chunks.push(chunk))
|
|
||||||
.on('end', () => {
|
|
||||||
const buffer = Buffer.concat(chunks);
|
|
||||||
res(buffer.toString('hex'));
|
|
||||||
});
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
Loading…
Reference in New Issue