From e6fca721715c0ee070aba922c2815c3d44825765 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Acid=20Chicken=20=28=E7=A1=AB=E9=85=B8=E9=B6=8F=29?= Date: Thu, 20 Jul 2023 17:00:54 +0900 Subject: [PATCH] perf: use slacc instead of unzipper (#10780) Co-authored-by: tamaina --- packages/backend/package.json | 2 - .../ImportCustomEmojisProcessorService.ts | 18 +++-- pnpm-lock.yaml | 79 +------------------ 3 files changed, 12 insertions(+), 87 deletions(-) diff --git a/packages/backend/package.json b/packages/backend/package.json index 52d6ec57d7..7f64c2a9ac 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -154,7 +154,6 @@ "typeorm": "0.3.17", "typescript": "5.1.6", "ulid": "2.3.0", - "unzipper": "0.10.14", "vary": "1.1.2", "web-push": "3.6.3", "ws": "8.13.0", @@ -194,7 +193,6 @@ "@types/sinonjs__fake-timers": "8.1.2", "@types/tinycolor2": "1.4.3", "@types/tmp": "0.2.3", - "@types/unzipper": "0.10.6", "@types/vary": "1.1.0", "@types/web-push": "3.3.2", "@types/ws": "8.5.5", diff --git a/packages/backend/src/queue/processors/ImportCustomEmojisProcessorService.ts b/packages/backend/src/queue/processors/ImportCustomEmojisProcessorService.ts index 4ba749ec52..37b929cb03 100644 --- a/packages/backend/src/queue/processors/ImportCustomEmojisProcessorService.ts +++ b/packages/backend/src/queue/processors/ImportCustomEmojisProcessorService.ts @@ -1,7 +1,7 @@ import * as fs from 'node:fs'; import { Inject, Injectable } from '@nestjs/common'; +import { ZipReader } from 'slacc'; import { DataSource } from 'typeorm'; -import unzipper from 'unzipper'; import { DI } from '@/di-symbols.js'; import type { EmojisRepository, DriveFilesRepository, UsersRepository } from '@/models/index.js'; import type { Config } from '@/config.js'; @@ -72,9 +72,9 @@ export class ImportCustomEmojisProcessorService { } const outputPath = path + '/emojis'; - const unzipStream = fs.createReadStream(destPath); - const extractor = unzipper.Extract({ path: outputPath }); - extractor.on('close', async () => { + try { + this.logger.succ(`Unzipping to ${outputPath}`); + ZipReader.withDestinationPath(outputPath).viaBuffer(await fs.promises.readFile(destPath)); const metaRaw = fs.readFileSync(outputPath + '/meta.json', 'utf-8'); const meta = JSON.parse(metaRaw); @@ -115,8 +115,12 @@ export class ImportCustomEmojisProcessorService { cleanup(); this.logger.succ('Imported'); - }); - unzipStream.pipe(extractor); - this.logger.succ(`Unzipping to ${outputPath}`); + } catch (e) { + if (e instanceof Error || typeof e === 'string') { + this.logger.error(e); + } + cleanup(); + throw e; + } } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c11b93d7b6..8812c2fabf 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -368,9 +368,6 @@ importers: ulid: specifier: 2.3.0 version: 2.3.0 - unzipper: - specifier: 0.10.14 - version: 0.10.14 vary: specifier: 1.1.2 version: 1.1.2 @@ -565,9 +562,6 @@ importers: '@types/tmp': specifier: 0.2.3 version: 0.2.3 - '@types/unzipper': - specifier: 0.10.6 - version: 0.10.6 '@types/vary': specifier: 1.1.0 version: 1.1.0 @@ -8381,12 +8375,6 @@ packages: resolution: {integrity: sha512-PBjIUxZHOuj0R15/xuwJYjFi+KZdNFrehocChv4g5hu6aFroHue8m0lBP0POdK2nKzbw0cgV1mws8+V/JAcEkQ==} dev: true - /@types/unzipper@0.10.6: - resolution: {integrity: sha512-zcBj329AHgKLQyz209N/S9R0GZqXSkUQO4tJSYE3x02qg4JuDFpgKMj50r82Erk1natCWQDIvSccDddt7jPzjA==} - dependencies: - '@types/node': 20.4.2 - dev: true - /@types/uuid@9.0.2: resolution: {integrity: sha512-kNnC1GFBLuhImSnV7w4njQkUiJi0ZXUycu1rUaouPqiKlXkh77JKgdRnTAp1x5eBwcIwbtI+3otwzuIDEuDoxQ==} dev: true @@ -9731,6 +9719,7 @@ packages: /big-integer@1.6.51: resolution: {integrity: sha512-GPEid2Y9QU1Exl1rpO9B2IPJGHPSupF5GnVIP0blYvNOMer2bTvSWs1jGOUg04hTmu67nmLsQ9TBo1puaotBHg==} engines: {node: '>=0.6'} + dev: true /bin-check@4.1.0: resolution: {integrity: sha512-b6weQyEUKsDGFlACWSIOfveEnImkJyK/FGW6FAG42loyoquvjdtOIqO6yBFzHyqyVVhNgNkQxxx09SFLK28YnA==} @@ -9761,13 +9750,6 @@ packages: resolution: {integrity: sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==} engines: {node: '>=8'} - /binary@0.3.0: - resolution: {integrity: sha512-D4H1y5KYwpJgK8wk1Cue5LLPgmwHKYSChkbspQg5JtVuR5ulGckxfR62H3AE9UDkdMC8yyXlqYihuz3Aqg2XZg==} - dependencies: - buffers: 0.1.1 - chainsaw: 0.1.0 - dev: false - /binaryextensions@2.3.0: resolution: {integrity: sha512-nAihlQsYGyc5Bwq6+EsubvANYGExeJKHDO3RjnvwU042fawQTQfM3Kxn7IHUXQOz4bzfwsGYYHGSvXyW4zOGLg==} engines: {node: '>=0.8'} @@ -9784,10 +9766,6 @@ packages: resolution: {integrity: sha512-T7JQa+zsXXEa6/8ZhHcQEW1UFfVM49Ts65uBkFL6fz2QmrElqmbajIDJvuA0tEhRe5eIjpV9ZF+0RfZR9voJFQ==} dev: true - /bluebird@3.4.7: - resolution: {integrity: sha512-iD3898SR7sWVRHbiQv+sHUtHnMvC1o3nW5rAcqnq3uOn07DSAppZYUkIGslDz6gXC7HfunPe7YVBgoEJASPcHA==} - dev: false - /bluebird@3.7.2: resolution: {integrity: sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==} dev: true @@ -9930,11 +9908,6 @@ packages: /buffer-from@1.1.2: resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==} - /buffer-indexof-polyfill@1.0.2: - resolution: {integrity: sha512-I7wzHwA3t1/lwXQh+A5PbNvJxgfo5r3xulgpYDB5zckTu/Z9oUK9biouBKQUjEqzaz3HnAT6TYoovmE+GqSf7A==} - engines: {node: '>=0.10'} - dev: false - /buffer-writer@2.0.0: resolution: {integrity: sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw==} engines: {node: '>=4'} @@ -9960,11 +9933,6 @@ packages: ieee754: 1.2.1 dev: false - /buffers@0.1.1: - resolution: {integrity: sha512-9q/rDEGSb/Qsvv2qvzIzdluL5k7AaJOTrw23z9reQthrbF7is4CtlT0DXyO1oei2DCp4uojjzQ7igaSHp1kAEQ==} - engines: {node: '>=0.2.0'} - dev: false - /bufferutil@4.0.7: resolution: {integrity: sha512-kukuqc39WOHtdxtw4UScxF/WVnMFVSQVKhtx3AjZJzhd0RGZZldcrfSEbVsWWe6KNH253574cq5F+wpv0G9pJw==} engines: {node: '>=6.14.2'} @@ -10185,12 +10153,6 @@ packages: type-detect: 4.0.8 dev: true - /chainsaw@0.1.0: - resolution: {integrity: sha512-75kWfWt6MEKNC8xYXIdRpDehRYY/tNSgwKaJq+dbbDcxORuVrrQ+SEHoWsniVn9XPYfP4gmdWIeDk/4YNp1rNQ==} - dependencies: - traverse: 0.3.9 - dev: false - /chalk-template@1.1.0: resolution: {integrity: sha512-T2VJbcDuZQ0Tb2EWwSotMPJjgpy1/tGee1BTpUNsGZ/qgNjV2t7Mvu+d4600U564nbLesN1x2dPL+xii174Ekg==} engines: {node: '>=14.16'} @@ -11446,12 +11408,6 @@ packages: resolution: {integrity: sha512-7GO6HghkA5fYG9TYnNxi14/7K9f5occMlp3zXAuSxn7CKCxt9xbNWG7yF8hTCSUchlfWSe3uLmlPfigevRItzQ==} engines: {node: '>=12'} - /duplexer2@0.1.4: - resolution: {integrity: sha512-asLFVfWWtJ90ZyOUHMqk7/S2w2guQKxUI2itj3d92ADHhxUSbCMGi1f1cBcJ7xM1To+pE/Khbwo1yuNbMEPKeA==} - dependencies: - readable-stream: 2.3.7 - dev: false - /duplexer@0.1.2: resolution: {integrity: sha512-jtD6YG370ZCIi/9GTaJKQxWTZD045+4R4hTk/x1UyoqadyJ9x9CgSi1RlVDQF8U2sxLLSnFkCaMihqljHIWgMg==} dev: true @@ -12921,16 +12877,6 @@ packages: requiresBuild: true optional: true - /fstream@1.0.12: - resolution: {integrity: sha512-WvJ193OHa0GHPEL+AycEJgxvBEwyfRkN1vhjca23OaPVMCaLCXTd5qAu82AjTcgP1UJmytkOKb63Ypde7raDIg==} - engines: {node: '>=0.6'} - dependencies: - graceful-fs: 4.2.11 - inherits: 2.0.4 - mkdirp: 0.5.6 - rimraf: 2.7.1 - dev: false - /function-bind@1.1.1: resolution: {integrity: sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==} @@ -15466,10 +15412,6 @@ packages: resolution: {integrity: sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==} dev: true - /listenercount@1.0.1: - resolution: {integrity: sha512-3mk/Zag0+IJxeDrxSgaDPy4zZ3w05PRZeJNnlWhzFz5OkX49J4krc+A8X2d2M69vGMBEX0uyl8M+W+8gH+kBqQ==} - dev: false - /listr2@3.14.0(enquirer@2.3.6): resolution: {integrity: sha512-TyWI8G99GX9GjE54cJ+RrNMcIFBfwMPxc3XTFiAYGN4s10hWROGtOg7+O6u6LE3mNkyld7RSLE6nrKBvTfcs3g==} engines: {node: '>=10.0.0'} @@ -20224,10 +20166,6 @@ packages: /trace-redirect@1.0.6: resolution: {integrity: sha512-UUfa1DjjU5flcjMdaFIiIEGDTyu2y/IiMjOX4uGXa7meKBS4vD4f2Uy/tken9Qkd4Jsm4sRsfZcIIPqrRVF3Mg==} - /traverse@0.3.9: - resolution: {integrity: sha512-iawgk0hLP3SxGKDfnDJf8wTz4p2qImnyihM5Hh/sGvQ3K37dPi/w8sRhdNIxYA1TwFwc5mDhIJq+O0RsvXBKdQ==} - dev: false - /trim-newlines@3.0.1: resolution: {integrity: sha512-c1PTsA3tYrIsLGkJkzHF+w9F2EyxfXGo4UyJc4pFL++FMjnq0HJS69T3M7d//gKrFKwy429bouPescbjecU+Zw==} engines: {node: '>=8'} @@ -20700,21 +20638,6 @@ packages: engines: {node: '>=8'} dev: true - /unzipper@0.10.14: - resolution: {integrity: sha512-ti4wZj+0bQTiX2KmKWuwj7lhV+2n//uXEotUmGuQqrbVZSEGFMbI68+c6JCQ8aAmUWYvtHEz2A8K6wXvueR/6g==} - dependencies: - big-integer: 1.6.51 - binary: 0.3.0 - bluebird: 3.4.7 - buffer-indexof-polyfill: 1.0.2 - duplexer2: 0.1.4 - fstream: 1.0.12 - graceful-fs: 4.2.11 - listenercount: 1.0.1 - readable-stream: 2.3.7 - setimmediate: 1.0.5 - dev: false - /update-browserslist-db@1.0.11(browserslist@4.21.9): resolution: {integrity: sha512-dCwEFf0/oT85M1fHBg4F0jtLwJrutGoHSQXCh7u4o2t1drG+c0a9Flnqww6XUKSfQMPpJBRjU8d4RXB09qtvaA==} hasBin: true