diff --git a/.github/workflows/report-backend-memory.yml b/.github/workflows/report-backend-memory.yml index c339ca49b4..bf2e311c83 100644 --- a/.github/workflows/report-backend-memory.yml +++ b/.github/workflows/report-backend-memory.yml @@ -54,55 +54,110 @@ jobs: BASE_MEMORY=$(cat ./artifacts/memory-base.json) HEAD_MEMORY=$(cat ./artifacts/memory-head.json) - BASE_RSS=$(echo "$BASE_MEMORY" | jq -r '.memory.rss // 0') - HEAD_RSS=$(echo "$HEAD_MEMORY" | jq -r '.memory.rss // 0') + variation() { + calc() { + BASE=$(echo "$BASE_MEMORY" | jq -r ".${1}.${2} // 0") + HEAD=$(echo "$HEAD_MEMORY" | jq -r ".${1}.${2} // 0") - # Calculate difference - if [ "$BASE_RSS" -gt 0 ] && [ "$HEAD_RSS" -gt 0 ]; then - DIFF=$((HEAD_RSS - BASE_RSS)) - DIFF_PERCENT=$(echo "scale=2; ($DIFF * 100) / $BASE_RSS" | bc) + DIFF=$((HEAD - BASE)) + if [ "$BASE" -gt 0 ]; then + DIFF_PERCENT=$(echo "scale=2; ($DIFF * 100) / $BASE" | bc) + else + DIFF_PERCENT=0 + fi - # Convert to MB for readability - BASE_MB=$(echo "scale=2; $BASE_RSS / 1048576" | bc) - HEAD_MB=$(echo "scale=2; $HEAD_RSS / 1048576" | bc) - DIFF_MB=$(echo "scale=2; $DIFF / 1048576" | bc) + # Convert KB to MB for readability + BASE_MB=$(echo "scale=2; $BASE / 1024" | bc) + HEAD_MB=$(echo "scale=2; $HEAD / 1024" | bc) + DIFF_MB=$(echo "scale=2; $DIFF / 1024" | bc) - echo "base_mb=$BASE_MB" >> "$GITHUB_OUTPUT" - echo "head_mb=$HEAD_MB" >> "$GITHUB_OUTPUT" - echo "diff_mb=$DIFF_MB" >> "$GITHUB_OUTPUT" - echo "diff_percent=$DIFF_PERCENT" >> "$GITHUB_OUTPUT" - echo "has_data=true" >> "$GITHUB_OUTPUT" + JSON=$(jq -c -n \ + --argjson base "$BASE_MB" \ + --argjson head "$HEAD_MB" \ + --argjson diff "$DIFF_MB" \ + --argjson diff_percent "$DIFF_PERCENT" \ + '{base: $base, head: $head, diff: $diff, diff_percent: $diff_percent}') - # Determine if this is a significant change (more than 5% increase) - if [ "$(echo "$DIFF_PERCENT > 5" | bc)" -eq 1 ]; then - echo "significant_increase=true" >> "$GITHUB_OUTPUT" - else - echo "significant_increase=false" >> "$GITHUB_OUTPUT" - fi - else - echo "has_data=false" >> "$GITHUB_OUTPUT" - fi + echo "$JSON" + } + + JSON=$(jq -c -n \ + --argjson VmRSS "$(calc $1 VmRSS)" \ + --argjson VmHWM "$(calc $1 VmHWM)" \ + --argjson VmSize "$(calc $1 VmSize)" \ + --argjson VmData "$(calc $1 VmData)" \ + '{VmRSS: $VmRSS, VmHWM: $VmHWM, VmSize: $VmSize, VmData: $VmData}') + + echo "$JSON" + } + + JSON=$(jq -c -n \ + --argjson beforeGc "$(variation beforeGc)" \ + --argjson afterGc "$(variation afterGc)" \ + --argjson afterRequest "$(variation afterRequest)" \ + '{beforeGc: $beforeGc, afterGc: $afterGc, afterRequest: $afterRequest}') + + echo "res=$JSON" >> "$GITHUB_OUTPUT" - id: build-comment name: Build memory comment + env: + RES: ${{ steps.compare.outputs.res }} run: | - HEADER="## Backend Memory Usage Comparison" + HEADER="## Backend memory usage comparison" FOOTER="[See workflow logs for details](https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID})" echo "$HEADER" > ./output.md echo >> ./output.md - if [ "${{ steps.compare.outputs.has_data }}" == "true" ]; then - echo "| Metric | base | head | Diff |" >> ./output.md - echo "|--------|------|------|------|" >> ./output.md - echo "| RSS | ${{ steps.compare.outputs.base_mb }} MB | ${{ steps.compare.outputs.head_mb }} MB | ${{ steps.compare.outputs.diff_mb }} MB (${{ steps.compare.outputs.diff_percent }}%) |" >> ./output.md - echo >> ./output.md + table() { + echo "| Metric | base (MB) | head (MB) | Diff (MB) | Diff (%) |" >> ./output.md + echo "|--------|------:|------:|------:|------:|" >> ./output.md - if [ "${{ steps.compare.outputs.significant_increase }}" == "true" ]; then - echo "⚠️ **Warning**: Memory usage has increased by more than 5%. Please verify this is not an unintended change." >> ./output.md - echo >> ./output.md - fi - else - echo "Could not retrieve memory usage data." >> ./output.md + line() { + METRIC=$2 + BASE=$(echo "$RES" | jq -r ".${1}.${2}.base") + HEAD=$(echo "$RES" | jq -r ".${1}.${2}.head") + DIFF=$(echo "$RES" | jq -r ".${1}.${2}.diff") + DIFF_PERCENT=$(echo "$RES" | jq -r ".${1}.${2}.diff_percent") + + if (( $(echo "$DIFF_PERCENT > 0" | bc -l) )); then + DIFF="+$DIFF" + DIFF_PERCENT="+$DIFF_PERCENT" + fi + + # highlight VmRSS + if [ "$2" = "VmRSS" ]; then + METRIC="**${METRIC}**" + BASE="**${BASE}**" + HEAD="**${HEAD}**" + DIFF="**${DIFF}**" + DIFF_PERCENT="**${DIFF_PERCENT}**" + fi + + echo "| ${METRIC} | ${BASE} MB | ${HEAD} MB | ${DIFF} MB | ${DIFF_PERCENT}% |" >> ./output.md + } + + line $1 VmRSS + line $1 VmHWM + line $1 VmSize + line $1 VmData + } + + echo "### Before GC" >> ./output.md + table beforeGc + echo >> ./output.md + + echo "### After GC" >> ./output.md + table afterGc + echo >> ./output.md + + echo "### After Request" >> ./output.md + table afterRequest + echo >> ./output.md + + # Determine if this is a significant change (more than 5% increase) + if [ "$(echo "$RES" | jq -r '.afterGc.VmRSS.diff_percent | tonumber > 5')" = "true" ]; then + echo "⚠️ **Warning**: Memory usage has increased by more than 5%. Please verify this is not an unintended change." >> ./output.md echo >> ./output.md fi diff --git a/packages/backend/package.json b/packages/backend/package.json index 003d763bc3..357107245e 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -96,7 +96,6 @@ "@swc/cli": "0.7.9", "@swc/core": "1.15.7", "@twemoji/parser": "16.0.0", - "@types/redis-info": "3.0.3", "accepts": "1.3.8", "ajv": "8.17.1", "archiver": "7.0.1", @@ -154,7 +153,6 @@ "random-seed": "0.3.0", "ratelimiter": "3.4.1", "re2": "1.23.0", - "redis-info": "3.1.0", "reflect-metadata": "0.2.2", "rename": "1.0.4", "rss-parser": "3.13.0", diff --git a/packages/backend/scripts/measure-memory.mjs b/packages/backend/scripts/measure-memory.mjs index baa4198adf..3f30e24fb4 100644 --- a/packages/backend/scripts/measure-memory.mjs +++ b/packages/backend/scripts/measure-memory.mjs @@ -14,16 +14,46 @@ import { fork } from 'node:child_process'; import { setTimeout } from 'node:timers/promises'; import { fileURLToPath } from 'node:url'; import { dirname, join } from 'node:path'; +import * as http from 'node:http'; +import * as fs from 'node:fs/promises'; const __filename = fileURLToPath(import.meta.url); const __dirname = dirname(__filename); +const SAMPLE_COUNT = 3; // Number of samples to measure const STARTUP_TIMEOUT = 120000; // 120 seconds timeout for server startup const MEMORY_SETTLE_TIME = 10000; // Wait 10 seconds after startup for memory to settle -async function measureMemory() { - const startTime = Date.now(); +const keys = { + VmPeak: 0, + VmSize: 0, + VmHWM: 0, + VmRSS: 0, + VmData: 0, + VmStk: 0, + VmExe: 0, + VmLib: 0, + VmPTE: 0, + VmSwap: 0, +}; +async function getMemoryUsage(pid) { + const status = await fs.readFile(`/proc/${pid}/status`, 'utf-8'); + + const result = {}; + for (const key of Object.keys(keys)) { + const match = status.match(new RegExp(`${key}:\\s+(\\d+)\\s+kB`)); + if (match) { + result[key] = parseInt(match[1], 10); + } else { + throw new Error(`Failed to parse ${key} from /proc/${pid}/status`); + } + } + + return result; +} + +async function measureMemory() { // Start the Misskey backend server using fork to enable IPC const serverProcess = fork(join(__dirname, '../built/boot/entry.js'), ['expose-gc'], { cwd: join(__dirname, '..'), @@ -31,9 +61,9 @@ async function measureMemory() { ...process.env, NODE_ENV: 'production', MK_DISABLE_CLUSTERING: '1', - MK_FORCE_GC: '1', }, stdio: ['pipe', 'pipe', 'pipe', 'ipc'], + execArgv: [...process.execArgv, '--expose-gc'], }); let serverReady = false; @@ -59,6 +89,40 @@ async function measureMemory() { process.stderr.write(`[server error] ${err}\n`); }); + async function triggerGc() { + const ok = new Promise((resolve) => { + serverProcess.once('message', (message) => { + if (message === 'gc ok') resolve(); + }); + }); + + serverProcess.send('gc'); + + await ok; + + await setTimeout(1000); + } + + function createRequest() { + return new Promise((resolve, reject) => { + const req = http.request({ + host: 'localhost', + port: 61812, + path: '/api/meta', + method: 'POST', + }, (res) => { + res.on('data', () => { }); + res.on('end', () => { + resolve(); + }); + }); + req.on('error', (err) => { + reject(err); + }); + req.end(); + }); + } + // Wait for server to be ready or timeout const startupStartTime = Date.now(); while (!serverReady) { @@ -75,46 +139,23 @@ async function measureMemory() { // Wait for memory to settle await setTimeout(MEMORY_SETTLE_TIME); - // Get memory usage from the server process via /proc const pid = serverProcess.pid; - let memoryInfo; - try { - const fs = await import('node:fs/promises'); + const beforeGc = await getMemoryUsage(pid); - // Read /proc/[pid]/status for detailed memory info - const status = await fs.readFile(`/proc/${pid}/status`, 'utf-8'); - const vmRssMatch = status.match(/VmRSS:\s+(\d+)\s+kB/); - const vmDataMatch = status.match(/VmData:\s+(\d+)\s+kB/); - const vmSizeMatch = status.match(/VmSize:\s+(\d+)\s+kB/); + await triggerGc(); - memoryInfo = { - rss: vmRssMatch ? parseInt(vmRssMatch[1], 10) * 1024 : null, - heapUsed: vmDataMatch ? parseInt(vmDataMatch[1], 10) * 1024 : null, - vmSize: vmSizeMatch ? parseInt(vmSizeMatch[1], 10) * 1024 : null, - }; - } catch (err) { - // Fallback: use ps command - process.stderr.write(`Warning: Could not read /proc/${pid}/status: ${err}\n`); + const afterGc = await getMemoryUsage(pid); - const { execSync } = await import('node:child_process'); - try { - const ps = execSync(`ps -o rss= -p ${pid}`, { encoding: 'utf-8' }); - const rssKb = parseInt(ps.trim(), 10); - memoryInfo = { - rss: rssKb * 1024, - heapUsed: null, - vmSize: null, - }; - } catch { - memoryInfo = { - rss: null, - heapUsed: null, - vmSize: null, - error: 'Could not measure memory', - }; - } - } + // create some http requests to simulate load + const REQUEST_COUNT = 10; + await Promise.all( + Array.from({ length: REQUEST_COUNT }).map(() => createRequest()), + ); + + await triggerGc(); + + const afterRequest = await getMemoryUsage(pid); // Stop the server serverProcess.kill('SIGTERM'); @@ -137,15 +178,51 @@ async function measureMemory() { const result = { timestamp: new Date().toISOString(), - startupTimeMs: startupTime, - memory: memoryInfo, + beforeGc, + afterGc, + afterRequest, + }; + + return result; +} + +async function main() { + // 直列の方が時間的に分散されて正確そうだから直列でやる + const results = []; + for (let i = 0; i < SAMPLE_COUNT; i++) { + const res = await measureMemory(); + results.push(res); + } + + // Calculate averages + const beforeGc = structuredClone(keys); + const afterGc = structuredClone(keys); + const afterRequest = structuredClone(keys); + for (const res of results) { + for (const key of Object.keys(keys)) { + beforeGc[key] += res.beforeGc[key]; + afterGc[key] += res.afterGc[key]; + afterRequest[key] += res.afterRequest[key]; + } + } + for (const key of Object.keys(keys)) { + beforeGc[key] = Math.round(beforeGc[key] / SAMPLE_COUNT); + afterGc[key] = Math.round(afterGc[key] / SAMPLE_COUNT); + afterRequest[key] = Math.round(afterRequest[key] / SAMPLE_COUNT); + } + + const result = { + timestamp: new Date().toISOString(), + beforeGc, + afterGc, + afterRequest, }; // Output as JSON to stdout console.log(JSON.stringify(result, null, 2)); } -measureMemory().catch((err) => { +main().catch((err) => { console.error(JSON.stringify({ error: err.message, timestamp: new Date().toISOString(), diff --git a/packages/backend/src/boot/entry.ts b/packages/backend/src/boot/entry.ts index 56b339b6aa..3a33d198a5 100644 --- a/packages/backend/src/boot/entry.ts +++ b/packages/backend/src/boot/entry.ts @@ -86,9 +86,17 @@ if (!envOption.disableClustering) { ev.mount(); } -if (envOption.forceGc && global.gc != null) { - global.gc(); -} +process.on('message', msg => { + if (msg === 'gc') { + if (global.gc != null) { + logger.info('Manual GC triggered'); + global.gc(); + if (process.send != null) process.send('gc ok'); + } else { + logger.warn('Manual GC requested but gc is not available. Start the process with --expose-gc to enable this feature.'); + } + } +}); readyRef.value = true; diff --git a/packages/backend/src/core/QueueService.ts b/packages/backend/src/core/QueueService.ts index 42782167bb..f90ae80731 100644 --- a/packages/backend/src/core/QueueService.ts +++ b/packages/backend/src/core/QueueService.ts @@ -6,7 +6,6 @@ import { randomUUID } from 'node:crypto'; import { Inject, Injectable } from '@nestjs/common'; import { MetricsTime, type JobType } from 'bullmq'; -import { parse as parseRedisInfo } from 'redis-info'; import type { IActivity } from '@/core/activitypub/type.js'; import type { MiDriveFile } from '@/models/DriveFile.js'; import type { MiWebhook, WebhookEventTypes } from '@/models/Webhook.js'; @@ -86,6 +85,19 @@ const REPEATABLE_SYSTEM_JOB_DEF = [{ pattern: '0 4 * * *', }]; +function parseRedisInfo(infoText: string): Record { + const fields = infoText + .split('\n') + .filter(line => line.length > 0 && !line.startsWith('#')) + .map(line => line.trim().split(':')); + + const result: Record = {}; + for (const [key, value] of fields) { + result[key] = value; + } + return result; +} + @Injectable() export class QueueService { constructor( @@ -890,7 +902,7 @@ export class QueueService { }, db: { version: db.redis_version, - mode: db.redis_mode, + mode: db.redis_mode as 'cluster' | 'standalone' | 'sentinel', runId: db.run_id, processId: db.process_id, port: parseInt(db.tcp_port), diff --git a/packages/backend/src/env.ts b/packages/backend/src/env.ts index 9957938467..ba44cfa2e6 100644 --- a/packages/backend/src/env.ts +++ b/packages/backend/src/env.ts @@ -11,7 +11,6 @@ const envOption = { verbose: false, withLogTime: false, quiet: false, - forceGc: false, }; for (const key of Object.keys(envOption) as (keyof typeof envOption)[]) { diff --git a/packages/frontend-embed/eslint.config.js b/packages/frontend-embed/eslint.config.js index 63767cfb3c..d1ca70617b 100644 --- a/packages/frontend-embed/eslint.config.js +++ b/packages/frontend-embed/eslint.config.js @@ -144,7 +144,15 @@ export default [ 'vue/return-in-computed-property': 'warn', 'vue/no-setup-props-reactivity-loss': 'warn', 'vue/max-attributes-per-line': 'off', - 'vue/html-self-closing': 'off', + 'vue/html-self-closing': ['error', { + html: { + void: 'any', + normal: 'never', + component: 'any', + }, + svg: 'any', + math: 'any', + }], 'vue/singleline-html-element-content-newline': 'off', 'vue/v-on-event-hyphenation': ['error', 'never', { autofix: true, diff --git a/packages/frontend-embed/src/components/EmAvatar.vue b/packages/frontend-embed/src/components/EmAvatar.vue index 58c35c8ef0..3f91e14403 100644 --- a/packages/frontend-embed/src/components/EmAvatar.vue +++ b/packages/frontend-embed/src/components/EmAvatar.vue @@ -9,16 +9,16 @@ SPDX-License-Identifier: AGPL-3.0-only
-
-
-
+
+
+
-
-
-
+
+
+
diff --git a/packages/frontend-embed/src/components/EmImgWithBlurhash.vue b/packages/frontend-embed/src/components/EmImgWithBlurhash.vue index 71f0ee9294..be18ce79d5 100644 --- a/packages/frontend-embed/src/components/EmImgWithBlurhash.vue +++ b/packages/frontend-embed/src/components/EmImgWithBlurhash.vue @@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only diff --git a/packages/frontend/eslint.config.js b/packages/frontend/eslint.config.js index 15baf449fe..adae471c0a 100644 --- a/packages/frontend/eslint.config.js +++ b/packages/frontend/eslint.config.js @@ -147,7 +147,15 @@ export default [ 'vue/return-in-computed-property': 'warn', 'vue/no-setup-props-reactivity-loss': 'warn', 'vue/max-attributes-per-line': 'off', - 'vue/html-self-closing': 'off', + 'vue/html-self-closing': ['error', { + html: { + void: 'any', + normal: 'never', + component: 'any', + }, + svg: 'any', + math: 'any', + }], 'vue/singleline-html-element-content-newline': 'off', 'vue/v-on-event-hyphenation': ['error', 'never', { autofix: true, diff --git a/packages/frontend/src/components/MkDialog.vue b/packages/frontend/src/components/MkDialog.vue index 4801b412f8..fb8b38de6d 100644 --- a/packages/frontend/src/components/MkDialog.vue +++ b/packages/frontend/src/components/MkDialog.vue @@ -25,8 +25,8 @@ SPDX-License-Identifier: AGPL-3.0-only diff --git a/packages/frontend/src/components/MkDivider.vue b/packages/frontend/src/components/MkDivider.vue index f72f091383..808a9ae2f8 100644 --- a/packages/frontend/src/components/MkDivider.vue +++ b/packages/frontend/src/components/MkDivider.vue @@ -12,7 +12,7 @@ SPDX-License-Identifier: AGPL-3.0-only borderWidth ? { borderWidth: borderWidth } : {}, borderColor ? { borderColor: borderColor } : {}, ]" -/> +>