From 3205eb6925136bb90ef2da04acaa330aabc81a65 Mon Sep 17 00:00:00 2001 From: syuilo <4439005+syuilo@users.noreply.github.com> Date: Thu, 22 Jan 2026 12:14:51 +0900 Subject: [PATCH 1/7] lint --- packages/backend/src/core/QueueService.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/src/core/QueueService.ts b/packages/backend/src/core/QueueService.ts index 8ef235b238..f90ae80731 100644 --- a/packages/backend/src/core/QueueService.ts +++ b/packages/backend/src/core/QueueService.ts @@ -902,7 +902,7 @@ export class QueueService { }, db: { version: db.redis_version, - mode: db.redis_mode, + mode: db.redis_mode as 'cluster' | 'standalone' | 'sentinel', runId: db.run_id, processId: db.process_id, port: parseInt(db.tcp_port), From 2b3d72bb73e00da9b748fcfe21f0dd598511d1dc Mon Sep 17 00:00:00 2001 From: syuilo <4439005+syuilo@users.noreply.github.com> Date: Thu, 22 Jan 2026 12:16:38 +0900 Subject: [PATCH 2/7] Update measure-memory.mjs (#17116) --- packages/backend/scripts/measure-memory.mjs | 44 ++++++++++++++++----- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/packages/backend/scripts/measure-memory.mjs b/packages/backend/scripts/measure-memory.mjs index baa4198adf..4358bfee5b 100644 --- a/packages/backend/scripts/measure-memory.mjs +++ b/packages/backend/scripts/measure-memory.mjs @@ -18,12 +18,11 @@ import { dirname, join } from 'node:path'; const __filename = fileURLToPath(import.meta.url); const __dirname = dirname(__filename); +const SAMPLE_COUNT = 3; // Number of samples to measure const STARTUP_TIMEOUT = 120000; // 120 seconds timeout for server startup const MEMORY_SETTLE_TIME = 10000; // Wait 10 seconds after startup for memory to settle async function measureMemory() { - const startTime = Date.now(); - // Start the Misskey backend server using fork to enable IPC const serverProcess = fork(join(__dirname, '../built/boot/entry.js'), ['expose-gc'], { cwd: join(__dirname, '..'), @@ -107,12 +106,7 @@ async function measureMemory() { vmSize: null, }; } catch { - memoryInfo = { - rss: null, - heapUsed: null, - vmSize: null, - error: 'Could not measure memory', - }; + throw new Error('Failed to get memory usage via ps command'); } } @@ -137,15 +131,45 @@ async function measureMemory() { const result = { timestamp: new Date().toISOString(), - startupTimeMs: startupTime, memory: memoryInfo, }; + return result; +} + +async function main() { + // 直列の方が時間的に分散されて正確そうだから直列でやる + const results = []; + for (let i = 0; i < SAMPLE_COUNT; i++) { + const res = await measureMemory(); + results.push(res); + } + + // Calculate averages + const avgMemory = { + rss: 0, + heapUsed: 0, + vmSize: 0, + }; + for (const res of results) { + avgMemory.rss += res.memory.rss ?? 0; + avgMemory.heapUsed += res.memory.heapUsed ?? 0; + avgMemory.vmSize += res.memory.vmSize ?? 0; + } + avgMemory.rss = Math.round(avgMemory.rss / SAMPLE_COUNT); + avgMemory.heapUsed = Math.round(avgMemory.heapUsed / SAMPLE_COUNT); + avgMemory.vmSize = Math.round(avgMemory.vmSize / SAMPLE_COUNT); + + const result = { + timestamp: new Date().toISOString(), + memory: avgMemory, + }; + // Output as JSON to stdout console.log(JSON.stringify(result, null, 2)); } -measureMemory().catch((err) => { +main().catch((err) => { console.error(JSON.stringify({ error: err.message, timestamp: new Date().toISOString(), From f744b5711f12a76a7be98bba41d6552737593a79 Mon Sep 17 00:00:00 2001 From: syuilo <4439005+syuilo@users.noreply.github.com> Date: Thu, 22 Jan 2026 13:14:05 +0900 Subject: [PATCH 3/7] enhance(dev): improve mem report (#17117) * wip * Update report-backend-memory.yml --- .github/workflows/report-backend-memory.yml | 69 ++++++++--------- packages/backend/scripts/measure-memory.mjs | 82 ++++++++++----------- 2 files changed, 70 insertions(+), 81 deletions(-) diff --git a/.github/workflows/report-backend-memory.yml b/.github/workflows/report-backend-memory.yml index c339ca49b4..47ec652cfd 100644 --- a/.github/workflows/report-backend-memory.yml +++ b/.github/workflows/report-backend-memory.yml @@ -54,55 +54,50 @@ jobs: BASE_MEMORY=$(cat ./artifacts/memory-base.json) HEAD_MEMORY=$(cat ./artifacts/memory-head.json) - BASE_RSS=$(echo "$BASE_MEMORY" | jq -r '.memory.rss // 0') - HEAD_RSS=$(echo "$HEAD_MEMORY" | jq -r '.memory.rss // 0') + calc() { + BASE=$(echo "$BASE_MEMORY" | jq -r '.memory.'"$1"' // 0') + HEAD=$(echo "$HEAD_MEMORY" | jq -r '.memory.'"$1"' // 0') - # Calculate difference - if [ "$BASE_RSS" -gt 0 ] && [ "$HEAD_RSS" -gt 0 ]; then - DIFF=$((HEAD_RSS - BASE_RSS)) - DIFF_PERCENT=$(echo "scale=2; ($DIFF * 100) / $BASE_RSS" | bc) - - # Convert to MB for readability - BASE_MB=$(echo "scale=2; $BASE_RSS / 1048576" | bc) - HEAD_MB=$(echo "scale=2; $HEAD_RSS / 1048576" | bc) - DIFF_MB=$(echo "scale=2; $DIFF / 1048576" | bc) - - echo "base_mb=$BASE_MB" >> "$GITHUB_OUTPUT" - echo "head_mb=$HEAD_MB" >> "$GITHUB_OUTPUT" - echo "diff_mb=$DIFF_MB" >> "$GITHUB_OUTPUT" - echo "diff_percent=$DIFF_PERCENT" >> "$GITHUB_OUTPUT" - echo "has_data=true" >> "$GITHUB_OUTPUT" - - # Determine if this is a significant change (more than 5% increase) - if [ "$(echo "$DIFF_PERCENT > 5" | bc)" -eq 1 ]; then - echo "significant_increase=true" >> "$GITHUB_OUTPUT" + DIFF=$((HEAD - BASE)) + if [ "$BASE" -gt 0 ]; then + DIFF_PERCENT=$(echo "scale=2; ($DIFF * 100) / $BASE" | bc) else - echo "significant_increase=false" >> "$GITHUB_OUTPUT" + DIFF_PERCENT=0 fi - else - echo "has_data=false" >> "$GITHUB_OUTPUT" - fi + + # Convert KB to MB for readability + BASE_MB=$(echo "scale=2; $BASE / 1024" | bc) + HEAD_MB=$(echo "scale=2; $HEAD / 1024" | bc) + DIFF_MB=$(echo "scale=2; $DIFF / 1024" | bc) + + echo "$1-base=$BASE_MB" >> "$GITHUB_OUTPUT" + echo "$1-head=$HEAD_MB" >> "$GITHUB_OUTPUT" + echo "$1-diff=$DIFF_MB" >> "$GITHUB_OUTPUT" + echo "$1-diff_percent=$DIFF_PERCENT" >> "$GITHUB_OUTPUT" + } + + calc VmRSS + calc VmHWM + calc VmSize - id: build-comment name: Build memory comment run: | - HEADER="## Backend Memory Usage Comparison" + HEADER="## Backend memory usage comparison" FOOTER="[See workflow logs for details](https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID})" echo "$HEADER" > ./output.md echo >> ./output.md - if [ "${{ steps.compare.outputs.has_data }}" == "true" ]; then - echo "| Metric | base | head | Diff |" >> ./output.md - echo "|--------|------|------|------|" >> ./output.md - echo "| RSS | ${{ steps.compare.outputs.base_mb }} MB | ${{ steps.compare.outputs.head_mb }} MB | ${{ steps.compare.outputs.diff_mb }} MB (${{ steps.compare.outputs.diff_percent }}%) |" >> ./output.md - echo >> ./output.md + echo "| Metric | base | head | Diff |" >> ./output.md + echo "|--------|------|------|------|" >> ./output.md + echo "| RSS | ${{ steps.compare.outputs.VmRSS-base }} MB | ${{ steps.compare.outputs.VmRSS-head }} MB | ${{ steps.compare.outputs.VmRSS-diff }} MB (${{ steps.compare.outputs.VmRSS-diff_percent }}%) |" >> ./output.md + echo "| HWM | ${{ steps.compare.outputs.VmHWM-base }} MB | ${{ steps.compare.outputs.VmHWM-head }} MB | ${{ steps.compare.outputs.VmHWM-diff }} MB (${{ steps.compare.outputs.VmHWM-diff_percent }}%) |" >> ./output.md + echo "| VMS | ${{ steps.compare.outputs.VmSize-base }} MB | ${{ steps.compare.outputs.VmSize-head }} MB | ${{ steps.compare.outputs.VmSize-diff }} MB (${{ steps.compare.outputs.VmSize-diff_percent }}%) |" >> ./output.md + echo >> ./output.md - if [ "${{ steps.compare.outputs.significant_increase }}" == "true" ]; then - echo "⚠️ **Warning**: Memory usage has increased by more than 5%. Please verify this is not an unintended change." >> ./output.md - echo >> ./output.md - fi - else - echo "Could not retrieve memory usage data." >> ./output.md + # Determine if this is a significant change (more than 5% increase) + if [ "$(echo "${{ steps.compare.outputs.VmRSS-diff_percent }} > 5" | bc)" -eq 1 ]; then + echo "⚠️ **Warning**: Memory usage has increased by more than 5%. Please verify this is not an unintended change." >> ./output.md echo >> ./output.md fi diff --git a/packages/backend/scripts/measure-memory.mjs b/packages/backend/scripts/measure-memory.mjs index 4358bfee5b..82a5a0bf0b 100644 --- a/packages/backend/scripts/measure-memory.mjs +++ b/packages/backend/scripts/measure-memory.mjs @@ -14,6 +14,7 @@ import { fork } from 'node:child_process'; import { setTimeout } from 'node:timers/promises'; import { fileURLToPath } from 'node:url'; import { dirname, join } from 'node:path'; +import * as fs from 'node:fs/promises'; const __filename = fileURLToPath(import.meta.url); const __dirname = dirname(__filename); @@ -22,6 +23,35 @@ const SAMPLE_COUNT = 3; // Number of samples to measure const STARTUP_TIMEOUT = 120000; // 120 seconds timeout for server startup const MEMORY_SETTLE_TIME = 10000; // Wait 10 seconds after startup for memory to settle +const keys = { + VmPeak: 0, + VmSize: 0, + VmHWM: 0, + VmRSS: 0, + VmData: 0, + VmStk: 0, + VmExe: 0, + VmLib: 0, + VmPTE: 0, + VmSwap: 0, +}; + +async function getMemoryUsage(pid) { + const status = await fs.readFile(`/proc/${pid}/status`, 'utf-8'); + + const result = {}; + for (const key of Object.keys(keys)) { + const match = status.match(new RegExp(`${key}:\\s+(\\d+)\\s+kB`)); + if (match) { + result[key] = parseInt(match[1], 10); + } else { + throw new Error(`Failed to parse ${key} from /proc/${pid}/status`); + } + } + + return result; +} + async function measureMemory() { // Start the Misskey backend server using fork to enable IPC const serverProcess = fork(join(__dirname, '../built/boot/entry.js'), ['expose-gc'], { @@ -76,39 +106,7 @@ async function measureMemory() { // Get memory usage from the server process via /proc const pid = serverProcess.pid; - let memoryInfo; - - try { - const fs = await import('node:fs/promises'); - - // Read /proc/[pid]/status for detailed memory info - const status = await fs.readFile(`/proc/${pid}/status`, 'utf-8'); - const vmRssMatch = status.match(/VmRSS:\s+(\d+)\s+kB/); - const vmDataMatch = status.match(/VmData:\s+(\d+)\s+kB/); - const vmSizeMatch = status.match(/VmSize:\s+(\d+)\s+kB/); - - memoryInfo = { - rss: vmRssMatch ? parseInt(vmRssMatch[1], 10) * 1024 : null, - heapUsed: vmDataMatch ? parseInt(vmDataMatch[1], 10) * 1024 : null, - vmSize: vmSizeMatch ? parseInt(vmSizeMatch[1], 10) * 1024 : null, - }; - } catch (err) { - // Fallback: use ps command - process.stderr.write(`Warning: Could not read /proc/${pid}/status: ${err}\n`); - - const { execSync } = await import('node:child_process'); - try { - const ps = execSync(`ps -o rss= -p ${pid}`, { encoding: 'utf-8' }); - const rssKb = parseInt(ps.trim(), 10); - memoryInfo = { - rss: rssKb * 1024, - heapUsed: null, - vmSize: null, - }; - } catch { - throw new Error('Failed to get memory usage via ps command'); - } - } + const memoryInfo = await getMemoryUsage(pid); // Stop the server serverProcess.kill('SIGTERM'); @@ -146,19 +144,15 @@ async function main() { } // Calculate averages - const avgMemory = { - rss: 0, - heapUsed: 0, - vmSize: 0, - }; + const avgMemory = structuredClone(keys); for (const res of results) { - avgMemory.rss += res.memory.rss ?? 0; - avgMemory.heapUsed += res.memory.heapUsed ?? 0; - avgMemory.vmSize += res.memory.vmSize ?? 0; + for (const key of Object.keys(avgMemory)) { + avgMemory[key] += res.memory[key]; + } + } + for (const key of Object.keys(avgMemory)) { + avgMemory[key] = Math.round(avgMemory[key] / SAMPLE_COUNT); } - avgMemory.rss = Math.round(avgMemory.rss / SAMPLE_COUNT); - avgMemory.heapUsed = Math.round(avgMemory.heapUsed / SAMPLE_COUNT); - avgMemory.vmSize = Math.round(avgMemory.vmSize / SAMPLE_COUNT); const result = { timestamp: new Date().toISOString(), From 2fa6ecc7efaaf9b9d189cdd3a3ebbb9171c86078 Mon Sep 17 00:00:00 2001 From: syuilo <4439005+syuilo@users.noreply.github.com> Date: Thu, 22 Jan 2026 14:32:57 +0900 Subject: [PATCH 4/7] enhance(dev): improve mem report (#17118) * wip * wip * Update report-backend-memory.yml * Update report-backend-memory.yml * Update .github/workflows/report-backend-memory.yml Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .github/workflows/report-backend-memory.yml | 87 +++++++++++++++------ packages/backend/scripts/measure-memory.mjs | 37 ++++++--- packages/backend/src/boot/entry.ts | 14 +++- packages/backend/src/env.ts | 1 - 4 files changed, 99 insertions(+), 40 deletions(-) diff --git a/.github/workflows/report-backend-memory.yml b/.github/workflows/report-backend-memory.yml index 47ec652cfd..451a8cf9e6 100644 --- a/.github/workflows/report-backend-memory.yml +++ b/.github/workflows/report-backend-memory.yml @@ -54,31 +54,48 @@ jobs: BASE_MEMORY=$(cat ./artifacts/memory-base.json) HEAD_MEMORY=$(cat ./artifacts/memory-head.json) - calc() { - BASE=$(echo "$BASE_MEMORY" | jq -r '.memory.'"$1"' // 0') - HEAD=$(echo "$HEAD_MEMORY" | jq -r '.memory.'"$1"' // 0') + variation() { + calc() { + BASE=$(echo "$BASE_MEMORY" | jq -r ".${1}.${2} // 0") + HEAD=$(echo "$HEAD_MEMORY" | jq -r ".${1}.${2} // 0") - DIFF=$((HEAD - BASE)) - if [ "$BASE" -gt 0 ]; then - DIFF_PERCENT=$(echo "scale=2; ($DIFF * 100) / $BASE" | bc) - else - DIFF_PERCENT=0 - fi + DIFF=$((HEAD - BASE)) + if [ "$BASE" -gt 0 ]; then + DIFF_PERCENT=$(echo "scale=2; ($DIFF * 100) / $BASE" | bc) + else + DIFF_PERCENT=0 + fi - # Convert KB to MB for readability - BASE_MB=$(echo "scale=2; $BASE / 1024" | bc) - HEAD_MB=$(echo "scale=2; $HEAD / 1024" | bc) - DIFF_MB=$(echo "scale=2; $DIFF / 1024" | bc) + # Convert KB to MB for readability + BASE_MB=$(echo "scale=2; $BASE / 1024" | bc) + HEAD_MB=$(echo "scale=2; $HEAD / 1024" | bc) + DIFF_MB=$(echo "scale=2; $DIFF / 1024" | bc) - echo "$1-base=$BASE_MB" >> "$GITHUB_OUTPUT" - echo "$1-head=$HEAD_MB" >> "$GITHUB_OUTPUT" - echo "$1-diff=$DIFF_MB" >> "$GITHUB_OUTPUT" - echo "$1-diff_percent=$DIFF_PERCENT" >> "$GITHUB_OUTPUT" + JSON=$(jq -c -n \ + --arg base "$BASE_MB" \ + --arg head "$HEAD_MB" \ + --arg diff "$DIFF_MB" \ + --arg diff_percent "$DIFF_PERCENT" \ + '{base: $base, head: $head, diff: $diff, diff_percent: $diff_percent}') + + echo "$JSON" + } + + JSON=$(jq -c -n \ + --argjson VmRSS "$(calc $1 VmRSS)" \ + --argjson VmHWM "$(calc $1 VmHWM)" \ + --argjson VmSize "$(calc $1 VmSize)" \ + '{VmRSS: $VmRSS, VmHWM: $VmHWM, VmSize: $VmSize}') + + echo "$JSON" } - calc VmRSS - calc VmHWM - calc VmSize + JSON=$(jq -c -n \ + --argjson beforeGc "$(variation beforeGc)" \ + --argjson afterGc "$(variation afterGc)" \ + '{beforeGc: $beforeGc, afterGc: $afterGc}') + + echo "res=$JSON" >> "$GITHUB_OUTPUT" - id: build-comment name: Build memory comment run: | @@ -88,15 +105,33 @@ jobs: echo "$HEADER" > ./output.md echo >> ./output.md - echo "| Metric | base | head | Diff |" >> ./output.md - echo "|--------|------|------|------|" >> ./output.md - echo "| RSS | ${{ steps.compare.outputs.VmRSS-base }} MB | ${{ steps.compare.outputs.VmRSS-head }} MB | ${{ steps.compare.outputs.VmRSS-diff }} MB (${{ steps.compare.outputs.VmRSS-diff_percent }}%) |" >> ./output.md - echo "| HWM | ${{ steps.compare.outputs.VmHWM-base }} MB | ${{ steps.compare.outputs.VmHWM-head }} MB | ${{ steps.compare.outputs.VmHWM-diff }} MB (${{ steps.compare.outputs.VmHWM-diff_percent }}%) |" >> ./output.md - echo "| VMS | ${{ steps.compare.outputs.VmSize-base }} MB | ${{ steps.compare.outputs.VmSize-head }} MB | ${{ steps.compare.outputs.VmSize-diff }} MB (${{ steps.compare.outputs.VmSize-diff_percent }}%) |" >> ./output.md + table() { + line() { + BASE=$(echo "${{ steps.compare.outputs.res }}" | jq -r ".${1}.${2}.base") + HEAD=$(echo "${{ steps.compare.outputs.res }}" | jq -r ".${1}.${2}.head") + DIFF=$(echo "${{ steps.compare.outputs.res }}" | jq -r ".${1}.${2}.diff") + DIFF_PERCENT=$(echo "${{ steps.compare.outputs.res }}" | jq -r ".${1}.${2}.diff_percent") + + echo "| ${2} | ${BASE} MB | ${HEAD} MB | ${DIFF} MB (${DIFF_PERCENT}%) |" >> ./output.md + } + + echo "| Metric | base | head | Diff |" >> ./output.md + echo "|--------|------|------|------|" >> ./output.md + line $1 VmRSS + line $1 VmHWM + line $1 VmSize + } + + echo "### Before GC" >> ./output.md + table beforeGc + echo >> ./output.md + + echo "### After GC" >> ./output.md + table afterGc echo >> ./output.md # Determine if this is a significant change (more than 5% increase) - if [ "$(echo "${{ steps.compare.outputs.VmRSS-diff_percent }} > 5" | bc)" -eq 1 ]; then + if [ "$(echo "${{ steps.compare.outputs.res }}" | jq -r '.afterGc.VmRSS.diff_percent | tonumber > 5')" = "true" ]; then echo "⚠️ **Warning**: Memory usage has increased by more than 5%. Please verify this is not an unintended change." >> ./output.md echo >> ./output.md fi diff --git a/packages/backend/scripts/measure-memory.mjs b/packages/backend/scripts/measure-memory.mjs index 82a5a0bf0b..749f550bcc 100644 --- a/packages/backend/scripts/measure-memory.mjs +++ b/packages/backend/scripts/measure-memory.mjs @@ -60,9 +60,9 @@ async function measureMemory() { ...process.env, NODE_ENV: 'production', MK_DISABLE_CLUSTERING: '1', - MK_FORCE_GC: '1', }, stdio: ['pipe', 'pipe', 'pipe', 'ipc'], + execArgv: [...process.execArgv, '--expose-gc'], }); let serverReady = false; @@ -104,9 +104,21 @@ async function measureMemory() { // Wait for memory to settle await setTimeout(MEMORY_SETTLE_TIME); - // Get memory usage from the server process via /proc const pid = serverProcess.pid; - const memoryInfo = await getMemoryUsage(pid); + + const beforeGc = await getMemoryUsage(pid); + + serverProcess.send('gc'); + + await new Promise((resolve) => { + serverProcess.once('message', (message) => { + if (message === 'gc ok') resolve(); + }); + }); + + await setTimeout(1000); + + const afterGc = await getMemoryUsage(pid); // Stop the server serverProcess.kill('SIGTERM'); @@ -129,7 +141,8 @@ async function measureMemory() { const result = { timestamp: new Date().toISOString(), - memory: memoryInfo, + beforeGc, + afterGc, }; return result; @@ -144,19 +157,23 @@ async function main() { } // Calculate averages - const avgMemory = structuredClone(keys); + const beforeGc = structuredClone(keys); + const afterGc = structuredClone(keys); for (const res of results) { - for (const key of Object.keys(avgMemory)) { - avgMemory[key] += res.memory[key]; + for (const key of Object.keys(keys)) { + beforeGc[key] += res.beforeGc[key]; + afterGc[key] += res.afterGc[key]; } } - for (const key of Object.keys(avgMemory)) { - avgMemory[key] = Math.round(avgMemory[key] / SAMPLE_COUNT); + for (const key of Object.keys(keys)) { + beforeGc[key] = Math.round(beforeGc[key] / SAMPLE_COUNT); + afterGc[key] = Math.round(afterGc[key] / SAMPLE_COUNT); } const result = { timestamp: new Date().toISOString(), - memory: avgMemory, + beforeGc, + afterGc, }; // Output as JSON to stdout diff --git a/packages/backend/src/boot/entry.ts b/packages/backend/src/boot/entry.ts index 56b339b6aa..3a33d198a5 100644 --- a/packages/backend/src/boot/entry.ts +++ b/packages/backend/src/boot/entry.ts @@ -86,9 +86,17 @@ if (!envOption.disableClustering) { ev.mount(); } -if (envOption.forceGc && global.gc != null) { - global.gc(); -} +process.on('message', msg => { + if (msg === 'gc') { + if (global.gc != null) { + logger.info('Manual GC triggered'); + global.gc(); + if (process.send != null) process.send('gc ok'); + } else { + logger.warn('Manual GC requested but gc is not available. Start the process with --expose-gc to enable this feature.'); + } + } +}); readyRef.value = true; diff --git a/packages/backend/src/env.ts b/packages/backend/src/env.ts index 9957938467..ba44cfa2e6 100644 --- a/packages/backend/src/env.ts +++ b/packages/backend/src/env.ts @@ -11,7 +11,6 @@ const envOption = { verbose: false, withLogTime: false, quiet: false, - forceGc: false, }; for (const key of Object.keys(envOption) as (keyof typeof envOption)[]) { From b6e737dc7635b5f1b7c97c5aca8771f6e73561cf Mon Sep 17 00:00:00 2001 From: syuilo <4439005+syuilo@users.noreply.github.com> Date: Thu, 22 Jan 2026 14:47:05 +0900 Subject: [PATCH 5/7] Update report-backend-memory.yml --- .github/workflows/report-backend-memory.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/report-backend-memory.yml b/.github/workflows/report-backend-memory.yml index 451a8cf9e6..a668bb27a1 100644 --- a/.github/workflows/report-backend-memory.yml +++ b/.github/workflows/report-backend-memory.yml @@ -72,10 +72,10 @@ jobs: DIFF_MB=$(echo "scale=2; $DIFF / 1024" | bc) JSON=$(jq -c -n \ - --arg base "$BASE_MB" \ - --arg head "$HEAD_MB" \ - --arg diff "$DIFF_MB" \ - --arg diff_percent "$DIFF_PERCENT" \ + --argjson base "$BASE_MB" \ + --argjson head "$HEAD_MB" \ + --argjson diff "$DIFF_MB" \ + --argjson diff_percent "$DIFF_PERCENT" \ '{base: $base, head: $head, diff: $diff, diff_percent: $diff_percent}') echo "$JSON" From 1adcb03b93823886007f7f0ad09cbb517b90679b Mon Sep 17 00:00:00 2001 From: syuilo <4439005+syuilo@users.noreply.github.com> Date: Thu, 22 Jan 2026 15:01:38 +0900 Subject: [PATCH 6/7] Update report-backend-memory.yml --- .github/workflows/report-backend-memory.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/report-backend-memory.yml b/.github/workflows/report-backend-memory.yml index a668bb27a1..30918f44db 100644 --- a/.github/workflows/report-backend-memory.yml +++ b/.github/workflows/report-backend-memory.yml @@ -98,6 +98,8 @@ jobs: echo "res=$JSON" >> "$GITHUB_OUTPUT" - id: build-comment name: Build memory comment + env: + RES: ${{ steps.compare.outputs.res }} run: | HEADER="## Backend memory usage comparison" FOOTER="[See workflow logs for details](https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID})" @@ -107,10 +109,10 @@ jobs: table() { line() { - BASE=$(echo "${{ steps.compare.outputs.res }}" | jq -r ".${1}.${2}.base") - HEAD=$(echo "${{ steps.compare.outputs.res }}" | jq -r ".${1}.${2}.head") - DIFF=$(echo "${{ steps.compare.outputs.res }}" | jq -r ".${1}.${2}.diff") - DIFF_PERCENT=$(echo "${{ steps.compare.outputs.res }}" | jq -r ".${1}.${2}.diff_percent") + BASE=$(echo "$RES" | jq -r ".${1}.${2}.base") + HEAD=$(echo "$RES" | jq -r ".${1}.${2}.head") + DIFF=$(echo "$RES" | jq -r ".${1}.${2}.diff") + DIFF_PERCENT=$(echo "$RES" | jq -r ".${1}.${2}.diff_percent") echo "| ${2} | ${BASE} MB | ${HEAD} MB | ${DIFF} MB (${DIFF_PERCENT}%) |" >> ./output.md } @@ -131,7 +133,7 @@ jobs: echo >> ./output.md # Determine if this is a significant change (more than 5% increase) - if [ "$(echo "${{ steps.compare.outputs.res }}" | jq -r '.afterGc.VmRSS.diff_percent | tonumber > 5')" = "true" ]; then + if [ "$(echo "$RES" | jq -r '.afterGc.VmRSS.diff_percent | tonumber > 5')" = "true" ]; then echo "⚠️ **Warning**: Memory usage has increased by more than 5%. Please verify this is not an unintended change." >> ./output.md echo >> ./output.md fi From a168e7b6486013d05ba836d9a1d084a7708c19fb Mon Sep 17 00:00:00 2001 From: syuilo <4439005+syuilo@users.noreply.github.com> Date: Thu, 22 Jan 2026 18:53:53 +0900 Subject: [PATCH 7/7] enhance(dev): Improve mem report (#17119) * wip * Update report-backend-memory.yml * Update report-backend-memory.yml * Update measure-memory.mjs * Update report-backend-memory.yml --- .github/workflows/report-backend-memory.yml | 33 ++++++++++-- packages/backend/scripts/measure-memory.mjs | 60 +++++++++++++++++---- 2 files changed, 79 insertions(+), 14 deletions(-) diff --git a/.github/workflows/report-backend-memory.yml b/.github/workflows/report-backend-memory.yml index 30918f44db..bf2e311c83 100644 --- a/.github/workflows/report-backend-memory.yml +++ b/.github/workflows/report-backend-memory.yml @@ -85,7 +85,8 @@ jobs: --argjson VmRSS "$(calc $1 VmRSS)" \ --argjson VmHWM "$(calc $1 VmHWM)" \ --argjson VmSize "$(calc $1 VmSize)" \ - '{VmRSS: $VmRSS, VmHWM: $VmHWM, VmSize: $VmSize}') + --argjson VmData "$(calc $1 VmData)" \ + '{VmRSS: $VmRSS, VmHWM: $VmHWM, VmSize: $VmSize, VmData: $VmData}') echo "$JSON" } @@ -93,7 +94,8 @@ jobs: JSON=$(jq -c -n \ --argjson beforeGc "$(variation beforeGc)" \ --argjson afterGc "$(variation afterGc)" \ - '{beforeGc: $beforeGc, afterGc: $afterGc}') + --argjson afterRequest "$(variation afterRequest)" \ + '{beforeGc: $beforeGc, afterGc: $afterGc, afterRequest: $afterRequest}') echo "res=$JSON" >> "$GITHUB_OUTPUT" - id: build-comment @@ -108,20 +110,37 @@ jobs: echo >> ./output.md table() { + echo "| Metric | base (MB) | head (MB) | Diff (MB) | Diff (%) |" >> ./output.md + echo "|--------|------:|------:|------:|------:|" >> ./output.md + line() { + METRIC=$2 BASE=$(echo "$RES" | jq -r ".${1}.${2}.base") HEAD=$(echo "$RES" | jq -r ".${1}.${2}.head") DIFF=$(echo "$RES" | jq -r ".${1}.${2}.diff") DIFF_PERCENT=$(echo "$RES" | jq -r ".${1}.${2}.diff_percent") - echo "| ${2} | ${BASE} MB | ${HEAD} MB | ${DIFF} MB (${DIFF_PERCENT}%) |" >> ./output.md + if (( $(echo "$DIFF_PERCENT > 0" | bc -l) )); then + DIFF="+$DIFF" + DIFF_PERCENT="+$DIFF_PERCENT" + fi + + # highlight VmRSS + if [ "$2" = "VmRSS" ]; then + METRIC="**${METRIC}**" + BASE="**${BASE}**" + HEAD="**${HEAD}**" + DIFF="**${DIFF}**" + DIFF_PERCENT="**${DIFF_PERCENT}**" + fi + + echo "| ${METRIC} | ${BASE} MB | ${HEAD} MB | ${DIFF} MB | ${DIFF_PERCENT}% |" >> ./output.md } - echo "| Metric | base | head | Diff |" >> ./output.md - echo "|--------|------|------|------|" >> ./output.md line $1 VmRSS line $1 VmHWM line $1 VmSize + line $1 VmData } echo "### Before GC" >> ./output.md @@ -132,6 +151,10 @@ jobs: table afterGc echo >> ./output.md + echo "### After Request" >> ./output.md + table afterRequest + echo >> ./output.md + # Determine if this is a significant change (more than 5% increase) if [ "$(echo "$RES" | jq -r '.afterGc.VmRSS.diff_percent | tonumber > 5')" = "true" ]; then echo "⚠️ **Warning**: Memory usage has increased by more than 5%. Please verify this is not an unintended change." >> ./output.md diff --git a/packages/backend/scripts/measure-memory.mjs b/packages/backend/scripts/measure-memory.mjs index 749f550bcc..3f30e24fb4 100644 --- a/packages/backend/scripts/measure-memory.mjs +++ b/packages/backend/scripts/measure-memory.mjs @@ -14,6 +14,7 @@ import { fork } from 'node:child_process'; import { setTimeout } from 'node:timers/promises'; import { fileURLToPath } from 'node:url'; import { dirname, join } from 'node:path'; +import * as http from 'node:http'; import * as fs from 'node:fs/promises'; const __filename = fileURLToPath(import.meta.url); @@ -88,6 +89,40 @@ async function measureMemory() { process.stderr.write(`[server error] ${err}\n`); }); + async function triggerGc() { + const ok = new Promise((resolve) => { + serverProcess.once('message', (message) => { + if (message === 'gc ok') resolve(); + }); + }); + + serverProcess.send('gc'); + + await ok; + + await setTimeout(1000); + } + + function createRequest() { + return new Promise((resolve, reject) => { + const req = http.request({ + host: 'localhost', + port: 61812, + path: '/api/meta', + method: 'POST', + }, (res) => { + res.on('data', () => { }); + res.on('end', () => { + resolve(); + }); + }); + req.on('error', (err) => { + reject(err); + }); + req.end(); + }); + } + // Wait for server to be ready or timeout const startupStartTime = Date.now(); while (!serverReady) { @@ -108,18 +143,20 @@ async function measureMemory() { const beforeGc = await getMemoryUsage(pid); - serverProcess.send('gc'); - - await new Promise((resolve) => { - serverProcess.once('message', (message) => { - if (message === 'gc ok') resolve(); - }); - }); - - await setTimeout(1000); + await triggerGc(); const afterGc = await getMemoryUsage(pid); + // create some http requests to simulate load + const REQUEST_COUNT = 10; + await Promise.all( + Array.from({ length: REQUEST_COUNT }).map(() => createRequest()), + ); + + await triggerGc(); + + const afterRequest = await getMemoryUsage(pid); + // Stop the server serverProcess.kill('SIGTERM'); @@ -143,6 +180,7 @@ async function measureMemory() { timestamp: new Date().toISOString(), beforeGc, afterGc, + afterRequest, }; return result; @@ -159,21 +197,25 @@ async function main() { // Calculate averages const beforeGc = structuredClone(keys); const afterGc = structuredClone(keys); + const afterRequest = structuredClone(keys); for (const res of results) { for (const key of Object.keys(keys)) { beforeGc[key] += res.beforeGc[key]; afterGc[key] += res.afterGc[key]; + afterRequest[key] += res.afterRequest[key]; } } for (const key of Object.keys(keys)) { beforeGc[key] = Math.round(beforeGc[key] / SAMPLE_COUNT); afterGc[key] = Math.round(afterGc[key] / SAMPLE_COUNT); + afterRequest[key] = Math.round(afterRequest[key] / SAMPLE_COUNT); } const result = { timestamp: new Date().toISOString(), beforeGc, afterGc, + afterRequest, }; // Output as JSON to stdout