diff --git a/.github/workflows/report-backend-memory.yml b/.github/workflows/report-backend-memory.yml index c339ca49b4..bf2e311c83 100644 --- a/.github/workflows/report-backend-memory.yml +++ b/.github/workflows/report-backend-memory.yml @@ -54,55 +54,110 @@ jobs: BASE_MEMORY=$(cat ./artifacts/memory-base.json) HEAD_MEMORY=$(cat ./artifacts/memory-head.json) - BASE_RSS=$(echo "$BASE_MEMORY" | jq -r '.memory.rss // 0') - HEAD_RSS=$(echo "$HEAD_MEMORY" | jq -r '.memory.rss // 0') + variation() { + calc() { + BASE=$(echo "$BASE_MEMORY" | jq -r ".${1}.${2} // 0") + HEAD=$(echo "$HEAD_MEMORY" | jq -r ".${1}.${2} // 0") - # Calculate difference - if [ "$BASE_RSS" -gt 0 ] && [ "$HEAD_RSS" -gt 0 ]; then - DIFF=$((HEAD_RSS - BASE_RSS)) - DIFF_PERCENT=$(echo "scale=2; ($DIFF * 100) / $BASE_RSS" | bc) + DIFF=$((HEAD - BASE)) + if [ "$BASE" -gt 0 ]; then + DIFF_PERCENT=$(echo "scale=2; ($DIFF * 100) / $BASE" | bc) + else + DIFF_PERCENT=0 + fi - # Convert to MB for readability - BASE_MB=$(echo "scale=2; $BASE_RSS / 1048576" | bc) - HEAD_MB=$(echo "scale=2; $HEAD_RSS / 1048576" | bc) - DIFF_MB=$(echo "scale=2; $DIFF / 1048576" | bc) + # Convert KB to MB for readability + BASE_MB=$(echo "scale=2; $BASE / 1024" | bc) + HEAD_MB=$(echo "scale=2; $HEAD / 1024" | bc) + DIFF_MB=$(echo "scale=2; $DIFF / 1024" | bc) - echo "base_mb=$BASE_MB" >> "$GITHUB_OUTPUT" - echo "head_mb=$HEAD_MB" >> "$GITHUB_OUTPUT" - echo "diff_mb=$DIFF_MB" >> "$GITHUB_OUTPUT" - echo "diff_percent=$DIFF_PERCENT" >> "$GITHUB_OUTPUT" - echo "has_data=true" >> "$GITHUB_OUTPUT" + JSON=$(jq -c -n \ + --argjson base "$BASE_MB" \ + --argjson head "$HEAD_MB" \ + --argjson diff "$DIFF_MB" \ + --argjson diff_percent "$DIFF_PERCENT" \ + '{base: $base, head: $head, diff: $diff, diff_percent: $diff_percent}') - # Determine if this is a significant change (more than 5% increase) - if [ "$(echo "$DIFF_PERCENT > 5" | bc)" -eq 1 ]; then - echo "significant_increase=true" >> "$GITHUB_OUTPUT" - else - echo "significant_increase=false" >> "$GITHUB_OUTPUT" - fi - else - echo "has_data=false" >> "$GITHUB_OUTPUT" - fi + echo "$JSON" + } + + JSON=$(jq -c -n \ + --argjson VmRSS "$(calc $1 VmRSS)" \ + --argjson VmHWM "$(calc $1 VmHWM)" \ + --argjson VmSize "$(calc $1 VmSize)" \ + --argjson VmData "$(calc $1 VmData)" \ + '{VmRSS: $VmRSS, VmHWM: $VmHWM, VmSize: $VmSize, VmData: $VmData}') + + echo "$JSON" + } + + JSON=$(jq -c -n \ + --argjson beforeGc "$(variation beforeGc)" \ + --argjson afterGc "$(variation afterGc)" \ + --argjson afterRequest "$(variation afterRequest)" \ + '{beforeGc: $beforeGc, afterGc: $afterGc, afterRequest: $afterRequest}') + + echo "res=$JSON" >> "$GITHUB_OUTPUT" - id: build-comment name: Build memory comment + env: + RES: ${{ steps.compare.outputs.res }} run: | - HEADER="## Backend Memory Usage Comparison" + HEADER="## Backend memory usage comparison" FOOTER="[See workflow logs for details](https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID})" echo "$HEADER" > ./output.md echo >> ./output.md - if [ "${{ steps.compare.outputs.has_data }}" == "true" ]; then - echo "| Metric | base | head | Diff |" >> ./output.md - echo "|--------|------|------|------|" >> ./output.md - echo "| RSS | ${{ steps.compare.outputs.base_mb }} MB | ${{ steps.compare.outputs.head_mb }} MB | ${{ steps.compare.outputs.diff_mb }} MB (${{ steps.compare.outputs.diff_percent }}%) |" >> ./output.md - echo >> ./output.md + table() { + echo "| Metric | base (MB) | head (MB) | Diff (MB) | Diff (%) |" >> ./output.md + echo "|--------|------:|------:|------:|------:|" >> ./output.md - if [ "${{ steps.compare.outputs.significant_increase }}" == "true" ]; then - echo "⚠️ **Warning**: Memory usage has increased by more than 5%. Please verify this is not an unintended change." >> ./output.md - echo >> ./output.md - fi - else - echo "Could not retrieve memory usage data." >> ./output.md + line() { + METRIC=$2 + BASE=$(echo "$RES" | jq -r ".${1}.${2}.base") + HEAD=$(echo "$RES" | jq -r ".${1}.${2}.head") + DIFF=$(echo "$RES" | jq -r ".${1}.${2}.diff") + DIFF_PERCENT=$(echo "$RES" | jq -r ".${1}.${2}.diff_percent") + + if (( $(echo "$DIFF_PERCENT > 0" | bc -l) )); then + DIFF="+$DIFF" + DIFF_PERCENT="+$DIFF_PERCENT" + fi + + # highlight VmRSS + if [ "$2" = "VmRSS" ]; then + METRIC="**${METRIC}**" + BASE="**${BASE}**" + HEAD="**${HEAD}**" + DIFF="**${DIFF}**" + DIFF_PERCENT="**${DIFF_PERCENT}**" + fi + + echo "| ${METRIC} | ${BASE} MB | ${HEAD} MB | ${DIFF} MB | ${DIFF_PERCENT}% |" >> ./output.md + } + + line $1 VmRSS + line $1 VmHWM + line $1 VmSize + line $1 VmData + } + + echo "### Before GC" >> ./output.md + table beforeGc + echo >> ./output.md + + echo "### After GC" >> ./output.md + table afterGc + echo >> ./output.md + + echo "### After Request" >> ./output.md + table afterRequest + echo >> ./output.md + + # Determine if this is a significant change (more than 5% increase) + if [ "$(echo "$RES" | jq -r '.afterGc.VmRSS.diff_percent | tonumber > 5')" = "true" ]; then + echo "⚠️ **Warning**: Memory usage has increased by more than 5%. Please verify this is not an unintended change." >> ./output.md echo >> ./output.md fi diff --git a/packages/backend/scripts/measure-memory.mjs b/packages/backend/scripts/measure-memory.mjs index baa4198adf..3f30e24fb4 100644 --- a/packages/backend/scripts/measure-memory.mjs +++ b/packages/backend/scripts/measure-memory.mjs @@ -14,16 +14,46 @@ import { fork } from 'node:child_process'; import { setTimeout } from 'node:timers/promises'; import { fileURLToPath } from 'node:url'; import { dirname, join } from 'node:path'; +import * as http from 'node:http'; +import * as fs from 'node:fs/promises'; const __filename = fileURLToPath(import.meta.url); const __dirname = dirname(__filename); +const SAMPLE_COUNT = 3; // Number of samples to measure const STARTUP_TIMEOUT = 120000; // 120 seconds timeout for server startup const MEMORY_SETTLE_TIME = 10000; // Wait 10 seconds after startup for memory to settle -async function measureMemory() { - const startTime = Date.now(); +const keys = { + VmPeak: 0, + VmSize: 0, + VmHWM: 0, + VmRSS: 0, + VmData: 0, + VmStk: 0, + VmExe: 0, + VmLib: 0, + VmPTE: 0, + VmSwap: 0, +}; +async function getMemoryUsage(pid) { + const status = await fs.readFile(`/proc/${pid}/status`, 'utf-8'); + + const result = {}; + for (const key of Object.keys(keys)) { + const match = status.match(new RegExp(`${key}:\\s+(\\d+)\\s+kB`)); + if (match) { + result[key] = parseInt(match[1], 10); + } else { + throw new Error(`Failed to parse ${key} from /proc/${pid}/status`); + } + } + + return result; +} + +async function measureMemory() { // Start the Misskey backend server using fork to enable IPC const serverProcess = fork(join(__dirname, '../built/boot/entry.js'), ['expose-gc'], { cwd: join(__dirname, '..'), @@ -31,9 +61,9 @@ async function measureMemory() { ...process.env, NODE_ENV: 'production', MK_DISABLE_CLUSTERING: '1', - MK_FORCE_GC: '1', }, stdio: ['pipe', 'pipe', 'pipe', 'ipc'], + execArgv: [...process.execArgv, '--expose-gc'], }); let serverReady = false; @@ -59,6 +89,40 @@ async function measureMemory() { process.stderr.write(`[server error] ${err}\n`); }); + async function triggerGc() { + const ok = new Promise((resolve) => { + serverProcess.once('message', (message) => { + if (message === 'gc ok') resolve(); + }); + }); + + serverProcess.send('gc'); + + await ok; + + await setTimeout(1000); + } + + function createRequest() { + return new Promise((resolve, reject) => { + const req = http.request({ + host: 'localhost', + port: 61812, + path: '/api/meta', + method: 'POST', + }, (res) => { + res.on('data', () => { }); + res.on('end', () => { + resolve(); + }); + }); + req.on('error', (err) => { + reject(err); + }); + req.end(); + }); + } + // Wait for server to be ready or timeout const startupStartTime = Date.now(); while (!serverReady) { @@ -75,46 +139,23 @@ async function measureMemory() { // Wait for memory to settle await setTimeout(MEMORY_SETTLE_TIME); - // Get memory usage from the server process via /proc const pid = serverProcess.pid; - let memoryInfo; - try { - const fs = await import('node:fs/promises'); + const beforeGc = await getMemoryUsage(pid); - // Read /proc/[pid]/status for detailed memory info - const status = await fs.readFile(`/proc/${pid}/status`, 'utf-8'); - const vmRssMatch = status.match(/VmRSS:\s+(\d+)\s+kB/); - const vmDataMatch = status.match(/VmData:\s+(\d+)\s+kB/); - const vmSizeMatch = status.match(/VmSize:\s+(\d+)\s+kB/); + await triggerGc(); - memoryInfo = { - rss: vmRssMatch ? parseInt(vmRssMatch[1], 10) * 1024 : null, - heapUsed: vmDataMatch ? parseInt(vmDataMatch[1], 10) * 1024 : null, - vmSize: vmSizeMatch ? parseInt(vmSizeMatch[1], 10) * 1024 : null, - }; - } catch (err) { - // Fallback: use ps command - process.stderr.write(`Warning: Could not read /proc/${pid}/status: ${err}\n`); + const afterGc = await getMemoryUsage(pid); - const { execSync } = await import('node:child_process'); - try { - const ps = execSync(`ps -o rss= -p ${pid}`, { encoding: 'utf-8' }); - const rssKb = parseInt(ps.trim(), 10); - memoryInfo = { - rss: rssKb * 1024, - heapUsed: null, - vmSize: null, - }; - } catch { - memoryInfo = { - rss: null, - heapUsed: null, - vmSize: null, - error: 'Could not measure memory', - }; - } - } + // create some http requests to simulate load + const REQUEST_COUNT = 10; + await Promise.all( + Array.from({ length: REQUEST_COUNT }).map(() => createRequest()), + ); + + await triggerGc(); + + const afterRequest = await getMemoryUsage(pid); // Stop the server serverProcess.kill('SIGTERM'); @@ -137,15 +178,51 @@ async function measureMemory() { const result = { timestamp: new Date().toISOString(), - startupTimeMs: startupTime, - memory: memoryInfo, + beforeGc, + afterGc, + afterRequest, + }; + + return result; +} + +async function main() { + // 直列の方が時間的に分散されて正確そうだから直列でやる + const results = []; + for (let i = 0; i < SAMPLE_COUNT; i++) { + const res = await measureMemory(); + results.push(res); + } + + // Calculate averages + const beforeGc = structuredClone(keys); + const afterGc = structuredClone(keys); + const afterRequest = structuredClone(keys); + for (const res of results) { + for (const key of Object.keys(keys)) { + beforeGc[key] += res.beforeGc[key]; + afterGc[key] += res.afterGc[key]; + afterRequest[key] += res.afterRequest[key]; + } + } + for (const key of Object.keys(keys)) { + beforeGc[key] = Math.round(beforeGc[key] / SAMPLE_COUNT); + afterGc[key] = Math.round(afterGc[key] / SAMPLE_COUNT); + afterRequest[key] = Math.round(afterRequest[key] / SAMPLE_COUNT); + } + + const result = { + timestamp: new Date().toISOString(), + beforeGc, + afterGc, + afterRequest, }; // Output as JSON to stdout console.log(JSON.stringify(result, null, 2)); } -measureMemory().catch((err) => { +main().catch((err) => { console.error(JSON.stringify({ error: err.message, timestamp: new Date().toISOString(), diff --git a/packages/backend/src/boot/entry.ts b/packages/backend/src/boot/entry.ts index 56b339b6aa..3a33d198a5 100644 --- a/packages/backend/src/boot/entry.ts +++ b/packages/backend/src/boot/entry.ts @@ -86,9 +86,17 @@ if (!envOption.disableClustering) { ev.mount(); } -if (envOption.forceGc && global.gc != null) { - global.gc(); -} +process.on('message', msg => { + if (msg === 'gc') { + if (global.gc != null) { + logger.info('Manual GC triggered'); + global.gc(); + if (process.send != null) process.send('gc ok'); + } else { + logger.warn('Manual GC requested but gc is not available. Start the process with --expose-gc to enable this feature.'); + } + } +}); readyRef.value = true; diff --git a/packages/backend/src/core/QueueService.ts b/packages/backend/src/core/QueueService.ts index 8ef235b238..f90ae80731 100644 --- a/packages/backend/src/core/QueueService.ts +++ b/packages/backend/src/core/QueueService.ts @@ -902,7 +902,7 @@ export class QueueService { }, db: { version: db.redis_version, - mode: db.redis_mode, + mode: db.redis_mode as 'cluster' | 'standalone' | 'sentinel', runId: db.run_id, processId: db.process_id, port: parseInt(db.tcp_port), diff --git a/packages/backend/src/env.ts b/packages/backend/src/env.ts index 9957938467..ba44cfa2e6 100644 --- a/packages/backend/src/env.ts +++ b/packages/backend/src/env.ts @@ -11,7 +11,6 @@ const envOption = { verbose: false, withLogTime: false, quiet: false, - forceGc: false, }; for (const key of Object.keys(envOption) as (keyof typeof envOption)[]) {