diff --git a/.github/workflows/report-backend-memory.yml b/.github/workflows/report-backend-memory.yml index c339ca49b4..47ec652cfd 100644 --- a/.github/workflows/report-backend-memory.yml +++ b/.github/workflows/report-backend-memory.yml @@ -54,55 +54,50 @@ jobs: BASE_MEMORY=$(cat ./artifacts/memory-base.json) HEAD_MEMORY=$(cat ./artifacts/memory-head.json) - BASE_RSS=$(echo "$BASE_MEMORY" | jq -r '.memory.rss // 0') - HEAD_RSS=$(echo "$HEAD_MEMORY" | jq -r '.memory.rss // 0') + calc() { + BASE=$(echo "$BASE_MEMORY" | jq -r '.memory.'"$1"' // 0') + HEAD=$(echo "$HEAD_MEMORY" | jq -r '.memory.'"$1"' // 0') - # Calculate difference - if [ "$BASE_RSS" -gt 0 ] && [ "$HEAD_RSS" -gt 0 ]; then - DIFF=$((HEAD_RSS - BASE_RSS)) - DIFF_PERCENT=$(echo "scale=2; ($DIFF * 100) / $BASE_RSS" | bc) - - # Convert to MB for readability - BASE_MB=$(echo "scale=2; $BASE_RSS / 1048576" | bc) - HEAD_MB=$(echo "scale=2; $HEAD_RSS / 1048576" | bc) - DIFF_MB=$(echo "scale=2; $DIFF / 1048576" | bc) - - echo "base_mb=$BASE_MB" >> "$GITHUB_OUTPUT" - echo "head_mb=$HEAD_MB" >> "$GITHUB_OUTPUT" - echo "diff_mb=$DIFF_MB" >> "$GITHUB_OUTPUT" - echo "diff_percent=$DIFF_PERCENT" >> "$GITHUB_OUTPUT" - echo "has_data=true" >> "$GITHUB_OUTPUT" - - # Determine if this is a significant change (more than 5% increase) - if [ "$(echo "$DIFF_PERCENT > 5" | bc)" -eq 1 ]; then - echo "significant_increase=true" >> "$GITHUB_OUTPUT" + DIFF=$((HEAD - BASE)) + if [ "$BASE" -gt 0 ]; then + DIFF_PERCENT=$(echo "scale=2; ($DIFF * 100) / $BASE" | bc) else - echo "significant_increase=false" >> "$GITHUB_OUTPUT" + DIFF_PERCENT=0 fi - else - echo "has_data=false" >> "$GITHUB_OUTPUT" - fi + + # Convert KB to MB for readability + BASE_MB=$(echo "scale=2; $BASE / 1024" | bc) + HEAD_MB=$(echo "scale=2; $HEAD / 1024" | bc) + DIFF_MB=$(echo "scale=2; $DIFF / 1024" | bc) + + echo "$1-base=$BASE_MB" >> "$GITHUB_OUTPUT" + echo "$1-head=$HEAD_MB" >> "$GITHUB_OUTPUT" + echo "$1-diff=$DIFF_MB" >> "$GITHUB_OUTPUT" + echo "$1-diff_percent=$DIFF_PERCENT" >> "$GITHUB_OUTPUT" + } + + calc VmRSS + calc VmHWM + calc VmSize - id: build-comment name: Build memory comment run: | - HEADER="## Backend Memory Usage Comparison" + HEADER="## Backend memory usage comparison" FOOTER="[See workflow logs for details](https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID})" echo "$HEADER" > ./output.md echo >> ./output.md - if [ "${{ steps.compare.outputs.has_data }}" == "true" ]; then - echo "| Metric | base | head | Diff |" >> ./output.md - echo "|--------|------|------|------|" >> ./output.md - echo "| RSS | ${{ steps.compare.outputs.base_mb }} MB | ${{ steps.compare.outputs.head_mb }} MB | ${{ steps.compare.outputs.diff_mb }} MB (${{ steps.compare.outputs.diff_percent }}%) |" >> ./output.md - echo >> ./output.md + echo "| Metric | base | head | Diff |" >> ./output.md + echo "|--------|------|------|------|" >> ./output.md + echo "| RSS | ${{ steps.compare.outputs.VmRSS-base }} MB | ${{ steps.compare.outputs.VmRSS-head }} MB | ${{ steps.compare.outputs.VmRSS-diff }} MB (${{ steps.compare.outputs.VmRSS-diff_percent }}%) |" >> ./output.md + echo "| HWM | ${{ steps.compare.outputs.VmHWM-base }} MB | ${{ steps.compare.outputs.VmHWM-head }} MB | ${{ steps.compare.outputs.VmHWM-diff }} MB (${{ steps.compare.outputs.VmHWM-diff_percent }}%) |" >> ./output.md + echo "| VMS | ${{ steps.compare.outputs.VmSize-base }} MB | ${{ steps.compare.outputs.VmSize-head }} MB | ${{ steps.compare.outputs.VmSize-diff }} MB (${{ steps.compare.outputs.VmSize-diff_percent }}%) |" >> ./output.md + echo >> ./output.md - if [ "${{ steps.compare.outputs.significant_increase }}" == "true" ]; then - echo "⚠️ **Warning**: Memory usage has increased by more than 5%. Please verify this is not an unintended change." >> ./output.md - echo >> ./output.md - fi - else - echo "Could not retrieve memory usage data." >> ./output.md + # Determine if this is a significant change (more than 5% increase) + if [ "$(echo "${{ steps.compare.outputs.VmRSS-diff_percent }} > 5" | bc)" -eq 1 ]; then + echo "⚠️ **Warning**: Memory usage has increased by more than 5%. Please verify this is not an unintended change." >> ./output.md echo >> ./output.md fi diff --git a/packages/backend/scripts/measure-memory.mjs b/packages/backend/scripts/measure-memory.mjs index 4358bfee5b..82a5a0bf0b 100644 --- a/packages/backend/scripts/measure-memory.mjs +++ b/packages/backend/scripts/measure-memory.mjs @@ -14,6 +14,7 @@ import { fork } from 'node:child_process'; import { setTimeout } from 'node:timers/promises'; import { fileURLToPath } from 'node:url'; import { dirname, join } from 'node:path'; +import * as fs from 'node:fs/promises'; const __filename = fileURLToPath(import.meta.url); const __dirname = dirname(__filename); @@ -22,6 +23,35 @@ const SAMPLE_COUNT = 3; // Number of samples to measure const STARTUP_TIMEOUT = 120000; // 120 seconds timeout for server startup const MEMORY_SETTLE_TIME = 10000; // Wait 10 seconds after startup for memory to settle +const keys = { + VmPeak: 0, + VmSize: 0, + VmHWM: 0, + VmRSS: 0, + VmData: 0, + VmStk: 0, + VmExe: 0, + VmLib: 0, + VmPTE: 0, + VmSwap: 0, +}; + +async function getMemoryUsage(pid) { + const status = await fs.readFile(`/proc/${pid}/status`, 'utf-8'); + + const result = {}; + for (const key of Object.keys(keys)) { + const match = status.match(new RegExp(`${key}:\\s+(\\d+)\\s+kB`)); + if (match) { + result[key] = parseInt(match[1], 10); + } else { + throw new Error(`Failed to parse ${key} from /proc/${pid}/status`); + } + } + + return result; +} + async function measureMemory() { // Start the Misskey backend server using fork to enable IPC const serverProcess = fork(join(__dirname, '../built/boot/entry.js'), ['expose-gc'], { @@ -76,39 +106,7 @@ async function measureMemory() { // Get memory usage from the server process via /proc const pid = serverProcess.pid; - let memoryInfo; - - try { - const fs = await import('node:fs/promises'); - - // Read /proc/[pid]/status for detailed memory info - const status = await fs.readFile(`/proc/${pid}/status`, 'utf-8'); - const vmRssMatch = status.match(/VmRSS:\s+(\d+)\s+kB/); - const vmDataMatch = status.match(/VmData:\s+(\d+)\s+kB/); - const vmSizeMatch = status.match(/VmSize:\s+(\d+)\s+kB/); - - memoryInfo = { - rss: vmRssMatch ? parseInt(vmRssMatch[1], 10) * 1024 : null, - heapUsed: vmDataMatch ? parseInt(vmDataMatch[1], 10) * 1024 : null, - vmSize: vmSizeMatch ? parseInt(vmSizeMatch[1], 10) * 1024 : null, - }; - } catch (err) { - // Fallback: use ps command - process.stderr.write(`Warning: Could not read /proc/${pid}/status: ${err}\n`); - - const { execSync } = await import('node:child_process'); - try { - const ps = execSync(`ps -o rss= -p ${pid}`, { encoding: 'utf-8' }); - const rssKb = parseInt(ps.trim(), 10); - memoryInfo = { - rss: rssKb * 1024, - heapUsed: null, - vmSize: null, - }; - } catch { - throw new Error('Failed to get memory usage via ps command'); - } - } + const memoryInfo = await getMemoryUsage(pid); // Stop the server serverProcess.kill('SIGTERM'); @@ -146,19 +144,15 @@ async function main() { } // Calculate averages - const avgMemory = { - rss: 0, - heapUsed: 0, - vmSize: 0, - }; + const avgMemory = structuredClone(keys); for (const res of results) { - avgMemory.rss += res.memory.rss ?? 0; - avgMemory.heapUsed += res.memory.heapUsed ?? 0; - avgMemory.vmSize += res.memory.vmSize ?? 0; + for (const key of Object.keys(avgMemory)) { + avgMemory[key] += res.memory[key]; + } + } + for (const key of Object.keys(avgMemory)) { + avgMemory[key] = Math.round(avgMemory[key] / SAMPLE_COUNT); } - avgMemory.rss = Math.round(avgMemory.rss / SAMPLE_COUNT); - avgMemory.heapUsed = Math.round(avgMemory.heapUsed / SAMPLE_COUNT); - avgMemory.vmSize = Math.round(avgMemory.vmSize / SAMPLE_COUNT); const result = { timestamp: new Date().toISOString(),