usbharu
/
misskey
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

enhance(dev): improve mem report (#17117) 

* wip

* Update report-backend-memory.yml
This commit is contained in:
syuilo 2026-01-22 13:14:05 +09:00 committed by GitHub
parent 2b3d72bb73
commit f744b5711f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 70 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
.github/workflows/report-backend-memory.yml vendored
View File

@ -54,55 +54,50 @@ jobs:
BASE_MEMORY=$(cat ./artifacts/memory-base.json) BASE_MEMORY=$(cat ./artifacts/memory-base.json)
HEAD_MEMORY=$(cat ./artifacts/memory-head.json) HEAD_MEMORY=$(cat ./artifacts/memory-head.json)
BASE_RSS=$(echo "$BASE_MEMORY" | jq -r '.memory.rss // 0') calc() {
HEAD_RSS=$(echo "$HEAD_MEMORY" | jq -r '.memory.rss // 0') BASE=$(echo "$BASE_MEMORY" | jq -r '.memory.'"$1"' // 0')
HEAD=$(echo "$HEAD_MEMORY" | jq -r '.memory.'"$1"' // 0')
# Calculate difference DIFF=$((HEAD - BASE))
if [ "$BASE_RSS" -gt 0 ] && [ "$HEAD_RSS" -gt 0 ]; then if [ "$BASE" -gt 0 ]; then
DIFF=$((HEAD_RSS - BASE_RSS)) DIFF_PERCENT=$(echo "scale=2; ($DIFF * 100) / $BASE" | bc)
DIFF_PERCENT=$(echo "scale=2; ($DIFF * 100) / $BASE_RSS" | bc)
# Convert to MB for readability
BASE_MB=$(echo "scale=2; $BASE_RSS / 1048576" | bc)
HEAD_MB=$(echo "scale=2; $HEAD_RSS / 1048576" | bc)
DIFF_MB=$(echo "scale=2; $DIFF / 1048576" | bc)
echo "base_mb=$BASE_MB" >> "$GITHUB_OUTPUT"
echo "head_mb=$HEAD_MB" >> "$GITHUB_OUTPUT"
echo "diff_mb=$DIFF_MB" >> "$GITHUB_OUTPUT"
echo "diff_percent=$DIFF_PERCENT" >> "$GITHUB_OUTPUT"
echo "has_data=true" >> "$GITHUB_OUTPUT"
# Determine if this is a significant change (more than 5% increase)
if [ "$(echo "$DIFF_PERCENT > 5" | bc)" -eq 1 ]; then
echo "significant_increase=true" >> "$GITHUB_OUTPUT"
else else
echo "significant_increase=false" >> "$GITHUB_OUTPUT" DIFF_PERCENT=0
fi fi
else
echo "has_data=false" >> "$GITHUB_OUTPUT" # Convert KB to MB for readability
fi BASE_MB=$(echo "scale=2; $BASE / 1024" | bc)
HEAD_MB=$(echo "scale=2; $HEAD / 1024" | bc)
DIFF_MB=$(echo "scale=2; $DIFF / 1024" | bc)
echo "$1-base=$BASE_MB" >> "$GITHUB_OUTPUT"
echo "$1-head=$HEAD_MB" >> "$GITHUB_OUTPUT"
echo "$1-diff=$DIFF_MB" >> "$GITHUB_OUTPUT"
echo "$1-diff_percent=$DIFF_PERCENT" >> "$GITHUB_OUTPUT"
}
calc VmRSS
calc VmHWM
calc VmSize
- id: build-comment - id: build-comment
name: Build memory comment name: Build memory comment
run: | run: |
HEADER="## Backend Memory Usage Comparison" HEADER="## Backend memory usage comparison"
FOOTER="[See workflow logs for details](https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID})" FOOTER="[See workflow logs for details](https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID})"
echo "$HEADER" > ./output.md echo "$HEADER" > ./output.md
echo >> ./output.md echo >> ./output.md
if [ "${{ steps.compare.outputs.has_data }}" == "true" ]; then echo "| Metric | base | head | Diff |" >> ./output.md
echo "| Metric | base | head | Diff |" >> ./output.md echo "|--------|------|------|------|" >> ./output.md
echo "|--------|------|------|------|" >> ./output.md echo "| RSS | ${{ steps.compare.outputs.VmRSS-base }} MB | ${{ steps.compare.outputs.VmRSS-head }} MB | ${{ steps.compare.outputs.VmRSS-diff }} MB (${{ steps.compare.outputs.VmRSS-diff_percent }}%) |" >> ./output.md
echo "| RSS | ${{ steps.compare.outputs.base_mb }} MB | ${{ steps.compare.outputs.head_mb }} MB | ${{ steps.compare.outputs.diff_mb }} MB (${{ steps.compare.outputs.diff_percent }}%) |" >> ./output.md echo "| HWM | ${{ steps.compare.outputs.VmHWM-base }} MB | ${{ steps.compare.outputs.VmHWM-head }} MB | ${{ steps.compare.outputs.VmHWM-diff }} MB (${{ steps.compare.outputs.VmHWM-diff_percent }}%) |" >> ./output.md
echo >> ./output.md echo "| VMS | ${{ steps.compare.outputs.VmSize-base }} MB | ${{ steps.compare.outputs.VmSize-head }} MB | ${{ steps.compare.outputs.VmSize-diff }} MB (${{ steps.compare.outputs.VmSize-diff_percent }}%) |" >> ./output.md
echo >> ./output.md
if [ "${{ steps.compare.outputs.significant_increase }}" == "true" ]; then # Determine if this is a significant change (more than 5% increase)
echo "⚠️ **Warning**: Memory usage has increased by more than 5%. Please verify this is not an unintended change." >> ./output.md if [ "$(echo "${{ steps.compare.outputs.VmRSS-diff_percent }} > 5" | bc)" -eq 1 ]; then
echo >> ./output.md echo "⚠️ **Warning**: Memory usage has increased by more than 5%. Please verify this is not an unintended change." >> ./output.md
fi
else
echo "Could not retrieve memory usage data." >> ./output.md
echo >> ./output.md echo >> ./output.md
fi fi

82
packages/backend/scripts/measure-memory.mjs
View File

@ -14,6 +14,7 @@ import { fork } from 'node:child_process';
import { setTimeout } from 'node:timers/promises'; import { setTimeout } from 'node:timers/promises';
import { fileURLToPath } from 'node:url'; import { fileURLToPath } from 'node:url';
import { dirname, join } from 'node:path'; import { dirname, join } from 'node:path';
import * as fs from 'node:fs/promises';
const __filename = fileURLToPath(import.meta.url); const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename); const __dirname = dirname(__filename);
@ -22,6 +23,35 @@ const SAMPLE_COUNT = 3; // Number of samples to measure
const STARTUP_TIMEOUT = 120000; // 120 seconds timeout for server startup const STARTUP_TIMEOUT = 120000; // 120 seconds timeout for server startup
const MEMORY_SETTLE_TIME = 10000; // Wait 10 seconds after startup for memory to settle const MEMORY_SETTLE_TIME = 10000; // Wait 10 seconds after startup for memory to settle
const keys = {
VmPeak: 0,
VmSize: 0,
VmHWM: 0,
VmRSS: 0,
VmData: 0,
VmStk: 0,
VmExe: 0,
VmLib: 0,
VmPTE: 0,
VmSwap: 0,
};
async function getMemoryUsage(pid) {
const status = await fs.readFile(`/proc/${pid}/status`, 'utf-8');
const result = {};
for (const key of Object.keys(keys)) {
const match = status.match(new RegExp(`${key}:\\s+(\\d+)\\s+kB`));
if (match) {
result[key] = parseInt(match[1], 10);
} else {
throw new Error(`Failed to parse ${key} from /proc/${pid}/status`);
}
}
return result;
}
async function measureMemory() { async function measureMemory() {
// Start the Misskey backend server using fork to enable IPC // Start the Misskey backend server using fork to enable IPC
const serverProcess = fork(join(__dirname, '../built/boot/entry.js'), ['expose-gc'], { const serverProcess = fork(join(__dirname, '../built/boot/entry.js'), ['expose-gc'], {
@ -76,39 +106,7 @@ async function measureMemory() {
// Get memory usage from the server process via /proc // Get memory usage from the server process via /proc
const pid = serverProcess.pid; const pid = serverProcess.pid;
let memoryInfo; const memoryInfo = await getMemoryUsage(pid);
try {
const fs = await import('node:fs/promises');
// Read /proc/[pid]/status for detailed memory info
const status = await fs.readFile(`/proc/${pid}/status`, 'utf-8');
const vmRssMatch = status.match(/VmRSS:\s+(\d+)\s+kB/);
const vmDataMatch = status.match(/VmData:\s+(\d+)\s+kB/);
const vmSizeMatch = status.match(/VmSize:\s+(\d+)\s+kB/);
memoryInfo = {
rss: vmRssMatch ? parseInt(vmRssMatch[1], 10) * 1024 : null,
heapUsed: vmDataMatch ? parseInt(vmDataMatch[1], 10) * 1024 : null,
vmSize: vmSizeMatch ? parseInt(vmSizeMatch[1], 10) * 1024 : null,
};
} catch (err) {
// Fallback: use ps command
process.stderr.write(`Warning: Could not read /proc/${pid}/status: ${err}\n`);
const { execSync } = await import('node:child_process');
try {
const ps = execSync(`ps -o rss= -p ${pid}`, { encoding: 'utf-8' });
const rssKb = parseInt(ps.trim(), 10);
memoryInfo = {
rss: rssKb * 1024,
heapUsed: null,
vmSize: null,
};
} catch {
throw new Error('Failed to get memory usage via ps command');
}
}
// Stop the server // Stop the server
serverProcess.kill('SIGTERM'); serverProcess.kill('SIGTERM');
@ -146,19 +144,15 @@ async function main() {
} }
// Calculate averages // Calculate averages
const avgMemory = { const avgMemory = structuredClone(keys);
rss: 0,
heapUsed: 0,
vmSize: 0,
};
for (const res of results) { for (const res of results) {
avgMemory.rss += res.memory.rss ?? 0; for (const key of Object.keys(avgMemory)) {
avgMemory.heapUsed += res.memory.heapUsed ?? 0; avgMemory[key] += res.memory[key];
avgMemory.vmSize += res.memory.vmSize ?? 0; }
}
for (const key of Object.keys(avgMemory)) {
avgMemory[key] = Math.round(avgMemory[key] / SAMPLE_COUNT);
} }
avgMemory.rss = Math.round(avgMemory.rss / SAMPLE_COUNT);
avgMemory.heapUsed = Math.round(avgMemory.heapUsed / SAMPLE_COUNT);
avgMemory.vmSize = Math.round(avgMemory.vmSize / SAMPLE_COUNT);
const result = { const result = {
timestamp: new Date().toISOString(), timestamp: new Date().toISOString(),