Merge 389861f1da into 0d46089f9a

.github/workflows/report-backend-memory.yml

@@ -54,50 +54,55 @@ jobs:
           BASE_MEMORY=$(cat ./artifacts/memory-base.json)
           HEAD_MEMORY=$(cat ./artifacts/memory-head.json)
 
-          calc() {
-            BASE=$(echo "$BASE_MEMORY" | jq -r '.memory.'"$1"' // 0')
-            HEAD=$(echo "$HEAD_MEMORY" | jq -r '.memory.'"$1"' // 0')
+          BASE_RSS=$(echo "$BASE_MEMORY" | jq -r '.memory.rss // 0')
+          HEAD_RSS=$(echo "$HEAD_MEMORY" | jq -r '.memory.rss // 0')
 
-            DIFF=$((HEAD - BASE))
-            if [ "$BASE" -gt 0 ]; then
-              DIFF_PERCENT=$(echo "scale=2; ($DIFF * 100) / $BASE" | bc)
+          # Calculate difference
+          if [ "$BASE_RSS" -gt 0 ] && [ "$HEAD_RSS" -gt 0 ]; then
+            DIFF=$((HEAD_RSS - BASE_RSS))
+            DIFF_PERCENT=$(echo "scale=2; ($DIFF * 100) / $BASE_RSS" | bc)
+
+            # Convert to MB for readability
+            BASE_MB=$(echo "scale=2; $BASE_RSS / 1048576" | bc)
+            HEAD_MB=$(echo "scale=2; $HEAD_RSS / 1048576" | bc)
+            DIFF_MB=$(echo "scale=2; $DIFF / 1048576" | bc)
+
+            echo "base_mb=$BASE_MB" >> "$GITHUB_OUTPUT"
+            echo "head_mb=$HEAD_MB" >> "$GITHUB_OUTPUT"
+            echo "diff_mb=$DIFF_MB" >> "$GITHUB_OUTPUT"
+            echo "diff_percent=$DIFF_PERCENT" >> "$GITHUB_OUTPUT"
+            echo "has_data=true" >> "$GITHUB_OUTPUT"
+
+            # Determine if this is a significant change (more than 5% increase)
+            if [ "$(echo "$DIFF_PERCENT > 5" | bc)" -eq 1 ]; then
+              echo "significant_increase=true" >> "$GITHUB_OUTPUT"
             else
-              DIFF_PERCENT=0
+              echo "significant_increase=false" >> "$GITHUB_OUTPUT"
             fi
-
-            # Convert KB to MB for readability
-            BASE_MB=$(echo "scale=2; $BASE / 1024" | bc)
-            HEAD_MB=$(echo "scale=2; $HEAD / 1024" | bc)
-            DIFF_MB=$(echo "scale=2; $DIFF / 1024" | bc)
-
-            echo "$1-base=$BASE_MB" >> "$GITHUB_OUTPUT"
-            echo "$1-head=$HEAD_MB" >> "$GITHUB_OUTPUT"
-            echo "$1-diff=$DIFF_MB" >> "$GITHUB_OUTPUT"
-            echo "$1-diff_percent=$DIFF_PERCENT" >> "$GITHUB_OUTPUT"
-          }
-
-          calc VmRSS
-          calc VmHWM
-          calc VmSize
+          else
+            echo "has_data=false" >> "$GITHUB_OUTPUT"
+          fi
       - id: build-comment
         name: Build memory comment
         run: |
-          HEADER="## Backend memory usage comparison"
+          HEADER="## Backend Memory Usage Comparison"
           FOOTER="[See workflow logs for details](https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID})"
 
           echo "$HEADER" > ./output.md
           echo >> ./output.md
 
-          echo "| Metric | base | head | Diff |" >> ./output.md
-          echo "|--------|------|------|------|" >> ./output.md
-          echo "| RSS | ${{ steps.compare.outputs.VmRSS-base }} MB | ${{ steps.compare.outputs.VmRSS-head }} MB | ${{ steps.compare.outputs.VmRSS-diff }} MB (${{ steps.compare.outputs.VmRSS-diff_percent }}%) |" >> ./output.md
-          echo "| HWM | ${{ steps.compare.outputs.VmHWM-base }} MB | ${{ steps.compare.outputs.VmHWM-head }} MB | ${{ steps.compare.outputs.VmHWM-diff }} MB (${{ steps.compare.outputs.VmHWM-diff_percent }}%) |" >> ./output.md
-          echo "| VMS | ${{ steps.compare.outputs.VmSize-base }} MB | ${{ steps.compare.outputs.VmSize-head }} MB | ${{ steps.compare.outputs.VmSize-diff }} MB (${{ steps.compare.outputs.VmSize-diff_percent }}%) |" >> ./output.md
-          echo >> ./output.md
+          if [ "${{ steps.compare.outputs.has_data }}" == "true" ]; then
+            echo "| Metric | base | head | Diff |" >> ./output.md
+            echo "|--------|------|------|------|" >> ./output.md
+            echo "| RSS | ${{ steps.compare.outputs.base_mb }} MB | ${{ steps.compare.outputs.head_mb }} MB | ${{ steps.compare.outputs.diff_mb }} MB (${{ steps.compare.outputs.diff_percent }}%) |" >> ./output.md
+            echo >> ./output.md
 
-          # Determine if this is a significant change (more than 5% increase)
-          if [ "$(echo "${{ steps.compare.outputs.VmRSS-diff_percent }} > 5" | bc)" -eq 1 ]; then
-            echo "⚠️ **Warning**: Memory usage has increased by more than 5%. Please verify this is not an unintended change." >> ./output.md
+            if [ "${{ steps.compare.outputs.significant_increase }}" == "true" ]; then
+              echo "⚠️ **Warning**: Memory usage has increased by more than 5%. Please verify this is not an unintended change." >> ./output.md
+              echo >> ./output.md
+            fi
+          else
+            echo "Could not retrieve memory usage data." >> ./output.md
             echo >> ./output.md
           fi
 

packages/backend/package.json

@@ -96,6 +96,7 @@
 		"@swc/cli": "0.7.9",
 		"@swc/core": "1.15.7",
 		"@twemoji/parser": "16.0.0",
+		"@types/redis-info": "3.0.3",
 		"accepts": "1.3.8",
 		"ajv": "8.17.1",
 		"archiver": "7.0.1",
@@ -153,6 +154,7 @@
 		"random-seed": "0.3.0",
 		"ratelimiter": "3.4.1",
 		"re2": "1.23.0",
+		"redis-info": "3.1.0",
 		"reflect-metadata": "0.2.2",
 		"rename": "1.0.4",
 		"rss-parser": "3.13.0",

packages/backend/scripts/measure-memory.mjs

@@ -14,45 +14,16 @@ import { fork } from 'node:child_process';
 import { setTimeout } from 'node:timers/promises';
 import { fileURLToPath } from 'node:url';
 import { dirname, join } from 'node:path';
-import * as fs from 'node:fs/promises';
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = dirname(__filename);
 
-const SAMPLE_COUNT = 3; // Number of samples to measure
 const STARTUP_TIMEOUT = 120000; // 120 seconds timeout for server startup
 const MEMORY_SETTLE_TIME = 10000; // Wait 10 seconds after startup for memory to settle
 
-const keys = {
-	VmPeak: 0,
-	VmSize: 0,
-	VmHWM: 0,
-	VmRSS: 0,
-	VmData: 0,
-	VmStk: 0,
-	VmExe: 0,
-	VmLib: 0,
-	VmPTE: 0,
-	VmSwap: 0,
-};
-
-async function getMemoryUsage(pid) {
-	const status = await fs.readFile(`/proc/${pid}/status`, 'utf-8');
-
-	const result = {};
-	for (const key of Object.keys(keys)) {
-		const match = status.match(new RegExp(`${key}:\\s+(\\d+)\\s+kB`));
-		if (match) {
-			result[key] = parseInt(match[1], 10);
-		} else {
-			throw new Error(`Failed to parse ${key} from /proc/${pid}/status`);
-		}
-	}
-
-	return result;
-}
-
 async function measureMemory() {
+	const startTime = Date.now();
+
 	// Start the Misskey backend server using fork to enable IPC
 	const serverProcess = fork(join(__dirname, '../built/boot/entry.js'), ['expose-gc'], {
 		cwd: join(__dirname, '..'),
@@ -106,7 +77,44 @@ async function measureMemory() {
 
 	// Get memory usage from the server process via /proc
 	const pid = serverProcess.pid;
-	const memoryInfo = await getMemoryUsage(pid);
+	let memoryInfo;
+
+	try {
+		const fs = await import('node:fs/promises');
+
+		// Read /proc/[pid]/status for detailed memory info
+		const status = await fs.readFile(`/proc/${pid}/status`, 'utf-8');
+		const vmRssMatch = status.match(/VmRSS:\s+(\d+)\s+kB/);
+		const vmDataMatch = status.match(/VmData:\s+(\d+)\s+kB/);
+		const vmSizeMatch = status.match(/VmSize:\s+(\d+)\s+kB/);
+
+		memoryInfo = {
+			rss: vmRssMatch ? parseInt(vmRssMatch[1], 10) * 1024 : null,
+			heapUsed: vmDataMatch ? parseInt(vmDataMatch[1], 10) * 1024 : null,
+			vmSize: vmSizeMatch ? parseInt(vmSizeMatch[1], 10) * 1024 : null,
+		};
+	} catch (err) {
+		// Fallback: use ps command
+		process.stderr.write(`Warning: Could not read /proc/${pid}/status: ${err}\n`);
+
+		const { execSync } = await import('node:child_process');
+		try {
+			const ps = execSync(`ps -o rss= -p ${pid}`, { encoding: 'utf-8' });
+			const rssKb = parseInt(ps.trim(), 10);
+			memoryInfo = {
+				rss: rssKb * 1024,
+				heapUsed: null,
+				vmSize: null,
+			};
+		} catch {
+			memoryInfo = {
+				rss: null,
+				heapUsed: null,
+				vmSize: null,
+				error: 'Could not measure memory',
+			};
+		}
+	}
 
 	// Stop the server
 	serverProcess.kill('SIGTERM');
@@ -129,41 +137,15 @@ async function measureMemory() {
 
 	const result = {
 		timestamp: new Date().toISOString(),
+		startupTimeMs: startupTime,
 		memory: memoryInfo,
 	};
 
-	return result;
-}
-
-async function main() {
-	// 直列の方が時間的に分散されて正確そうだから直列でやる
-	const results = [];
-	for (let i = 0; i < SAMPLE_COUNT; i++) {
-		const res = await measureMemory();
-		results.push(res);
-	}
-
-	// Calculate averages
-	const avgMemory = structuredClone(keys);
-	for (const res of results) {
-		for (const key of Object.keys(avgMemory)) {
-			avgMemory[key] += res.memory[key];
-		}
-	}
-	for (const key of Object.keys(avgMemory)) {
-		avgMemory[key] = Math.round(avgMemory[key] / SAMPLE_COUNT);
-	}
-
-	const result = {
-		timestamp: new Date().toISOString(),
-		memory: avgMemory,
-	};
-
 	// Output as JSON to stdout
 	console.log(JSON.stringify(result, null, 2));
 }
 
-main().catch((err) => {
+measureMemory().catch((err) => {
 	console.error(JSON.stringify({
 		error: err.message,
 		timestamp: new Date().toISOString(),

packages/backend/src/core/QueueService.ts

@@ -6,6 +6,7 @@
 import { randomUUID } from 'node:crypto';
 import { Inject, Injectable } from '@nestjs/common';
 import { MetricsTime, type JobType } from 'bullmq';
+import { parse as parseRedisInfo } from 'redis-info';
 import type { IActivity } from '@/core/activitypub/type.js';
 import type { MiDriveFile } from '@/models/DriveFile.js';
 import type { MiWebhook, WebhookEventTypes } from '@/models/Webhook.js';
@@ -85,19 +86,6 @@ const REPEATABLE_SYSTEM_JOB_DEF = [{
 	pattern: '0 4 * * *',
 }];
 
-function parseRedisInfo(infoText: string): Record<string, string> {
-	const fields = infoText
-		.split('\n')
-		.filter(line => line.length > 0 && !line.startsWith('#'))
-		.map(line => line.trim().split(':'));
-
-	const result: Record<string, string> = {};
-	for (const [key, value] of fields) {
-		result[key] = value;
-	}
-	return result;
-}
-
 @Injectable()
 export class QueueService {
 	constructor(
@@ -902,7 +890,7 @@ export class QueueService {
 			},
 			db: {
 				version: db.redis_version,
-				mode: db.redis_mode as 'cluster' | 'standalone' | 'sentinel',
+				mode: db.redis_mode,
 				runId: db.run_id,
 				processId: db.process_id,
 				port: parseInt(db.tcp_port),

pnpm-lock.yaml

@@ -165,6 +165,9 @@ importers:
       '@twemoji/parser':
         specifier: 16.0.0
         version: 16.0.0
+      '@types/redis-info':
+        specifier: 3.0.3
+        version: 3.0.3
       accepts:
         specifier: 1.3.8
         version: 1.3.8
@@ -336,6 +339,9 @@ importers:
       re2:
         specifier: 1.23.0
         version: 1.23.0
+      redis-info:
+        specifier: 3.1.0
+        version: 3.1.0
       reflect-metadata:
         specifier: 0.2.2
         version: 0.2.2
@@ -4678,6 +4684,9 @@ packages:
   '@types/readdir-glob@1.1.5':
     resolution: {integrity: sha512-raiuEPUYqXu+nvtY2Pe8s8FEmZ3x5yAH4VkLdihcPdalvsHltomrRC9BzuStrJ9yk06470hS0Crw0f1pXqD+Hg==}
 
+  '@types/redis-info@3.0.3':
+    resolution: {integrity: sha512-VIkNy6JbYI/RLdbPHdm9JQvv6RVld2uE2/6Hdid38Qdq+zvDli2FTpImI8pC5zwp8xS8qVqfzlfyAub8xZEd5g==}
+
   '@types/rename@1.0.7':
     resolution: {integrity: sha512-E9qapfghUGfBMi3jNhsmCKPIp3f2zvNKpaX1BDGLGJNjzpgsZ/RTx7NaNksFjGoJ+r9NvWF1NSM5vVecnNjVmw==}
 
@@ -9513,6 +9522,9 @@ packages:
     resolution: {integrity: sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==}
     engines: {node: '>=4'}
 
+  redis-info@3.1.0:
+    resolution: {integrity: sha512-ER4L9Sh/vm63DkIE0bkSjxluQlioBiBgf5w1UuldaW/3vPcecdljVDisZhmnCMvsxHNiARTTDDHGg9cGwTfrKg==}
+
   redis-parser@3.0.0:
     resolution: {integrity: sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==}
     engines: {node: '>=4'}
@@ -10935,9 +10947,6 @@ packages:
   vue-component-type-helpers@3.2.1:
     resolution: {integrity: sha512-gKV7XOkQl4urSuLHNY1tnVQf7wVgtb/mKbRyxSLWGZUY9RK7aDPhBenTjm+i8ZFe0zC2PZeHMPtOZXZfyaFOzQ==}
 
-  vue-component-type-helpers@3.2.2:
-    resolution: {integrity: sha512-x8C2nx5XlUNM0WirgfTkHjJGO/ABBxlANZDtHw2HclHtQnn+RFPTnbjMJn8jHZW4TlUam0asHcA14lf1C6Jb+A==}
-
   vue-demi@0.14.10:
     resolution: {integrity: sha512-nMZBOwuzabUO0nLgIcc6rycZEebF6eeUfaiQx9+WSk8e29IbLvPU9feI6tqW4kTo3hvoYAJkMh8n8D0fuISphg==}
     engines: {node: '>=12'}
@@ -14932,7 +14941,7 @@ snapshots:
       storybook: 10.1.10(@testing-library/dom@10.4.0)(bufferutil@4.1.0)(prettier@3.7.4)(react-dom@19.2.3(react@19.2.3))(react@19.2.3)(utf-8-validate@6.0.6)
       type-fest: 2.19.0
       vue: 3.5.26(typescript@5.9.3)
-      vue-component-type-helpers: 3.2.2
+      vue-component-type-helpers: 3.2.1
 
   '@stylistic/eslint-plugin@5.5.0(eslint@9.39.2)':
     dependencies:
@@ -15469,6 +15478,8 @@ snapshots:
     dependencies:
       '@types/node': 24.10.4
 
+  '@types/redis-info@3.0.3': {}
+
   '@types/rename@1.0.7': {}
 
   '@types/resolve@1.20.6': {}
@@ -21341,6 +21352,10 @@ snapshots:
 
   redis-errors@1.2.0: {}
 
+  redis-info@3.1.0:
+    dependencies:
+      lodash: 4.17.21
+
   redis-parser@3.0.0:
     dependencies:
       redis-errors: 1.2.0
@@ -22840,8 +22855,6 @@ snapshots:
 
   vue-component-type-helpers@3.2.1: {}
 
-  vue-component-type-helpers@3.2.2: {}
-
   vue-demi@0.14.10(vue@3.5.26(typescript@5.9.3)):
     dependencies:
       vue: 3.5.26(typescript@5.9.3)