Merge f744b5711f into 0d46089f9a

* wip

* Update report-backend-memory.yml

.github/workflows/report-backend-memory.yml

@@ -54,55 +54,50 @@ jobs:
           BASE_MEMORY=$(cat ./artifacts/memory-base.json)
           HEAD_MEMORY=$(cat ./artifacts/memory-head.json)
 
-          BASE_RSS=$(echo "$BASE_MEMORY" | jq -r '.memory.rss // 0')
-          HEAD_RSS=$(echo "$HEAD_MEMORY" | jq -r '.memory.rss // 0')
+          calc() {
+            BASE=$(echo "$BASE_MEMORY" | jq -r '.memory.'"$1"' // 0')
+            HEAD=$(echo "$HEAD_MEMORY" | jq -r '.memory.'"$1"' // 0')
 
-          # Calculate difference
-          if [ "$BASE_RSS" -gt 0 ] && [ "$HEAD_RSS" -gt 0 ]; then
-            DIFF=$((HEAD_RSS - BASE_RSS))
-            DIFF_PERCENT=$(echo "scale=2; ($DIFF * 100) / $BASE_RSS" | bc)
-
-            # Convert to MB for readability
-            BASE_MB=$(echo "scale=2; $BASE_RSS / 1048576" | bc)
-            HEAD_MB=$(echo "scale=2; $HEAD_RSS / 1048576" | bc)
-            DIFF_MB=$(echo "scale=2; $DIFF / 1048576" | bc)
-
-            echo "base_mb=$BASE_MB" >> "$GITHUB_OUTPUT"
-            echo "head_mb=$HEAD_MB" >> "$GITHUB_OUTPUT"
-            echo "diff_mb=$DIFF_MB" >> "$GITHUB_OUTPUT"
-            echo "diff_percent=$DIFF_PERCENT" >> "$GITHUB_OUTPUT"
-            echo "has_data=true" >> "$GITHUB_OUTPUT"
-
-            # Determine if this is a significant change (more than 5% increase)
-            if [ "$(echo "$DIFF_PERCENT > 5" | bc)" -eq 1 ]; then
-              echo "significant_increase=true" >> "$GITHUB_OUTPUT"
+            DIFF=$((HEAD - BASE))
+            if [ "$BASE" -gt 0 ]; then
+              DIFF_PERCENT=$(echo "scale=2; ($DIFF * 100) / $BASE" | bc)
             else
-              echo "significant_increase=false" >> "$GITHUB_OUTPUT"
+              DIFF_PERCENT=0
             fi
-          else
-            echo "has_data=false" >> "$GITHUB_OUTPUT"
-          fi
+
+            # Convert KB to MB for readability
+            BASE_MB=$(echo "scale=2; $BASE / 1024" | bc)
+            HEAD_MB=$(echo "scale=2; $HEAD / 1024" | bc)
+            DIFF_MB=$(echo "scale=2; $DIFF / 1024" | bc)
+
+            echo "$1-base=$BASE_MB" >> "$GITHUB_OUTPUT"
+            echo "$1-head=$HEAD_MB" >> "$GITHUB_OUTPUT"
+            echo "$1-diff=$DIFF_MB" >> "$GITHUB_OUTPUT"
+            echo "$1-diff_percent=$DIFF_PERCENT" >> "$GITHUB_OUTPUT"
+          }
+
+          calc VmRSS
+          calc VmHWM
+          calc VmSize
       - id: build-comment
         name: Build memory comment
         run: |
-          HEADER="## Backend Memory Usage Comparison"
+          HEADER="## Backend memory usage comparison"
           FOOTER="[See workflow logs for details](https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID})"
 
           echo "$HEADER" > ./output.md
           echo >> ./output.md
 
-          if [ "${{ steps.compare.outputs.has_data }}" == "true" ]; then
-            echo "| Metric | base | head | Diff |" >> ./output.md
-            echo "|--------|------|------|------|" >> ./output.md
-            echo "| RSS | ${{ steps.compare.outputs.base_mb }} MB | ${{ steps.compare.outputs.head_mb }} MB | ${{ steps.compare.outputs.diff_mb }} MB (${{ steps.compare.outputs.diff_percent }}%) |" >> ./output.md
-            echo >> ./output.md
+          echo "| Metric | base | head | Diff |" >> ./output.md
+          echo "|--------|------|------|------|" >> ./output.md
+          echo "| RSS | ${{ steps.compare.outputs.VmRSS-base }} MB | ${{ steps.compare.outputs.VmRSS-head }} MB | ${{ steps.compare.outputs.VmRSS-diff }} MB (${{ steps.compare.outputs.VmRSS-diff_percent }}%) |" >> ./output.md
+          echo "| HWM | ${{ steps.compare.outputs.VmHWM-base }} MB | ${{ steps.compare.outputs.VmHWM-head }} MB | ${{ steps.compare.outputs.VmHWM-diff }} MB (${{ steps.compare.outputs.VmHWM-diff_percent }}%) |" >> ./output.md
+          echo "| VMS | ${{ steps.compare.outputs.VmSize-base }} MB | ${{ steps.compare.outputs.VmSize-head }} MB | ${{ steps.compare.outputs.VmSize-diff }} MB (${{ steps.compare.outputs.VmSize-diff_percent }}%) |" >> ./output.md
+          echo >> ./output.md
 
-            if [ "${{ steps.compare.outputs.significant_increase }}" == "true" ]; then
-              echo "⚠️ **Warning**: Memory usage has increased by more than 5%. Please verify this is not an unintended change." >> ./output.md
-              echo >> ./output.md
-            fi
-          else
-            echo "Could not retrieve memory usage data." >> ./output.md
+          # Determine if this is a significant change (more than 5% increase)
+          if [ "$(echo "${{ steps.compare.outputs.VmRSS-diff_percent }} > 5" | bc)" -eq 1 ]; then
+            echo "⚠️ **Warning**: Memory usage has increased by more than 5%. Please verify this is not an unintended change." >> ./output.md
             echo >> ./output.md
           fi
 

packages/backend/package.json

@@ -96,7 +96,6 @@
 		"@swc/cli": "0.7.9",
 		"@swc/core": "1.15.7",
 		"@twemoji/parser": "16.0.0",
-		"@types/redis-info": "3.0.3",
 		"accepts": "1.3.8",
 		"ajv": "8.17.1",
 		"archiver": "7.0.1",
@@ -154,7 +153,6 @@
 		"random-seed": "0.3.0",
 		"ratelimiter": "3.4.1",
 		"re2": "1.23.0",
-		"redis-info": "3.1.0",
 		"reflect-metadata": "0.2.2",
 		"rename": "1.0.4",
 		"rss-parser": "3.13.0",

packages/backend/scripts/measure-memory.mjs

@@ -14,16 +14,45 @@ import { fork } from 'node:child_process';
 import { setTimeout } from 'node:timers/promises';
 import { fileURLToPath } from 'node:url';
 import { dirname, join } from 'node:path';
+import * as fs from 'node:fs/promises';
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = dirname(__filename);
 
+const SAMPLE_COUNT = 3; // Number of samples to measure
 const STARTUP_TIMEOUT = 120000; // 120 seconds timeout for server startup
 const MEMORY_SETTLE_TIME = 10000; // Wait 10 seconds after startup for memory to settle
 
-async function measureMemory() {
-	const startTime = Date.now();
+const keys = {
+	VmPeak: 0,
+	VmSize: 0,
+	VmHWM: 0,
+	VmRSS: 0,
+	VmData: 0,
+	VmStk: 0,
+	VmExe: 0,
+	VmLib: 0,
+	VmPTE: 0,
+	VmSwap: 0,
+};
 
+async function getMemoryUsage(pid) {
+	const status = await fs.readFile(`/proc/${pid}/status`, 'utf-8');
+
+	const result = {};
+	for (const key of Object.keys(keys)) {
+		const match = status.match(new RegExp(`${key}:\\s+(\\d+)\\s+kB`));
+		if (match) {
+			result[key] = parseInt(match[1], 10);
+		} else {
+			throw new Error(`Failed to parse ${key} from /proc/${pid}/status`);
+		}
+	}
+
+	return result;
+}
+
+async function measureMemory() {
 	// Start the Misskey backend server using fork to enable IPC
 	const serverProcess = fork(join(__dirname, '../built/boot/entry.js'), ['expose-gc'], {
 		cwd: join(__dirname, '..'),
@@ -77,44 +106,7 @@ async function measureMemory() {
 
 	// Get memory usage from the server process via /proc
 	const pid = serverProcess.pid;
-	let memoryInfo;
-
-	try {
-		const fs = await import('node:fs/promises');
-
-		// Read /proc/[pid]/status for detailed memory info
-		const status = await fs.readFile(`/proc/${pid}/status`, 'utf-8');
-		const vmRssMatch = status.match(/VmRSS:\s+(\d+)\s+kB/);
-		const vmDataMatch = status.match(/VmData:\s+(\d+)\s+kB/);
-		const vmSizeMatch = status.match(/VmSize:\s+(\d+)\s+kB/);
-
-		memoryInfo = {
-			rss: vmRssMatch ? parseInt(vmRssMatch[1], 10) * 1024 : null,
-			heapUsed: vmDataMatch ? parseInt(vmDataMatch[1], 10) * 1024 : null,
-			vmSize: vmSizeMatch ? parseInt(vmSizeMatch[1], 10) * 1024 : null,
-		};
-	} catch (err) {
-		// Fallback: use ps command
-		process.stderr.write(`Warning: Could not read /proc/${pid}/status: ${err}\n`);
-
-		const { execSync } = await import('node:child_process');
-		try {
-			const ps = execSync(`ps -o rss= -p ${pid}`, { encoding: 'utf-8' });
-			const rssKb = parseInt(ps.trim(), 10);
-			memoryInfo = {
-				rss: rssKb * 1024,
-				heapUsed: null,
-				vmSize: null,
-			};
-		} catch {
-			memoryInfo = {
-				rss: null,
-				heapUsed: null,
-				vmSize: null,
-				error: 'Could not measure memory',
-			};
-		}
-	}
+	const memoryInfo = await getMemoryUsage(pid);
 
 	// Stop the server
 	serverProcess.kill('SIGTERM');
@@ -137,15 +129,41 @@ async function measureMemory() {
 
 	const result = {
 		timestamp: new Date().toISOString(),
-		startupTimeMs: startupTime,
 		memory: memoryInfo,
 	};
 
+	return result;
+}
+
+async function main() {
+	// 直列の方が時間的に分散されて正確そうだから直列でやる
+	const results = [];
+	for (let i = 0; i < SAMPLE_COUNT; i++) {
+		const res = await measureMemory();
+		results.push(res);
+	}
+
+	// Calculate averages
+	const avgMemory = structuredClone(keys);
+	for (const res of results) {
+		for (const key of Object.keys(avgMemory)) {
+			avgMemory[key] += res.memory[key];
+		}
+	}
+	for (const key of Object.keys(avgMemory)) {
+		avgMemory[key] = Math.round(avgMemory[key] / SAMPLE_COUNT);
+	}
+
+	const result = {
+		timestamp: new Date().toISOString(),
+		memory: avgMemory,
+	};
+
 	// Output as JSON to stdout
 	console.log(JSON.stringify(result, null, 2));
 }
 
-measureMemory().catch((err) => {
+main().catch((err) => {
 	console.error(JSON.stringify({
 		error: err.message,
 		timestamp: new Date().toISOString(),

packages/backend/src/core/QueueService.ts

@@ -6,7 +6,6 @@
 import { randomUUID } from 'node:crypto';
 import { Inject, Injectable } from '@nestjs/common';
 import { MetricsTime, type JobType } from 'bullmq';
-import { parse as parseRedisInfo } from 'redis-info';
 import type { IActivity } from '@/core/activitypub/type.js';
 import type { MiDriveFile } from '@/models/DriveFile.js';
 import type { MiWebhook, WebhookEventTypes } from '@/models/Webhook.js';
@@ -86,6 +85,19 @@ const REPEATABLE_SYSTEM_JOB_DEF = [{
 	pattern: '0 4 * * *',
 }];
 
+function parseRedisInfo(infoText: string): Record<string, string> {
+	const fields = infoText
+		.split('\n')
+		.filter(line => line.length > 0 && !line.startsWith('#'))
+		.map(line => line.trim().split(':'));
+
+	const result: Record<string, string> = {};
+	for (const [key, value] of fields) {
+		result[key] = value;
+	}
+	return result;
+}
+
 @Injectable()
 export class QueueService {
 	constructor(
@@ -890,7 +902,7 @@ export class QueueService {
 			},
 			db: {
 				version: db.redis_version,
-				mode: db.redis_mode,
+				mode: db.redis_mode as 'cluster' | 'standalone' | 'sentinel',
 				runId: db.run_id,
 				processId: db.process_id,
 				port: parseInt(db.tcp_port),

pnpm-lock.yaml

@@ -165,9 +165,6 @@ importers:
       '@twemoji/parser':
         specifier: 16.0.0
         version: 16.0.0
-      '@types/redis-info':
-        specifier: 3.0.3
-        version: 3.0.3
       accepts:
         specifier: 1.3.8
         version: 1.3.8
@@ -339,9 +336,6 @@ importers:
       re2:
         specifier: 1.23.0
         version: 1.23.0
-      redis-info:
-        specifier: 3.1.0
-        version: 3.1.0
       reflect-metadata:
         specifier: 0.2.2
         version: 0.2.2
@@ -4684,9 +4678,6 @@ packages:
   '@types/readdir-glob@1.1.5':
     resolution: {integrity: sha512-raiuEPUYqXu+nvtY2Pe8s8FEmZ3x5yAH4VkLdihcPdalvsHltomrRC9BzuStrJ9yk06470hS0Crw0f1pXqD+Hg==}
 
-  '@types/redis-info@3.0.3':
-    resolution: {integrity: sha512-VIkNy6JbYI/RLdbPHdm9JQvv6RVld2uE2/6Hdid38Qdq+zvDli2FTpImI8pC5zwp8xS8qVqfzlfyAub8xZEd5g==}
-
   '@types/rename@1.0.7':
     resolution: {integrity: sha512-E9qapfghUGfBMi3jNhsmCKPIp3f2zvNKpaX1BDGLGJNjzpgsZ/RTx7NaNksFjGoJ+r9NvWF1NSM5vVecnNjVmw==}
 
@@ -9522,9 +9513,6 @@ packages:
     resolution: {integrity: sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==}
     engines: {node: '>=4'}
 
-  redis-info@3.1.0:
-    resolution: {integrity: sha512-ER4L9Sh/vm63DkIE0bkSjxluQlioBiBgf5w1UuldaW/3vPcecdljVDisZhmnCMvsxHNiARTTDDHGg9cGwTfrKg==}
-
   redis-parser@3.0.0:
     resolution: {integrity: sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==}
     engines: {node: '>=4'}
@@ -10947,6 +10935,9 @@ packages:
   vue-component-type-helpers@3.2.1:
     resolution: {integrity: sha512-gKV7XOkQl4urSuLHNY1tnVQf7wVgtb/mKbRyxSLWGZUY9RK7aDPhBenTjm+i8ZFe0zC2PZeHMPtOZXZfyaFOzQ==}
 
+  vue-component-type-helpers@3.2.2:
+    resolution: {integrity: sha512-x8C2nx5XlUNM0WirgfTkHjJGO/ABBxlANZDtHw2HclHtQnn+RFPTnbjMJn8jHZW4TlUam0asHcA14lf1C6Jb+A==}
+
   vue-demi@0.14.10:
     resolution: {integrity: sha512-nMZBOwuzabUO0nLgIcc6rycZEebF6eeUfaiQx9+WSk8e29IbLvPU9feI6tqW4kTo3hvoYAJkMh8n8D0fuISphg==}
     engines: {node: '>=12'}
@@ -14941,7 +14932,7 @@ snapshots:
       storybook: 10.1.10(@testing-library/dom@10.4.0)(bufferutil@4.1.0)(prettier@3.7.4)(react-dom@19.2.3(react@19.2.3))(react@19.2.3)(utf-8-validate@6.0.6)
       type-fest: 2.19.0
       vue: 3.5.26(typescript@5.9.3)
-      vue-component-type-helpers: 3.2.1
+      vue-component-type-helpers: 3.2.2
 
   '@stylistic/eslint-plugin@5.5.0(eslint@9.39.2)':
     dependencies:
@@ -15478,8 +15469,6 @@ snapshots:
     dependencies:
       '@types/node': 24.10.4
 
-  '@types/redis-info@3.0.3': {}
-
   '@types/rename@1.0.7': {}
 
   '@types/resolve@1.20.6': {}
@@ -21352,10 +21341,6 @@ snapshots:
 
   redis-errors@1.2.0: {}
 
-  redis-info@3.1.0:
-    dependencies:
-      lodash: 4.17.21
-
   redis-parser@3.0.0:
     dependencies:
       redis-errors: 1.2.0
@@ -22855,6 +22840,8 @@ snapshots:
 
   vue-component-type-helpers@3.2.1: {}
 
+  vue-component-type-helpers@3.2.2: {}
+
   vue-demi@0.14.10(vue@3.5.26(typescript@5.9.3)):
     dependencies:
       vue: 3.5.26(typescript@5.9.3)