refactor(backend): 変換後.config.jsonに統一するように+修正など (#16929)

* wip

* Update config.ts

* wip

* convertは元ファイルを変更するようなニュアンスを若干感じるのでcompileに改名

* wip

* Update package.json

* Revert "Update package.json"

This reverts commit e5c2802316.

* wip

* wip

* 謎

* clean up

* wip

* wip

* Revert "wip"

This reverts commit 3aa25ac7cf.

* wip

* wip

* Update dummy.yml

* wip

* Update compile_config.js

* Update compile_config.js

* wip

* Revert "wip"

This reverts commit fd78e097c6.

* Update dummy.yml

* Update compile_config.js

.github/workflows/get-backend-memory.yml

@@ -57,7 +57,7 @@ jobs:
     - name: Copy Configure
       run: cp .github/misskey/test.yml .config/default.yml
     - name: Compile Configure
-      run: pnpm convert:config
+      run: pnpm compile-config
     - name: Build
       run: pnpm build
     - name: Run migrations

package.json

@@ -22,15 +22,15 @@
 	],
 	"private": true,
 	"scripts": {
-		"convert:config": "cd packages/backend && pnpm convert:config",
+		"compile-config": "cd packages/backend && pnpm compile-config",
 		"build-pre": "node ./scripts/build-pre.js",
 		"build-assets": "node ./scripts/build-assets.mjs",
 		"build": "pnpm build-pre && pnpm -r build && pnpm build-assets",
 		"build-storybook": "pnpm --filter frontend build-storybook",
 		"build-misskey-js-with-types": "pnpm build-pre && pnpm --filter backend... --filter=!misskey-js build && pnpm --filter backend generate-api-json --no-build && ncp packages/backend/built/api.json packages/misskey-js/generator/api.json && pnpm --filter misskey-js update-autogen-code && pnpm --filter misskey-js build && pnpm --filter misskey-js api",
-		"start": "pnpm check:connect && cd packages/backend && pnpm convert:config && node ./built/boot/entry.js",
-		"start:inspect": "cd packages/backend && pnpm convert:config && node --inspect ./built/boot/entry.js",
-		"start:test": "ncp ./.github/misskey/test.yml ./.config/test.yml && cd packages/backend && pnpm convert:config && cross-env NODE_ENV=test node ./built/boot/entry.js",
+		"start": "pnpm check:connect && cd packages/backend && pnpm compile-config && node ./built/boot/entry.js",
+		"start:inspect": "cd packages/backend && pnpm compile-config && node --inspect ./built/boot/entry.js",
+		"start:test": "ncp ./.github/misskey/test.yml ./.config/test.yml && cd packages/backend && cross-env NODE_ENV=test pnpm compile-config && cross-env NODE_ENV=test node ./built/boot/entry.js",
 		"cli": "cd packages/backend && pnpm cli",
 		"init": "pnpm migrate",
 		"migrate": "cd packages/backend && pnpm migrate",

packages/backend/migration/1745378064470-composite-note-index.js

@@ -3,14 +3,14 @@
  * SPDX-License-Identifier: AGPL-3.0-only
  */
 
-import { isConcurrentIndexMigrationEnabled } from "./js/migration-config.js";
+const isConcurrentIndexMigrationEnabled = process.env.MISSKEY_MIGRATION_CREATE_INDEX_CONCURRENTLY === '1';
 
 export class CompositeNoteIndex1745378064470 {
 	name = 'CompositeNoteIndex1745378064470';
-	transaction = isConcurrentIndexMigrationEnabled() ? false : undefined;
+	transaction = isConcurrentIndexMigrationEnabled ? false : undefined;
 
 	async up(queryRunner) {
-		const concurrently = isConcurrentIndexMigrationEnabled();
+		const concurrently = isConcurrentIndexMigrationEnabled;
 
 		if (concurrently) {
 			const hasValidIndex = await queryRunner.query(`SELECT indisvalid FROM pg_index INNER JOIN pg_class ON pg_index.indexrelid = pg_class.oid WHERE pg_class.relname = 'IDX_724b311e6f883751f261ebe378'`);
@@ -29,7 +29,7 @@ export class CompositeNoteIndex1745378064470 {
 	}
 
 	async down(queryRunner) {
-		const mayConcurrently = isConcurrentIndexMigrationEnabled() ? 'CONCURRENTLY' : '';
+		const mayConcurrently = isConcurrentIndexMigrationEnabled ? 'CONCURRENTLY' : '';
 		await queryRunner.query(`DROP INDEX IF EXISTS "IDX_724b311e6f883751f261ebe378"`);
 		await queryRunner.query(`CREATE INDEX ${mayConcurrently} "IDX_5b87d9d19127bd5d92026017a7" ON "note" ("userId")`);
 	}

packages/backend/migration/1746949539915-migrateSomeConfigFileSettingsToMeta.js

@@ -3,17 +3,15 @@
  * SPDX-License-Identifier: AGPL-3.0-only
  */
 
-import {loadConfig} from "./js/migration-config.js";
 
 export class MigrateSomeConfigFileSettingsToMeta1746949539915 {
     name = 'MigrateSomeConfigFileSettingsToMeta1746949539915'
 
     async up(queryRunner) {
-        const config = loadConfig();
         // $1 cannot be used in ALTER TABLE queries
-        await queryRunner.query(`ALTER TABLE "meta" ADD "proxyRemoteFiles" boolean NOT NULL DEFAULT ${config.proxyRemoteFiles}`);
-        await queryRunner.query(`ALTER TABLE "meta" ADD "signToActivityPubGet" boolean NOT NULL DEFAULT ${config.signToActivityPubGet}`);
-        await queryRunner.query(`ALTER TABLE "meta" ADD "allowExternalApRedirect" boolean NOT NULL DEFAULT ${!config.disallowExternalApRedirect}`);
+        await queryRunner.query(`ALTER TABLE "meta" ADD "proxyRemoteFiles" boolean NOT NULL DEFAULT TRUE`);
+        await queryRunner.query(`ALTER TABLE "meta" ADD "signToActivityPubGet" boolean NOT NULL DEFAULT TRUE`);
+        await queryRunner.query(`ALTER TABLE "meta" ADD "allowExternalApRedirect" boolean NOT NULL DEFAULT TRUE`);
     }
 
     async down(queryRunner) {

packages/backend/migration/js/migration-config.js

@@ -1,31 +0,0 @@
-/*
- * SPDX-FileCopyrightText: syuilo and misskey-project
- * SPDX-License-Identifier: AGPL-3.0-only
- */
-
-import { path as configYamlPath } from '../../built/config.js';
-import * as yaml from 'js-yaml';
-import fs from "node:fs";
-
-export function isConcurrentIndexMigrationEnabled() {
-	return process.env.MISSKEY_MIGRATION_CREATE_INDEX_CONCURRENTLY === '1';
-}
-
-let loadedConfigCache = undefined;
-
-function loadConfigInternal() {
-	const config = yaml.load(fs.readFileSync(configYamlPath, 'utf-8'));
-
-	return {
-		disallowExternalApRedirect: Boolean(config.disallowExternalApRedirect ?? false),
-		proxyRemoteFiles: Boolean(config.proxyRemoteFiles ?? false),
-		signToActivityPubGet: Boolean(config.signToActivityPubGet ?? true),
-	}
-}
-
-export function loadConfig() {
-	if (loadedConfigCache === undefined) {
-		loadedConfigCache = loadConfigInternal();
-	}
-	return loadedConfigCache;
-}

packages/backend/ormconfig.js

@@ -1,7 +1,8 @@
 import { DataSource } from 'typeorm';
 import { loadConfig } from './built/config.js';
 import { entities } from './built/postgres.js';
-import { isConcurrentIndexMigrationEnabled } from "./migration/js/migration-config.js";
+
+const isConcurrentIndexMigrationEnabled = process.env.MISSKEY_MIGRATION_CREATE_INDEX_CONCURRENTLY === '1';
 
 const config = loadConfig();
 
@@ -15,5 +16,5 @@ export default new DataSource({
 	extra: config.db.extra,
 	entities: entities,
 	migrations: ['migration/*.js'],
-	migrationsTransactionMode: isConcurrentIndexMigrationEnabled() ? 'each' : 'all',
+	migrationsTransactionMode: isConcurrentIndexMigrationEnabled ? 'each' : 'all',
 });

packages/backend/package.json

@@ -7,37 +7,37 @@
 		"node": "^22.15.0 || ^24.10.0"
 	},
 	"scripts": {
-		"start": "pnpm convert:config && node ./built/boot/entry.js",
-		"start:inspect": "pnpm convert:config && node --inspect ./built/boot/entry.js",
-		"start:test": "pnpm convert:config && cross-env NODE_ENV=test node ./built/boot/entry.js",
-		"migrate": "pnpm convert:config && pnpm typeorm migration:run -d ormconfig.js",
-		"revert": "pnpm convert:config && pnpm typeorm migration:revert -d ormconfig.js",
-		"cli": "pnpm convert:config && node ./built/boot/cli.js",
-		"check:connect": "pnpm convert:config && node ./scripts/check_connect.js",
-		"convert:config": "node ./scripts/convert_config.js",
+		"start": "pnpm compile-config && node ./built/boot/entry.js",
+		"start:inspect": "pnpm compile-config && node --inspect ./built/boot/entry.js",
+		"start:test": "cross-env NODE_ENV=test pnpm compile-config && cross-env NODE_ENV=test node ./built/boot/entry.js",
+		"migrate": "pnpm compile-config && pnpm typeorm migration:run -d ormconfig.js",
+		"revert": "pnpm compile-config && pnpm typeorm migration:revert -d ormconfig.js",
+		"cli": "pnpm compile-config && node ./built/boot/cli.js",
+		"check:connect": "pnpm compile-config && node ./scripts/check_connect.js",
+		"compile-config": "node ./scripts/compile_config.js",
 		"build": "swc src -d built -D --strip-leading-paths",
 		"build:test": "swc test-server -d built-test -D --config-file test-server/.swcrc --strip-leading-paths",
 		"watch:swc": "swc src -d built -D -w --strip-leading-paths",
 		"build:tsc": "tsc -p tsconfig.json && tsc-alias -p tsconfig.json",
-		"watch": "pnpm convert:config && node ./scripts/watch.mjs",
+		"watch": "pnpm compile-config && node ./scripts/watch.mjs",
 		"restart": "pnpm build && pnpm start",
-		"dev": "pnpm convert:config && node ./scripts/dev.mjs",
+		"dev": "pnpm compile-config && node ./scripts/dev.mjs",
 		"typecheck": "tsc --noEmit && tsc -p test --noEmit && tsc -p test-federation --noEmit",
 		"eslint": "eslint --quiet \"{src,test-federation}/**/*.ts\"",
 		"lint": "pnpm typecheck && pnpm eslint",
-		"jest": "pnpm convert:config && cross-env NODE_ENV=test node ./jest.js --forceExit --config jest.config.unit.cjs",
-		"jest:e2e": "pnpm convert:config && cross-env NODE_ENV=test node ./jest.js --forceExit --config jest.config.e2e.cjs",
-		"jest:fed": "pnpm convert:config && node ./jest.js --forceExit --config jest.config.fed.cjs",
-		"jest-and-coverage": "pnpm convert:config && cross-env NODE_ENV=test node ./jest.js --coverage --forceExit --config jest.config.unit.cjs",
-		"jest-and-coverage:e2e": "pnpm convert:config && cross-env NODE_ENV=test node ./jest.js --coverage --forceExit --config jest.config.e2e.cjs",
-		"jest-clear": "pnpm convert:config && cross-env NODE_ENV=test node ./jest.js --clearCache",
+		"jest": "cross-env NODE_ENV=test pnpm compile-config && cross-env NODE_ENV=test node ./jest.js --forceExit --config jest.config.unit.cjs",
+		"jest:e2e": "cross-env NODE_ENV=test pnpm compile-config && cross-env NODE_ENV=test node ./jest.js --forceExit --config jest.config.e2e.cjs",
+		"jest:fed": "pnpm compile-config && node ./jest.js --forceExit --config jest.config.fed.cjs",
+		"jest-and-coverage": "cross-env NODE_ENV=test pnpm compile-config && cross-env NODE_ENV=test node ./jest.js --coverage --forceExit --config jest.config.unit.cjs",
+		"jest-and-coverage:e2e": "cross-env NODE_ENV=test pnpm compile-config && cross-env NODE_ENV=test node ./jest.js --coverage --forceExit --config jest.config.e2e.cjs",
+		"jest-clear": "cross-env NODE_ENV=test pnpm compile-config && cross-env NODE_ENV=test node ./jest.js --clearCache",
 		"test": "pnpm jest",
 		"test:e2e": "pnpm build && pnpm build:test && pnpm jest:e2e",
 		"test:fed": "pnpm jest:fed",
 		"test-and-coverage": "pnpm jest-and-coverage",
 		"test-and-coverage:e2e": "pnpm build && pnpm build:test && pnpm jest-and-coverage:e2e",
 		"check-migrations": "node scripts/check_migrations_clean.js",
-		"generate-api-json": "pnpm convert:config && node ./scripts/generate_api_json.js"
+		"generate-api-json": "pnpm compile-config && node ./scripts/generate_api_json.js"
 	},
 	"optionalDependencies": {
 		"@swc/core-android-arm64": "1.3.11",

packages/backend/scripts/compile_config.js

@@ -17,43 +17,38 @@ const _filename = fileURLToPath(import.meta.url);
 const _dirname = dirname(_filename);
 
 const configDir = resolve(_dirname, '../../../.config');
+const OUTPUT_PATH = resolve(_dirname, '../../../built/.config.json');
+
+// TODO: yamlのパースに失敗したときのエラーハンドリング
 
 /**
  * YAMLファイルをJSONファイルに変換
  * @param {string} ymlPath - YAMLファイルのパス
- * @param {string} jsonPath - JSONファイルの出力パス
  */
-function convertYamlToJson(ymlPath, jsonPath) {
+function yamlToJson(ymlPath) {
 	if (!fs.existsSync(ymlPath)) {
-		console.log(`skipped: ${ymlPath} is not found`);
+		console.warn(`YAML file not found: ${ymlPath}`);
 		return;
 	}
 
+	console.log(`${ymlPath} → ${OUTPUT_PATH}`);
+
 	const yamlContent = fs.readFileSync(ymlPath, 'utf-8');
 	const jsonContent = yaml.load(yamlContent);
-	fs.writeFileSync(jsonPath, JSON.stringify({
+	if (!fs.existsSync(dirname(OUTPUT_PATH))) {
+		fs.mkdirSync(dirname(OUTPUT_PATH), { recursive: true });
+	}
+	fs.writeFileSync(OUTPUT_PATH, JSON.stringify({
 		'_NOTE_': 'This file is auto-generated from YAML file. DO NOT EDIT.',
 		...jsonContent,
 	}), 'utf-8');
-	console.log(`✓ ${ymlPath} → ${jsonPath}`);
 }
 
-// default.yml と test.yml を変換
-convertYamlToJson(
-	resolve(configDir, 'default.yml'),
-	resolve(configDir, 'default.json'),
-);
-
-convertYamlToJson(
-	resolve(configDir, 'test.yml'),
-	resolve(configDir, 'test.json'),
-);
-
-// MISSKEY_CONFIG_YML 環境変数が指定されている場合も変換
 if (process.env.MISSKEY_CONFIG_YML) {
 	const customYmlPath = resolve(configDir, process.env.MISSKEY_CONFIG_YML);
-	const customJsonPath = customYmlPath.replace(/\.ya?ml$/i, '.json');
-	convertYamlToJson(customYmlPath, customJsonPath);
+	yamlToJson(customYmlPath);
+} else {
+	yamlToJson(resolve(configDir, process.env.NODE_ENV === 'test' ? 'test.yml' : 'default.yml'));
 }
 
-console.log('Configuration compiled');
+console.log('Configuration compiled ✓');

packages/backend/src/config.ts

@@ -217,21 +217,15 @@ export type FulltextSearchProvider = 'sqlLike' | 'sqlPgroonga' | 'meilisearch';
 const _filename = fileURLToPath(import.meta.url);
 const _dirname = dirname(_filename);
 
-/**
- * Path of configuration directory
- */
-const dir = `${_dirname}/../../../.config`;
+const compiledConfigFilePathForTest = resolve(_dirname, '../../../built/._config_.json');
 
-/**
- * Path of configuration file
- */
-export const path = process.env.MISSKEY_CONFIG_YML
-	? resolve(dir, process.env.MISSKEY_CONFIG_YML).replace(/\.ya?ml$/i, '.json')
-	: process.env.NODE_ENV === 'test'
-		? resolve(dir, 'test.json')
-		: resolve(dir, 'default.json');
+export const compiledConfigFilePath = fs.existsSync(compiledConfigFilePathForTest) ? compiledConfigFilePathForTest : resolve(_dirname, '../../../built/.config.json');
 
 export function loadConfig(): Config {
+	if (!fs.existsSync(compiledConfigFilePath)) {
+		throw new Error('Compiled configuration file not found. Try running \'pnpm compile-config\'.');
+	}
+
 	const meta = JSON.parse(fs.readFileSync(`${_dirname}/../../../built/meta.json`, 'utf-8'));
 
 	const frontendManifestExists = fs.existsSync(_dirname + '/../../../built/_frontend_vite_/manifest.json');
@@ -243,7 +237,7 @@ export function loadConfig(): Config {
 		JSON.parse(fs.readFileSync(`${_dirname}/../../../built/_frontend_embed_vite_/manifest.json`, 'utf-8'))
 		: { 'src/boot.ts': { file: null } };
 
-	const config = JSON.parse(fs.readFileSync(path, 'utf-8')) as Source;
+	const config = JSON.parse(fs.readFileSync(compiledConfigFilePath, 'utf-8')) as Source;
 
 	const url = tryCreateUrl(config.url ?? process.env.MISSKEY_URL ?? '');
 	const version = meta.version;

packages/backend/test-federation/.config/dummy.yml

@@ -0,0 +1,2 @@
+url: https://example.com/
+port: 3000

packages/backend/test-federation/.config/example.config.json

@@ -0,0 +1,29 @@
+{
+  "url": "https://${HOST}/",
+  "port": 3000,
+  "db": {
+    "host": "db.${HOST}",
+    "port": 5432,
+    "db": "misskey",
+    "user": "postgres",
+    "pass": "postgres"
+  },
+  "dbReplications": false,
+  "trustProxy": true,
+  "redis": {
+    "host": "redis.test",
+    "port": 6379
+  },
+  "id": "aidx",
+  "proxyBypassHosts": [
+    "api.deepl.com",
+    "api-free.deepl.com",
+    "www.recaptcha.net",
+    "hcaptcha.com",
+    "challenges.cloudflare.com"
+  ],
+  "allowedPrivateNetworks": [
+    "127.0.0.1/32",
+    "172.20.0.0/16"
+  ]
+}

packages/backend/test-federation/.config/example.default.yml

@@ -1,23 +0,0 @@
-url: https://${HOST}/
-port: 3000
-db:
-  host: db.${HOST}
-  port: 5432
-  db: misskey
-  user: postgres
-  pass: postgres
-dbReplications: false
-trustProxy: true
-redis:
-  host: redis.test
-  port: 6379
-id: 'aidx'
-proxyBypassHosts:
-  - api.deepl.com
-  - api-free.deepl.com
-  - www.recaptcha.net
-  - hcaptcha.com
-  - challenges.cloudflare.com
-allowedPrivateNetworks:
-  - 127.0.0.1/32
-  - 172.20.0.0/16

packages/backend/test-federation/compose.a.yml

@@ -37,12 +37,8 @@ services:
       - internal_network_a
     volumes:
       - type: bind
-        source: ./.config/a.test.default.yml
-        target: /misskey/.config/default.yml
-        read_only: true
-      - type: bind
-        source: ../scripts/convert_config.js
-        target: /misskey/packages/backend/scripts/convert_config.js
+        source: ./.config/a.test.config.json
+        target: /misskey/built/._config_.json
         read_only: true
 
   db.a.test:

packages/backend/test-federation/compose.b.yml

@@ -37,12 +37,8 @@ services:
       - internal_network_b
     volumes:
       - type: bind
-        source: ./.config/b.test.default.yml
-        target: /misskey/.config/default.yml
-        read_only: true
-      - type: bind
-        source: ../scripts/convert_config.js
-        target: /misskey/packages/backend/scripts/convert_config.js
+        source: ./.config/b.test.config.json
+        target: /misskey/built/._config_.json
         read_only: true
 
   db.b.test:

packages/backend/test-federation/compose.tpl.yml

@@ -21,6 +21,10 @@ services:
       - type: bind
         source: ../../../built
         target: /misskey/built
+        read_only: false
+      - type: bind
+        source: ./.config/dummy.yml
+        target: /misskey/.config/default.yml
         read_only: true
       - type: bind
         source: ../assets
@@ -43,8 +47,8 @@ services:
         target: /misskey/packages/backend/package.json
         read_only: true
       - type: bind
-        source: ../scripts/convert_config.js
-        target: /misskey/packages/backend/scripts/convert_config.js
+        source: ../scripts/compile_config.js
+        target: /misskey/packages/backend/scripts/compile_config.js
         read_only: true
       - type: bind
         source: ../../misskey-js/built

packages/backend/test-federation/compose.yml

@@ -55,8 +55,8 @@ services:
         target: /misskey/packages/backend/jest.js
         read_only: true
       - type: bind
-        source: ../scripts/convert_config.js
-        target: /misskey/packages/backend/scripts/convert_config.js
+        source: ../scripts/compile_config.js
+        target: /misskey/packages/backend/scripts/compile_config.js
         read_only: true
       - type: bind
         source: ../../misskey-js/built

packages/backend/test-federation/setup.sh

@@ -28,7 +28,7 @@ function generate {
     -days 500
   if [ ! -f .config/docker.env ]; then cp .config/example.docker.env .config/docker.env; fi
   if [ ! -f .config/$1.conf ]; then sed "s/\${HOST}/$1/g" .config/example.conf > .config/$1.conf; fi
-  if [ ! -f .config/$1.default.yml ]; then sed "s/\${HOST}/$1/g" .config/example.default.yml > .config/$1.default.yml; fi
+  if [ ! -f .config/$1.default.yml ]; then sed "s/\${HOST}/$1/g" .config/example.config.json > .config/$1.config.json; fi
 }
 
 generate a.test