diff --git a/.github/workflows/get-backend-memory.yml b/.github/workflows/get-backend-memory.yml index b14885ec34..458f303f0f 100644 --- a/.github/workflows/get-backend-memory.yml +++ b/.github/workflows/get-backend-memory.yml @@ -57,7 +57,7 @@ jobs: - name: Copy Configure run: cp .github/misskey/test.yml .config/default.yml - name: Compile Configure - run: pnpm convert:config + run: pnpm compile-config - name: Build run: pnpm build - name: Run migrations diff --git a/package.json b/package.json index e72bb70d5f..7fe2c36ffc 100644 --- a/package.json +++ b/package.json @@ -22,15 +22,15 @@ ], "private": true, "scripts": { - "convert:config": "cd packages/backend && pnpm convert:config", + "compile-config": "cd packages/backend && pnpm compile-config", "build-pre": "node ./scripts/build-pre.js", "build-assets": "node ./scripts/build-assets.mjs", "build": "pnpm build-pre && pnpm -r build && pnpm build-assets", "build-storybook": "pnpm --filter frontend build-storybook", "build-misskey-js-with-types": "pnpm build-pre && pnpm --filter backend... --filter=!misskey-js build && pnpm --filter backend generate-api-json --no-build && ncp packages/backend/built/api.json packages/misskey-js/generator/api.json && pnpm --filter misskey-js update-autogen-code && pnpm --filter misskey-js build && pnpm --filter misskey-js api", - "start": "pnpm check:connect && cd packages/backend && pnpm convert:config && node ./built/boot/entry.js", - "start:inspect": "cd packages/backend && pnpm convert:config && node --inspect ./built/boot/entry.js", - "start:test": "ncp ./.github/misskey/test.yml ./.config/test.yml && cd packages/backend && pnpm convert:config && cross-env NODE_ENV=test node ./built/boot/entry.js", + "start": "pnpm check:connect && cd packages/backend && pnpm compile-config && node ./built/boot/entry.js", + "start:inspect": "cd packages/backend && pnpm compile-config && node --inspect ./built/boot/entry.js", + "start:test": "ncp ./.github/misskey/test.yml ./.config/test.yml && cd packages/backend && cross-env NODE_ENV=test pnpm compile-config && cross-env NODE_ENV=test node ./built/boot/entry.js", "cli": "cd packages/backend && pnpm cli", "init": "pnpm migrate", "migrate": "cd packages/backend && pnpm migrate", diff --git a/packages/backend/migration/1745378064470-composite-note-index.js b/packages/backend/migration/1745378064470-composite-note-index.js index 12108a6b3c..576bf7d19a 100644 --- a/packages/backend/migration/1745378064470-composite-note-index.js +++ b/packages/backend/migration/1745378064470-composite-note-index.js @@ -3,14 +3,14 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import { isConcurrentIndexMigrationEnabled } from "./js/migration-config.js"; +const isConcurrentIndexMigrationEnabled = process.env.MISSKEY_MIGRATION_CREATE_INDEX_CONCURRENTLY === '1'; export class CompositeNoteIndex1745378064470 { name = 'CompositeNoteIndex1745378064470'; - transaction = isConcurrentIndexMigrationEnabled() ? false : undefined; + transaction = isConcurrentIndexMigrationEnabled ? false : undefined; async up(queryRunner) { - const concurrently = isConcurrentIndexMigrationEnabled(); + const concurrently = isConcurrentIndexMigrationEnabled; if (concurrently) { const hasValidIndex = await queryRunner.query(`SELECT indisvalid FROM pg_index INNER JOIN pg_class ON pg_index.indexrelid = pg_class.oid WHERE pg_class.relname = 'IDX_724b311e6f883751f261ebe378'`); @@ -29,7 +29,7 @@ export class CompositeNoteIndex1745378064470 { } async down(queryRunner) { - const mayConcurrently = isConcurrentIndexMigrationEnabled() ? 'CONCURRENTLY' : ''; + const mayConcurrently = isConcurrentIndexMigrationEnabled ? 'CONCURRENTLY' : ''; await queryRunner.query(`DROP INDEX IF EXISTS "IDX_724b311e6f883751f261ebe378"`); await queryRunner.query(`CREATE INDEX ${mayConcurrently} "IDX_5b87d9d19127bd5d92026017a7" ON "note" ("userId")`); } diff --git a/packages/backend/migration/1746949539915-migrateSomeConfigFileSettingsToMeta.js b/packages/backend/migration/1746949539915-migrateSomeConfigFileSettingsToMeta.js index 3243f43b91..cb8bb33459 100644 --- a/packages/backend/migration/1746949539915-migrateSomeConfigFileSettingsToMeta.js +++ b/packages/backend/migration/1746949539915-migrateSomeConfigFileSettingsToMeta.js @@ -3,17 +3,15 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import {loadConfig} from "./js/migration-config.js"; export class MigrateSomeConfigFileSettingsToMeta1746949539915 { name = 'MigrateSomeConfigFileSettingsToMeta1746949539915' async up(queryRunner) { - const config = loadConfig(); // $1 cannot be used in ALTER TABLE queries - await queryRunner.query(`ALTER TABLE "meta" ADD "proxyRemoteFiles" boolean NOT NULL DEFAULT ${config.proxyRemoteFiles}`); - await queryRunner.query(`ALTER TABLE "meta" ADD "signToActivityPubGet" boolean NOT NULL DEFAULT ${config.signToActivityPubGet}`); - await queryRunner.query(`ALTER TABLE "meta" ADD "allowExternalApRedirect" boolean NOT NULL DEFAULT ${!config.disallowExternalApRedirect}`); + await queryRunner.query(`ALTER TABLE "meta" ADD "proxyRemoteFiles" boolean NOT NULL DEFAULT TRUE`); + await queryRunner.query(`ALTER TABLE "meta" ADD "signToActivityPubGet" boolean NOT NULL DEFAULT TRUE`); + await queryRunner.query(`ALTER TABLE "meta" ADD "allowExternalApRedirect" boolean NOT NULL DEFAULT TRUE`); } async down(queryRunner) { diff --git a/packages/backend/migration/js/migration-config.js b/packages/backend/migration/js/migration-config.js deleted file mode 100644 index 853735661b..0000000000 --- a/packages/backend/migration/js/migration-config.js +++ /dev/null @@ -1,31 +0,0 @@ -/* - * SPDX-FileCopyrightText: syuilo and misskey-project - * SPDX-License-Identifier: AGPL-3.0-only - */ - -import { path as configYamlPath } from '../../built/config.js'; -import * as yaml from 'js-yaml'; -import fs from "node:fs"; - -export function isConcurrentIndexMigrationEnabled() { - return process.env.MISSKEY_MIGRATION_CREATE_INDEX_CONCURRENTLY === '1'; -} - -let loadedConfigCache = undefined; - -function loadConfigInternal() { - const config = yaml.load(fs.readFileSync(configYamlPath, 'utf-8')); - - return { - disallowExternalApRedirect: Boolean(config.disallowExternalApRedirect ?? false), - proxyRemoteFiles: Boolean(config.proxyRemoteFiles ?? false), - signToActivityPubGet: Boolean(config.signToActivityPubGet ?? true), - } -} - -export function loadConfig() { - if (loadedConfigCache === undefined) { - loadedConfigCache = loadConfigInternal(); - } - return loadedConfigCache; -} diff --git a/packages/backend/ormconfig.js b/packages/backend/ormconfig.js index f979c36ad7..dabc0893f4 100644 --- a/packages/backend/ormconfig.js +++ b/packages/backend/ormconfig.js @@ -1,7 +1,8 @@ import { DataSource } from 'typeorm'; import { loadConfig } from './built/config.js'; import { entities } from './built/postgres.js'; -import { isConcurrentIndexMigrationEnabled } from "./migration/js/migration-config.js"; + +const isConcurrentIndexMigrationEnabled = process.env.MISSKEY_MIGRATION_CREATE_INDEX_CONCURRENTLY === '1'; const config = loadConfig(); @@ -15,5 +16,5 @@ export default new DataSource({ extra: config.db.extra, entities: entities, migrations: ['migration/*.js'], - migrationsTransactionMode: isConcurrentIndexMigrationEnabled() ? 'each' : 'all', + migrationsTransactionMode: isConcurrentIndexMigrationEnabled ? 'each' : 'all', }); diff --git a/packages/backend/package.json b/packages/backend/package.json index 25d84a1e14..e9143b525a 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -7,37 +7,37 @@ "node": "^22.15.0 || ^24.10.0" }, "scripts": { - "start": "pnpm convert:config && node ./built/boot/entry.js", - "start:inspect": "pnpm convert:config && node --inspect ./built/boot/entry.js", - "start:test": "pnpm convert:config && cross-env NODE_ENV=test node ./built/boot/entry.js", - "migrate": "pnpm convert:config && pnpm typeorm migration:run -d ormconfig.js", - "revert": "pnpm convert:config && pnpm typeorm migration:revert -d ormconfig.js", - "cli": "pnpm convert:config && node ./built/boot/cli.js", - "check:connect": "pnpm convert:config && node ./scripts/check_connect.js", - "convert:config": "node ./scripts/convert_config.js", + "start": "pnpm compile-config && node ./built/boot/entry.js", + "start:inspect": "pnpm compile-config && node --inspect ./built/boot/entry.js", + "start:test": "cross-env NODE_ENV=test pnpm compile-config && cross-env NODE_ENV=test node ./built/boot/entry.js", + "migrate": "pnpm compile-config && pnpm typeorm migration:run -d ormconfig.js", + "revert": "pnpm compile-config && pnpm typeorm migration:revert -d ormconfig.js", + "cli": "pnpm compile-config && node ./built/boot/cli.js", + "check:connect": "pnpm compile-config && node ./scripts/check_connect.js", + "compile-config": "node ./scripts/compile_config.js", "build": "swc src -d built -D --strip-leading-paths", "build:test": "swc test-server -d built-test -D --config-file test-server/.swcrc --strip-leading-paths", "watch:swc": "swc src -d built -D -w --strip-leading-paths", "build:tsc": "tsc -p tsconfig.json && tsc-alias -p tsconfig.json", - "watch": "pnpm convert:config && node ./scripts/watch.mjs", + "watch": "pnpm compile-config && node ./scripts/watch.mjs", "restart": "pnpm build && pnpm start", - "dev": "pnpm convert:config && node ./scripts/dev.mjs", + "dev": "pnpm compile-config && node ./scripts/dev.mjs", "typecheck": "tsc --noEmit && tsc -p test --noEmit && tsc -p test-federation --noEmit", "eslint": "eslint --quiet \"{src,test-federation}/**/*.ts\"", "lint": "pnpm typecheck && pnpm eslint", - "jest": "pnpm convert:config && cross-env NODE_ENV=test node ./jest.js --forceExit --config jest.config.unit.cjs", - "jest:e2e": "pnpm convert:config && cross-env NODE_ENV=test node ./jest.js --forceExit --config jest.config.e2e.cjs", - "jest:fed": "pnpm convert:config && node ./jest.js --forceExit --config jest.config.fed.cjs", - "jest-and-coverage": "pnpm convert:config && cross-env NODE_ENV=test node ./jest.js --coverage --forceExit --config jest.config.unit.cjs", - "jest-and-coverage:e2e": "pnpm convert:config && cross-env NODE_ENV=test node ./jest.js --coverage --forceExit --config jest.config.e2e.cjs", - "jest-clear": "pnpm convert:config && cross-env NODE_ENV=test node ./jest.js --clearCache", + "jest": "cross-env NODE_ENV=test pnpm compile-config && cross-env NODE_ENV=test node ./jest.js --forceExit --config jest.config.unit.cjs", + "jest:e2e": "cross-env NODE_ENV=test pnpm compile-config && cross-env NODE_ENV=test node ./jest.js --forceExit --config jest.config.e2e.cjs", + "jest:fed": "pnpm compile-config && node ./jest.js --forceExit --config jest.config.fed.cjs", + "jest-and-coverage": "cross-env NODE_ENV=test pnpm compile-config && cross-env NODE_ENV=test node ./jest.js --coverage --forceExit --config jest.config.unit.cjs", + "jest-and-coverage:e2e": "cross-env NODE_ENV=test pnpm compile-config && cross-env NODE_ENV=test node ./jest.js --coverage --forceExit --config jest.config.e2e.cjs", + "jest-clear": "cross-env NODE_ENV=test pnpm compile-config && cross-env NODE_ENV=test node ./jest.js --clearCache", "test": "pnpm jest", "test:e2e": "pnpm build && pnpm build:test && pnpm jest:e2e", "test:fed": "pnpm jest:fed", "test-and-coverage": "pnpm jest-and-coverage", "test-and-coverage:e2e": "pnpm build && pnpm build:test && pnpm jest-and-coverage:e2e", "check-migrations": "node scripts/check_migrations_clean.js", - "generate-api-json": "pnpm convert:config && node ./scripts/generate_api_json.js" + "generate-api-json": "pnpm compile-config && node ./scripts/generate_api_json.js" }, "optionalDependencies": { "@swc/core-android-arm64": "1.3.11", diff --git a/packages/backend/scripts/convert_config.js b/packages/backend/scripts/compile_config.js similarity index 59% rename from packages/backend/scripts/convert_config.js rename to packages/backend/scripts/compile_config.js index e93baa409a..e78fa3dc9f 100644 --- a/packages/backend/scripts/convert_config.js +++ b/packages/backend/scripts/compile_config.js @@ -17,43 +17,38 @@ const _filename = fileURLToPath(import.meta.url); const _dirname = dirname(_filename); const configDir = resolve(_dirname, '../../../.config'); +const OUTPUT_PATH = resolve(_dirname, '../../../built/.config.json'); + +// TODO: yamlのパースに失敗したときのエラーハンドリング /** * YAMLファイルをJSONファイルに変換 * @param {string} ymlPath - YAMLファイルのパス - * @param {string} jsonPath - JSONファイルの出力パス */ -function convertYamlToJson(ymlPath, jsonPath) { +function yamlToJson(ymlPath) { if (!fs.existsSync(ymlPath)) { - console.log(`skipped: ${ymlPath} is not found`); + console.warn(`YAML file not found: ${ymlPath}`); return; } + console.log(`${ymlPath} → ${OUTPUT_PATH}`); + const yamlContent = fs.readFileSync(ymlPath, 'utf-8'); const jsonContent = yaml.load(yamlContent); - fs.writeFileSync(jsonPath, JSON.stringify({ + if (!fs.existsSync(dirname(OUTPUT_PATH))) { + fs.mkdirSync(dirname(OUTPUT_PATH), { recursive: true }); + } + fs.writeFileSync(OUTPUT_PATH, JSON.stringify({ '_NOTE_': 'This file is auto-generated from YAML file. DO NOT EDIT.', ...jsonContent, }), 'utf-8'); - console.log(`✓ ${ymlPath} → ${jsonPath}`); } -// default.yml と test.yml を変換 -convertYamlToJson( - resolve(configDir, 'default.yml'), - resolve(configDir, 'default.json'), -); - -convertYamlToJson( - resolve(configDir, 'test.yml'), - resolve(configDir, 'test.json'), -); - -// MISSKEY_CONFIG_YML 環境変数が指定されている場合も変換 if (process.env.MISSKEY_CONFIG_YML) { const customYmlPath = resolve(configDir, process.env.MISSKEY_CONFIG_YML); - const customJsonPath = customYmlPath.replace(/\.ya?ml$/i, '.json'); - convertYamlToJson(customYmlPath, customJsonPath); + yamlToJson(customYmlPath); +} else { + yamlToJson(resolve(configDir, process.env.NODE_ENV === 'test' ? 'test.yml' : 'default.yml')); } -console.log('Configuration compiled'); +console.log('Configuration compiled ✓'); diff --git a/packages/backend/src/config.ts b/packages/backend/src/config.ts index 53946a0bdc..f9852d3578 100644 --- a/packages/backend/src/config.ts +++ b/packages/backend/src/config.ts @@ -217,21 +217,15 @@ export type FulltextSearchProvider = 'sqlLike' | 'sqlPgroonga' | 'meilisearch'; const _filename = fileURLToPath(import.meta.url); const _dirname = dirname(_filename); -/** - * Path of configuration directory - */ -const dir = `${_dirname}/../../../.config`; +const compiledConfigFilePathForTest = resolve(_dirname, '../../../built/._config_.json'); -/** - * Path of configuration file - */ -export const path = process.env.MISSKEY_CONFIG_YML - ? resolve(dir, process.env.MISSKEY_CONFIG_YML).replace(/\.ya?ml$/i, '.json') - : process.env.NODE_ENV === 'test' - ? resolve(dir, 'test.json') - : resolve(dir, 'default.json'); +export const compiledConfigFilePath = fs.existsSync(compiledConfigFilePathForTest) ? compiledConfigFilePathForTest : resolve(_dirname, '../../../built/.config.json'); export function loadConfig(): Config { + if (!fs.existsSync(compiledConfigFilePath)) { + throw new Error('Compiled configuration file not found. Try running \'pnpm compile-config\'.'); + } + const meta = JSON.parse(fs.readFileSync(`${_dirname}/../../../built/meta.json`, 'utf-8')); const frontendManifestExists = fs.existsSync(_dirname + '/../../../built/_frontend_vite_/manifest.json'); @@ -243,7 +237,7 @@ export function loadConfig(): Config { JSON.parse(fs.readFileSync(`${_dirname}/../../../built/_frontend_embed_vite_/manifest.json`, 'utf-8')) : { 'src/boot.ts': { file: null } }; - const config = JSON.parse(fs.readFileSync(path, 'utf-8')) as Source; + const config = JSON.parse(fs.readFileSync(compiledConfigFilePath, 'utf-8')) as Source; const url = tryCreateUrl(config.url ?? process.env.MISSKEY_URL ?? ''); const version = meta.version; diff --git a/packages/backend/test-federation/.config/dummy.yml b/packages/backend/test-federation/.config/dummy.yml new file mode 100644 index 0000000000..841cab9783 --- /dev/null +++ b/packages/backend/test-federation/.config/dummy.yml @@ -0,0 +1,2 @@ +url: https://example.com/ +port: 3000 diff --git a/packages/backend/test-federation/.config/example.config.json b/packages/backend/test-federation/.config/example.config.json new file mode 100644 index 0000000000..2035d1a200 --- /dev/null +++ b/packages/backend/test-federation/.config/example.config.json @@ -0,0 +1,29 @@ +{ + "url": "https://${HOST}/", + "port": 3000, + "db": { + "host": "db.${HOST}", + "port": 5432, + "db": "misskey", + "user": "postgres", + "pass": "postgres" + }, + "dbReplications": false, + "trustProxy": true, + "redis": { + "host": "redis.test", + "port": 6379 + }, + "id": "aidx", + "proxyBypassHosts": [ + "api.deepl.com", + "api-free.deepl.com", + "www.recaptcha.net", + "hcaptcha.com", + "challenges.cloudflare.com" + ], + "allowedPrivateNetworks": [ + "127.0.0.1/32", + "172.20.0.0/16" + ] +} diff --git a/packages/backend/test-federation/.config/example.default.yml b/packages/backend/test-federation/.config/example.default.yml deleted file mode 100644 index b1cd3bdba8..0000000000 --- a/packages/backend/test-federation/.config/example.default.yml +++ /dev/null @@ -1,23 +0,0 @@ -url: https://${HOST}/ -port: 3000 -db: - host: db.${HOST} - port: 5432 - db: misskey - user: postgres - pass: postgres -dbReplications: false -trustProxy: true -redis: - host: redis.test - port: 6379 -id: 'aidx' -proxyBypassHosts: - - api.deepl.com - - api-free.deepl.com - - www.recaptcha.net - - hcaptcha.com - - challenges.cloudflare.com -allowedPrivateNetworks: - - 127.0.0.1/32 - - 172.20.0.0/16 diff --git a/packages/backend/test-federation/compose.a.yml b/packages/backend/test-federation/compose.a.yml index 1d3d8fd404..4fd4eb3851 100644 --- a/packages/backend/test-federation/compose.a.yml +++ b/packages/backend/test-federation/compose.a.yml @@ -37,12 +37,8 @@ services: - internal_network_a volumes: - type: bind - source: ./.config/a.test.default.yml - target: /misskey/.config/default.yml - read_only: true - - type: bind - source: ../scripts/convert_config.js - target: /misskey/packages/backend/scripts/convert_config.js + source: ./.config/a.test.config.json + target: /misskey/built/._config_.json read_only: true db.a.test: diff --git a/packages/backend/test-federation/compose.b.yml b/packages/backend/test-federation/compose.b.yml index 9a3bfb8abe..753da22822 100644 --- a/packages/backend/test-federation/compose.b.yml +++ b/packages/backend/test-federation/compose.b.yml @@ -37,12 +37,8 @@ services: - internal_network_b volumes: - type: bind - source: ./.config/b.test.default.yml - target: /misskey/.config/default.yml - read_only: true - - type: bind - source: ../scripts/convert_config.js - target: /misskey/packages/backend/scripts/convert_config.js + source: ./.config/b.test.config.json + target: /misskey/built/._config_.json read_only: true db.b.test: diff --git a/packages/backend/test-federation/compose.tpl.yml b/packages/backend/test-federation/compose.tpl.yml index ad9e127b3a..1404345e2a 100644 --- a/packages/backend/test-federation/compose.tpl.yml +++ b/packages/backend/test-federation/compose.tpl.yml @@ -21,6 +21,10 @@ services: - type: bind source: ../../../built target: /misskey/built + read_only: false + - type: bind + source: ./.config/dummy.yml + target: /misskey/.config/default.yml read_only: true - type: bind source: ../assets @@ -43,8 +47,8 @@ services: target: /misskey/packages/backend/package.json read_only: true - type: bind - source: ../scripts/convert_config.js - target: /misskey/packages/backend/scripts/convert_config.js + source: ../scripts/compile_config.js + target: /misskey/packages/backend/scripts/compile_config.js read_only: true - type: bind source: ../../misskey-js/built diff --git a/packages/backend/test-federation/compose.yml b/packages/backend/test-federation/compose.yml index 07c0e03f99..25475a89ab 100644 --- a/packages/backend/test-federation/compose.yml +++ b/packages/backend/test-federation/compose.yml @@ -55,8 +55,8 @@ services: target: /misskey/packages/backend/jest.js read_only: true - type: bind - source: ../scripts/convert_config.js - target: /misskey/packages/backend/scripts/convert_config.js + source: ../scripts/compile_config.js + target: /misskey/packages/backend/scripts/compile_config.js read_only: true - type: bind source: ../../misskey-js/built diff --git a/packages/backend/test-federation/setup.sh b/packages/backend/test-federation/setup.sh index 1bc3a2a87c..15aa2eee7f 100644 --- a/packages/backend/test-federation/setup.sh +++ b/packages/backend/test-federation/setup.sh @@ -28,7 +28,7 @@ function generate { -days 500 if [ ! -f .config/docker.env ]; then cp .config/example.docker.env .config/docker.env; fi if [ ! -f .config/$1.conf ]; then sed "s/\${HOST}/$1/g" .config/example.conf > .config/$1.conf; fi - if [ ! -f .config/$1.default.yml ]; then sed "s/\${HOST}/$1/g" .config/example.default.yml > .config/$1.default.yml; fi + if [ ! -f .config/$1.default.yml ]; then sed "s/\${HOST}/$1/g" .config/example.config.json > .config/$1.config.json; fi } generate a.test