Compare commits

...

12 Commits

Author SHA1 Message Date
syuilo 1f8d6b286b Update package.json 2025-12-02 11:26:49 +09:00
syuilo 731a1daaf2 wip 2025-12-02 11:22:57 +09:00
syuilo 791684efc0 wip 2025-12-02 11:03:23 +09:00
syuilo 570a795785 wip 2025-12-02 09:53:27 +09:00
syuilo 36bec1cadb
Merge branch 'develop' into copilot/remove-js-yaml-runtime-dependency 2025-12-02 09:35:01 +09:00
syuilo 811d6dc8b3 Reapply "Fix federation test: add scripts mount and pre-compile config on host"
This reverts commit a7e4518131.
2025-12-02 09:15:28 +09:00
syuilo a7e4518131 Revert "Fix federation test: add scripts mount and pre-compile config on host"
This reverts commit d6d5606d68.
2025-12-01 14:10:51 +09:00
copilot-swe-agent[bot] d6d5606d68 Fix federation test: add scripts mount and pre-compile config on host
Co-authored-by: syuilo <4439005+syuilo@users.noreply.github.com>
2025-12-01 04:24:32 +00:00
syuilo bf50e2ddde
Update generate-api-json script command 2025-12-01 12:50:58 +09:00
copilot-swe-agent[bot] ffdc78b354 Use safe yaml.JSON_SCHEMA to prevent code execution vulnerabilities
Co-authored-by: syuilo <4439005+syuilo@users.noreply.github.com>
2025-12-01 02:18:44 +00:00
copilot-swe-agent[bot] 69c8d8e102 Remove js-yaml from runtime dependencies, use pre-compiled JSON instead
Co-authored-by: syuilo <4439005+syuilo@users.noreply.github.com>
2025-12-01 02:11:02 +00:00
copilot-swe-agent[bot] 5e6fe5bef6 Initial plan 2025-12-01 02:00:34 +00:00
10 changed files with 76 additions and 52 deletions

View File

@ -22,31 +22,32 @@
], ],
"private": true, "private": true,
"scripts": { "scripts": {
"compile-config": "node ./scripts/compile_config.js",
"build-pre": "node ./scripts/build-pre.js", "build-pre": "node ./scripts/build-pre.js",
"build-assets": "node ./scripts/build-assets.mjs", "build-assets": "node ./scripts/build-assets.mjs",
"build": "pnpm build-pre && pnpm -r build && pnpm build-assets", "build": "pnpm build-pre && pnpm -r build && pnpm build-assets",
"build-storybook": "pnpm --filter frontend build-storybook", "build-storybook": "pnpm --filter frontend build-storybook",
"build-misskey-js-with-types": "pnpm build-pre && pnpm --filter backend... --filter=!misskey-js build && pnpm --filter backend generate-api-json --no-build && ncp packages/backend/built/api.json packages/misskey-js/generator/api.json && pnpm --filter misskey-js update-autogen-code && pnpm --filter misskey-js build && pnpm --filter misskey-js api", "build-misskey-js-with-types": "pnpm build-pre && pnpm --filter backend... --filter=!misskey-js build && pnpm --filter backend generate-api-json --no-build && ncp packages/backend/built/api.json packages/misskey-js/generator/api.json && pnpm --filter misskey-js update-autogen-code && pnpm --filter misskey-js build && pnpm --filter misskey-js api",
"start": "pnpm check:connect && cd packages/backend && node ./built/boot/entry.js", "start": "pnpm compile-config && pnpm check:connect && cd packages/backend && pnpm start",
"start:inspect": "cd packages/backend && node --inspect ./built/boot/entry.js", "start:inspect": "cd packages/backend && pnpm start:inspect",
"start:test": "ncp ./.github/misskey/test.yml ./.config/test.yml && cd packages/backend && cross-env NODE_ENV=test node ./built/boot/entry.js", "start:test": "ncp ./.github/misskey/test.yml ./.config/test.yml && cross-env NODE_ENV=test pnpm compile-config && cd packages/backend && pnpm start:test",
"cli": "cd packages/backend && pnpm cli", "cli": "cd packages/backend && pnpm cli",
"init": "pnpm migrate", "init": "pnpm migrate",
"migrate": "cd packages/backend && pnpm migrate", "migrate": "pnpm compile-config && cd packages/backend && pnpm migrate",
"revert": "cd packages/backend && pnpm revert", "revert": "pnpm compile-config && cd packages/backend && pnpm revert",
"check:connect": "cd packages/backend && pnpm check:connect", "check:connect": "pnpm compile-config && cd packages/backend && pnpm check:connect",
"migrateandstart": "pnpm migrate && pnpm start", "migrateandstart": "pnpm migrate && pnpm start",
"watch": "pnpm dev", "watch": "pnpm dev",
"dev": "node scripts/dev.mjs", "dev": "node scripts/dev.mjs",
"lint": "pnpm -r lint", "lint": "pnpm -r lint",
"cy:open": "pnpm cypress open --config-file=cypress.config.ts", "cy:open": "pnpm cypress open --config-file=cypress.config.ts",
"cy:run": "pnpm cypress run", "cy:run": "pnpm cypress run",
"e2e": "pnpm start-server-and-test start:test http://localhost:61812 cy:run", "e2e": "cross-env NODE_ENV=test pnpm compile-config && pnpm start-server-and-test start:test http://localhost:61812 cy:run",
"e2e-dev-container": "ncp ./.config/cypress-devcontainer.yml ./.config/test.yml && pnpm start-server-and-test start:test http://localhost:61812 cy:run", "e2e-dev-container": "ncp ./.config/cypress-devcontainer.yml ./.config/test.yml && cross-env NODE_ENV=test pnpm compile-config && pnpm start-server-and-test start:test http://localhost:61812 cy:run",
"jest": "cd packages/backend && pnpm jest", "jest": "cross-env NODE_ENV=test pnpm compile-config && cd packages/backend && pnpm jest",
"jest-and-coverage": "cd packages/backend && pnpm jest-and-coverage", "jest-and-coverage": "cross-env NODE_ENV=test pnpm compile-config && cd packages/backend && pnpm jest-and-coverage",
"test": "pnpm -r test", "test": "cross-env NODE_ENV=test pnpm compile-config && pnpm -r test",
"test-and-coverage": "pnpm -r test-and-coverage", "test-and-coverage": "cross-env NODE_ENV=test pnpm compile-config && pnpm -r test-and-coverage",
"clean": "node ./scripts/clean.js", "clean": "node ./scripts/clean.js",
"clean-all": "node ./scripts/clean-all.js", "clean-all": "node ./scripts/clean-all.js",
"cleanall": "pnpm clean-all" "cleanall": "pnpm clean-all"

View File

@ -3,8 +3,7 @@
* SPDX-License-Identifier: AGPL-3.0-only * SPDX-License-Identifier: AGPL-3.0-only
*/ */
import { path as configYamlPath } from '../../built/config.js'; import { path as configJsonPath } from '../../built/config.js';
import * as yaml from 'js-yaml';
import fs from "node:fs"; import fs from "node:fs";
export function isConcurrentIndexMigrationEnabled() { export function isConcurrentIndexMigrationEnabled() {
@ -14,7 +13,7 @@ export function isConcurrentIndexMigrationEnabled() {
let loadedConfigCache = undefined; let loadedConfigCache = undefined;
function loadConfigInternal() { function loadConfigInternal() {
const config = yaml.load(fs.readFileSync(configYamlPath, 'utf-8')); const config = JSON.parse(fs.readFileSync(configJsonPath, 'utf-8'));
return { return {
disallowExternalApRedirect: Boolean(config.disallowExternalApRedirect ?? false), disallowExternalApRedirect: Boolean(config.disallowExternalApRedirect ?? false),

View File

@ -128,7 +128,6 @@
"ip-cidr": "4.0.2", "ip-cidr": "4.0.2",
"ipaddr.js": "2.2.0", "ipaddr.js": "2.2.0",
"is-svg": "6.1.0", "is-svg": "6.1.0",
"js-yaml": "4.1.1",
"json5": "2.2.3", "json5": "2.2.3",
"jsonld": "9.0.0", "jsonld": "9.0.0",
"jsrsasign": "11.1.0", "jsrsasign": "11.1.0",
@ -198,7 +197,6 @@
"@types/fluent-ffmpeg": "2.1.28", "@types/fluent-ffmpeg": "2.1.28",
"@types/http-link-header": "1.0.7", "@types/http-link-header": "1.0.7",
"@types/jest": "29.5.14", "@types/jest": "29.5.14",
"@types/js-yaml": "4.0.9",
"@types/jsonld": "1.5.15", "@types/jsonld": "1.5.15",
"@types/jsrsasign": "10.5.15", "@types/jsrsasign": "10.5.15",
"@types/mime-types": "3.0.1", "@types/mime-types": "3.0.1",

View File

@ -6,7 +6,6 @@
import * as fs from 'node:fs'; import * as fs from 'node:fs';
import { fileURLToPath } from 'node:url'; import { fileURLToPath } from 'node:url';
import { dirname, resolve } from 'node:path'; import { dirname, resolve } from 'node:path';
import * as yaml from 'js-yaml';
import { type FastifyServerOptions } from 'fastify'; import { type FastifyServerOptions } from 'fastify';
import type * as Sentry from '@sentry/node'; import type * as Sentry from '@sentry/node';
import type * as SentryVue from '@sentry/vue'; import type * as SentryVue from '@sentry/vue';
@ -226,11 +225,7 @@ const dir = `${_dirname}/../../../.config`;
/** /**
* Path of configuration file * Path of configuration file
*/ */
export const path = process.env.MISSKEY_CONFIG_YML export const path = resolve(dir, 'config.json');
? resolve(dir, process.env.MISSKEY_CONFIG_YML)
: process.env.NODE_ENV === 'test'
? resolve(dir, 'test.yml')
: resolve(dir, 'default.yml');
export function loadConfig(): Config { export function loadConfig(): Config {
const meta = JSON.parse(fs.readFileSync(`${_dirname}/../../../built/meta.json`, 'utf-8')); const meta = JSON.parse(fs.readFileSync(`${_dirname}/../../../built/meta.json`, 'utf-8'));
@ -244,7 +239,7 @@ export function loadConfig(): Config {
JSON.parse(fs.readFileSync(`${_dirname}/../../../built/_frontend_embed_vite_/manifest.json`, 'utf-8')) JSON.parse(fs.readFileSync(`${_dirname}/../../../built/_frontend_embed_vite_/manifest.json`, 'utf-8'))
: { 'src/boot.ts': { file: null } }; : { 'src/boot.ts': { file: null } };
const config = yaml.load(fs.readFileSync(path, 'utf-8')) as Source; const config = JSON.parse(fs.readFileSync(path, 'utf-8')) as Source;
const url = tryCreateUrl(config.url ?? process.env.MISSKEY_URL ?? ''); const url = tryCreateUrl(config.url ?? process.env.MISSKEY_URL ?? '');
const version = meta.version; const version = meta.version;

View File

@ -11,6 +11,7 @@ import * as esbuild from 'esbuild';
import { build } from 'esbuild'; import { build } from 'esbuild';
import { execa } from 'execa'; import { execa } from 'execa';
import { globSync } from 'glob'; import { globSync } from 'glob';
import * as yaml from 'js-yaml';
import { generateLocaleInterface } from './scripts/generateLocaleInterface.js'; import { generateLocaleInterface } from './scripts/generateLocaleInterface.js';
import type { BuildOptions, BuildResult, Plugin, PluginBuild } from 'esbuild'; import type { BuildOptions, BuildResult, Plugin, PluginBuild } from 'esbuild';
@ -49,7 +50,18 @@ if (args.includes('--watch')) {
await buildSrc(); await buildSrc();
} }
function copyLocales(): void { /**
* YAMLが壊れるので取り除く
*/
function clean(text: string) {
return text.replace(new RegExp(String.fromCodePoint(0x08), 'g'), '');
}
/**
* Convert locale YAML files to JSON during build
* This allows runtime to avoid loading js-yaml
*/
function compileLocales(): void {
const srcDir = _localesDir; const srcDir = _localesDir;
const destDir = resolve(_dirname, 'built/locales'); const destDir = resolve(_dirname, 'built/locales');
@ -57,9 +69,12 @@ function copyLocales(): void {
const files = fs.readdirSync(srcDir).filter(f => f.endsWith('.yml')); const files = fs.readdirSync(srcDir).filter(f => f.endsWith('.yml'));
for (const file of files) { for (const file of files) {
fs.copyFileSync(resolve(srcDir, file), resolve(destDir, file)); const yamlContent = clean(fs.readFileSync(resolve(srcDir, file), 'utf-8'));
const jsonContent = yaml.load(yamlContent, { schema: yaml.JSON_SCHEMA });
const jsonFile = file.replace(/\.yml$/, '.json');
fs.writeFileSync(resolve(destDir, jsonFile), JSON.stringify(jsonContent), 'utf-8');
} }
console.log(`[${_package.name}] locales copied (${files.length} files).`); console.log(`[${_package.name}] locales compiled to JSON (${files.length} files).`);
} }
/** /**
@ -87,7 +102,7 @@ async function buildSrc(): Promise<void> {
process.exit(1); process.exit(1);
}); });
copyLocales(); compileLocales();
await writeFrontendLocalesJson(); await writeFrontendLocalesJson();
if (process.env.NODE_ENV === 'production') { if (process.env.NODE_ENV === 'production') {
@ -123,7 +138,7 @@ async function watchSrc(): Promise<void> {
localesWatcher.on('all', async (event, path) => { localesWatcher.on('all', async (event, path) => {
if (!path.endsWith('.yml')) return; if (!path.endsWith('.yml')) return;
console.log(`[${_package.name}] locales changed: ${event} ${path}`); console.log(`[${_package.name}] locales changed: ${event} ${path}`);
copyLocales(); compileLocales();
await writeFrontendLocalesJson(); await writeFrontendLocalesJson();
await generateLocaleInterface(_localesDir); await generateLocaleInterface(_localesDir);
}); });

View File

@ -36,11 +36,9 @@
"esbuild": "0.27.0", "esbuild": "0.27.0",
"execa": "9.6.0", "execa": "9.6.0",
"glob": "11.1.0", "glob": "11.1.0",
"js-yaml": "4.1.1",
"nodemon": "3.1.11", "nodemon": "3.1.11",
"tsx": "4.20.6", "tsx": "4.20.6",
"typescript": "5.9.3" "typescript": "5.9.3"
},
"dependencies": {
"js-yaml": "4.1.1"
} }
} }

View File

@ -62,7 +62,7 @@ function createMembers(record: LocaleRecord): ts.TypeElement[] {
} }
export async function generateLocaleInterface(localesDir: string): Promise<void> { export async function generateLocaleInterface(localesDir: string): Promise<void> {
const locale = yaml.load(fs.readFileSync(`${localesDir}/ja-JP.yml`, 'utf-8').toString()) as LocaleRecord; const locale = yaml.load(fs.readFileSync(`${localesDir}/ja-JP.yml`, 'utf-8').toString(), { schema: yaml.JSON_SCHEMA }) as LocaleRecord;
const members = createMembers(locale); const members = createMembers(locale);
const elements: ts.Statement[] = [ const elements: ts.Statement[] = [

View File

@ -8,7 +8,6 @@
*/ */
import * as fs from 'node:fs'; import * as fs from 'node:fs';
import * as yaml from 'js-yaml';
import { languages, primaries } from './const.js'; import { languages, primaries } from './const.js';
import type { Locale } from './autogen/locale.js'; import type { Locale } from './autogen/locale.js';
import type { ILocale, ParameterizedString } from './types.js'; import type { ILocale, ParameterizedString } from './types.js';
@ -35,13 +34,6 @@ function merge<T extends ILocale>(...args: (T | ILocale | undefined)[]): T {
}), {} as ILocale) as T; }), {} as ILocale) as T;
} }
/**
* YAMLが壊れるので取り除く
*/
function clean (text: string) {
return text.replace(new RegExp(String.fromCodePoint(0x08), 'g'), '');
}
/** /**
* *
*/ */
@ -62,7 +54,7 @@ function build(): Record<Language, Locale> {
// https://github.com/misskey-dev/misskey/pull/14057#issuecomment-2192833785 // https://github.com/misskey-dev/misskey/pull/14057#issuecomment-2192833785
const metaUrl = import.meta.url; const metaUrl = import.meta.url;
const locales = languages.reduce<Locales>((a, lang) => { const locales = languages.reduce<Locales>((a, lang) => {
a[lang] = (yaml.load(clean(fs.readFileSync(new URL(`./locales/${lang}.yml`, metaUrl), 'utf-8'))) ?? {}) as ILocale; a[lang] = (JSON.parse(fs.readFileSync(new URL(`./locales/${lang}.json`, metaUrl), 'utf-8')) ?? {}) as ILocale;
return a; return a;
}, {} as Locales); }, {} as Locales);

View File

@ -267,9 +267,6 @@ importers:
is-svg: is-svg:
specifier: 6.1.0 specifier: 6.1.0
version: 6.1.0 version: 6.1.0
js-yaml:
specifier: 4.1.1
version: 4.1.1
json5: json5:
specifier: 2.2.3 specifier: 2.2.3
version: 2.2.3 version: 2.2.3
@ -472,9 +469,6 @@ importers:
'@types/jest': '@types/jest':
specifier: 29.5.14 specifier: 29.5.14
version: 29.5.14 version: 29.5.14
'@types/js-yaml':
specifier: 4.0.9
version: 4.0.9
'@types/jsonld': '@types/jsonld':
specifier: 1.5.15 specifier: 1.5.15
version: 1.5.15 version: 1.5.15
@ -1331,10 +1325,6 @@ importers:
version: 10.2.0(eslint@9.39.1) version: 10.2.0(eslint@9.39.1)
packages/i18n: packages/i18n:
dependencies:
js-yaml:
specifier: 4.1.1
version: 4.1.1
devDependencies: devDependencies:
'@types/js-yaml': '@types/js-yaml':
specifier: 4.0.9 specifier: 4.0.9
@ -1360,6 +1350,9 @@ importers:
glob: glob:
specifier: 11.1.0 specifier: 11.1.0
version: 11.1.0 version: 11.1.0
js-yaml:
specifier: 4.1.1
version: 4.1.1
nodemon: nodemon:
specifier: 3.1.11 specifier: 3.1.11
version: 3.1.11 version: 3.1.11

33
scripts/compile_config.js Normal file
View File

@ -0,0 +1,33 @@
/*
* SPDX-FileCopyrightText: syuilo and misskey-project
* SPDX-License-Identifier: AGPL-3.0-only
*/
import * as fs from 'node:fs';
import { fileURLToPath } from 'node:url';
import { dirname, resolve } from 'node:path';
import * as yaml from 'js-yaml';
const _filename = fileURLToPath(import.meta.url);
const _dirname = dirname(_filename);
const dir = `${_dirname}/../.config`;
const configYmlPath = process.env.MISSKEY_CONFIG_YML
? resolve(dir, process.env.MISSKEY_CONFIG_YML)
: process.env.NODE_ENV === 'test'
? resolve(dir, 'test.yml')
: resolve(dir, 'default.yml');
const configJsonPath = resolve(dir, 'config.json');
if (!fs.existsSync(configYmlPath)) {
console.error(`Configuration file not found: ${configYmlPath}`);
process.exit(1);
}
const yamlContent = fs.readFileSync(configYmlPath, 'utf-8');
const config = yaml.load(yamlContent, { schema: yaml.JSON_SCHEMA });
fs.writeFileSync(configJsonPath, JSON.stringify(config), 'utf-8');
console.log(`Compiled config: ${configYmlPath} -> ${configJsonPath}`);