From 16236fbc80a35eafc6b0e55df7eeb972befa04c5 Mon Sep 17 00:00:00 2001 From: anatawa12 Date: Wed, 30 Apr 2025 21:48:40 +0900 Subject: [PATCH 1/9] chore: CREATE INDEX CONCURRENTLY for "userId" "id" composite note index --- .../migration/1745378064470-composite-note-index.js | 5 +++-- packages/backend/ormconfig.js | 1 + packages/backend/src/models/Note.ts | 8 +++++++- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/packages/backend/migration/1745378064470-composite-note-index.js b/packages/backend/migration/1745378064470-composite-note-index.js index 49e835d38c..773afb636c 100644 --- a/packages/backend/migration/1745378064470-composite-note-index.js +++ b/packages/backend/migration/1745378064470-composite-note-index.js @@ -5,9 +5,10 @@ export class CompositeNoteIndex1745378064470 { name = 'CompositeNoteIndex1745378064470'; + transaction = false; async up(queryRunner) { - await queryRunner.query(`CREATE INDEX "IDX_724b311e6f883751f261ebe378" ON "note" ("userId", "id" DESC)`); + await queryRunner.query(`CREATE INDEX CONCURRENTLY "IDX_724b311e6f883751f261ebe378" ON "note" ("userId", "id" DESC)`); await queryRunner.query(`DROP INDEX IF EXISTS "IDX_5b87d9d19127bd5d92026017a7"`); // Flush all cached Linear Scan Plans and redo statistics for composite index // this is important for Postgres to learn that even in highly complex queries, using this index first can reduce the result set significantly @@ -16,6 +17,6 @@ export class CompositeNoteIndex1745378064470 { async down(queryRunner) { await queryRunner.query(`DROP INDEX IF EXISTS "IDX_724b311e6f883751f261ebe378"`); - await queryRunner.query(`CREATE INDEX "IDX_5b87d9d19127bd5d92026017a7" ON "note" ("userId")`); + await queryRunner.query(`CREATE INDEX CONCURRENTLY "IDX_5b87d9d19127bd5d92026017a7" ON "note" ("userId")`); } } diff --git a/packages/backend/ormconfig.js b/packages/backend/ormconfig.js index 229e5bf1fe..23774f21fd 100644 --- a/packages/backend/ormconfig.js +++ b/packages/backend/ormconfig.js @@ -14,4 +14,5 @@ export default new DataSource({ extra: config.db.extra, entities: entities, migrations: ['migration/*.js'], + migrationsTransactionMode: 'each', }); diff --git a/packages/backend/src/models/Note.ts b/packages/backend/src/models/Note.ts index c5ca2b5776..fe499bdc0b 100644 --- a/packages/backend/src/models/Note.ts +++ b/packages/backend/src/models/Note.ts @@ -10,7 +10,13 @@ import { MiUser } from './User.js'; import { MiChannel } from './Channel.js'; import type { MiDriveFile } from './DriveFile.js'; -@Index(['userId', 'id']) +// Note: When you create a new index for existing column of this table, +// it might be better to index concurrently by setting `{ concurrent: true }`. +// Since this table is very large, and it takes a long time to create index in most cases. +// Please note that `CREATE INDEX CONCURRENTLY` is not supported in transaction, +// so you need to set `transaction = false` in migration. +// Please refer 1745378064470-composite-note-index.js for example. +@Index(['userId', 'id'], { concurrent: true }) @Entity('note') export class MiNote { @PrimaryColumn(id()) From 61cbb1c60bc2390f91fad4492908431c3509623e Mon Sep 17 00:00:00 2001 From: anatawa12 Date: Wed, 30 Apr 2025 22:51:20 +0900 Subject: [PATCH 2/9] chore: remove { concurrent: true } and comment why --- packages/backend/src/models/Note.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/models/Note.ts b/packages/backend/src/models/Note.ts index fe499bdc0b..4f8a0f9184 100644 --- a/packages/backend/src/models/Note.ts +++ b/packages/backend/src/models/Note.ts @@ -11,12 +11,14 @@ import { MiChannel } from './Channel.js'; import type { MiDriveFile } from './DriveFile.js'; // Note: When you create a new index for existing column of this table, -// it might be better to index concurrently by setting `{ concurrent: true }`. +// it might be better to index concurrently by editing generated migration file // Since this table is very large, and it takes a long time to create index in most cases. // Please note that `CREATE INDEX CONCURRENTLY` is not supported in transaction, // so you need to set `transaction = false` in migration. // Please refer 1745378064470-composite-note-index.js for example. -@Index(['userId', 'id'], { concurrent: true }) +// You should not use `@Index({ concurrent: true })` decorator because +// it will break database initialization on test, because it will always run CREATE INDEX in transaction. +@Index(['userId', 'id']) @Entity('note') export class MiNote { @PrimaryColumn(id()) From 6b21c249d44c746dd39ed9bb9b907b74061cf232 Mon Sep 17 00:00:00 2001 From: anatawa12 Date: Thu, 1 May 2025 11:36:01 +0900 Subject: [PATCH 3/9] update comment --- packages/backend/src/models/Note.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/models/Note.ts b/packages/backend/src/models/Note.ts index 4f8a0f9184..d10172072b 100644 --- a/packages/backend/src/models/Note.ts +++ b/packages/backend/src/models/Note.ts @@ -16,8 +16,9 @@ import type { MiDriveFile } from './DriveFile.js'; // Please note that `CREATE INDEX CONCURRENTLY` is not supported in transaction, // so you need to set `transaction = false` in migration. // Please refer 1745378064470-composite-note-index.js for example. -// You should not use `@Index({ concurrent: true })` decorator because -// it will break database initialization on test, because it will always run CREATE INDEX in transaction. +// You should not use `@Index({ concurrent: true })` decorator because database initialization for test will fail +// because it will always run CREATE INDEX in transaction based on decorators. +// Not appending `{ concurrent: true }` to `@Index` will not cause any problem in production, @Index(['userId', 'id']) @Entity('note') export class MiNote { From c37c4fb51b053d462a4b3f623d384c2435cd67b2 Mon Sep 17 00:00:00 2001 From: anatawa12 Date: Thu, 1 May 2025 11:45:15 +0900 Subject: [PATCH 4/9] feat: add MISSKEY_MIGRATION_CREATE_INDEX_CONCURRENTLY option --- .../migration/1745378064470-composite-note-index.js | 10 +++++++--- packages/backend/migration/js/migration-config.js | 4 ++++ packages/backend/ormconfig.js | 3 ++- packages/backend/src/GlobalModule.ts | 9 +++++++-- 4 files changed, 20 insertions(+), 6 deletions(-) create mode 100644 packages/backend/migration/js/migration-config.js diff --git a/packages/backend/migration/1745378064470-composite-note-index.js b/packages/backend/migration/1745378064470-composite-note-index.js index 773afb636c..6bbeb95979 100644 --- a/packages/backend/migration/1745378064470-composite-note-index.js +++ b/packages/backend/migration/1745378064470-composite-note-index.js @@ -3,12 +3,15 @@ * SPDX-License-Identifier: AGPL-3.0-only */ +import { isConcurrentIndexMigrationEnabled } from "./js/migration-config.js"; + export class CompositeNoteIndex1745378064470 { name = 'CompositeNoteIndex1745378064470'; - transaction = false; + transaction = isConcurrentIndexMigrationEnabled() ? false : undefined; async up(queryRunner) { - await queryRunner.query(`CREATE INDEX CONCURRENTLY "IDX_724b311e6f883751f261ebe378" ON "note" ("userId", "id" DESC)`); + const mayConcurrently = isConcurrentIndexMigrationEnabled() ? 'CONCURRENTLY' : ''; + await queryRunner.query(`CREATE INDEX ${mayConcurrently} "IDX_724b311e6f883751f261ebe378" ON "note" ("userId", "id" DESC)`); await queryRunner.query(`DROP INDEX IF EXISTS "IDX_5b87d9d19127bd5d92026017a7"`); // Flush all cached Linear Scan Plans and redo statistics for composite index // this is important for Postgres to learn that even in highly complex queries, using this index first can reduce the result set significantly @@ -16,7 +19,8 @@ export class CompositeNoteIndex1745378064470 { } async down(queryRunner) { + const mayConcurrently = isConcurrentIndexMigrationEnabled() ? 'CONCURRENTLY' : ''; await queryRunner.query(`DROP INDEX IF EXISTS "IDX_724b311e6f883751f261ebe378"`); - await queryRunner.query(`CREATE INDEX CONCURRENTLY "IDX_5b87d9d19127bd5d92026017a7" ON "note" ("userId")`); + await queryRunner.query(`CREATE INDEX ${mayConcurrently} "IDX_5b87d9d19127bd5d92026017a7" ON "note" ("userId")`); } } diff --git a/packages/backend/migration/js/migration-config.js b/packages/backend/migration/js/migration-config.js new file mode 100644 index 0000000000..5a718fa66c --- /dev/null +++ b/packages/backend/migration/js/migration-config.js @@ -0,0 +1,4 @@ + +export function isConcurrentIndexMigrationEnabled() { + return process.env.MISSKEY_MIGRATION_CREATE_INDEX_CONCURRENTLY === '1'; +} diff --git a/packages/backend/ormconfig.js b/packages/backend/ormconfig.js index 23774f21fd..f979c36ad7 100644 --- a/packages/backend/ormconfig.js +++ b/packages/backend/ormconfig.js @@ -1,6 +1,7 @@ import { DataSource } from 'typeorm'; import { loadConfig } from './built/config.js'; import { entities } from './built/postgres.js'; +import { isConcurrentIndexMigrationEnabled } from "./migration/js/migration-config.js"; const config = loadConfig(); @@ -14,5 +15,5 @@ export default new DataSource({ extra: config.db.extra, entities: entities, migrations: ['migration/*.js'], - migrationsTransactionMode: 'each', + migrationsTransactionMode: isConcurrentIndexMigrationEnabled() ? 'each' : 'all', }); diff --git a/packages/backend/src/GlobalModule.ts b/packages/backend/src/GlobalModule.ts index 5544eeeddd..435bd8dd45 100644 --- a/packages/backend/src/GlobalModule.ts +++ b/packages/backend/src/GlobalModule.ts @@ -24,8 +24,13 @@ const $config: Provider = { const $db: Provider = { provide: DI.db, useFactory: async (config) => { - const db = createPostgresDataSource(config); - return await db.initialize(); + try { + const db = createPostgresDataSource(config); + return await db.initialize(); + } catch (e) { + console.log(e); + throw e; + } }, inject: [DI.config], }; From a577b703022b0b0e465cdfa4d8bb77a0ecd26468 Mon Sep 17 00:00:00 2001 From: anatawa12 Date: Thu, 1 May 2025 11:47:05 +0900 Subject: [PATCH 5/9] fix: spdx license header --- packages/backend/migration/js/migration-config.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/backend/migration/js/migration-config.js b/packages/backend/migration/js/migration-config.js index 5a718fa66c..8cfbb21470 100644 --- a/packages/backend/migration/js/migration-config.js +++ b/packages/backend/migration/js/migration-config.js @@ -1,3 +1,7 @@ +/* + * SPDX-FileCopyrightText: syuilo and misskey-project + * SPDX-License-Identifier: AGPL-3.0-only + */ export function isConcurrentIndexMigrationEnabled() { return process.env.MISSKEY_MIGRATION_CREATE_INDEX_CONCURRENTLY === '1'; From ae42805a8ba2d9085a1860b543b47c5cd9ec991d Mon Sep 17 00:00:00 2001 From: anatawa12 Date: Thu, 1 May 2025 11:48:48 +0900 Subject: [PATCH 6/9] alter comment --- packages/backend/src/models/Note.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/backend/src/models/Note.ts b/packages/backend/src/models/Note.ts index d10172072b..840acd3372 100644 --- a/packages/backend/src/models/Note.ts +++ b/packages/backend/src/models/Note.ts @@ -11,10 +11,11 @@ import { MiChannel } from './Channel.js'; import type { MiDriveFile } from './DriveFile.js'; // Note: When you create a new index for existing column of this table, -// it might be better to index concurrently by editing generated migration file -// Since this table is very large, and it takes a long time to create index in most cases. +// it might be better to index concurrently under isConcurrentIndexMigrationEnabled flag +// by editing generated migration file since this table is very large, +// and it will make a long lock to create index in most cases. // Please note that `CREATE INDEX CONCURRENTLY` is not supported in transaction, -// so you need to set `transaction = false` in migration. +// so you need to set `transaction = false` in migration if isConcurrentIndexMigrationEnabled() is true. // Please refer 1745378064470-composite-note-index.js for example. // You should not use `@Index({ concurrent: true })` decorator because database initialization for test will fail // because it will always run CREATE INDEX in transaction based on decorators. From ceb525d4a0c3be1050841ac3ad1690c9b4f179a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=A5=BA=E5=AD=90w=20=28Yumechi=29?= <35571479+eternal-flame-ad@users.noreply.github.com> Date: Thu, 1 May 2025 13:02:18 +0900 Subject: [PATCH 7/9] chore: improve behavior when migration failure --- .../1745378064470-composite-note-index.js | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/packages/backend/migration/1745378064470-composite-note-index.js b/packages/backend/migration/1745378064470-composite-note-index.js index 6bbeb95979..1487aa9630 100644 --- a/packages/backend/migration/1745378064470-composite-note-index.js +++ b/packages/backend/migration/1745378064470-composite-note-index.js @@ -10,8 +10,18 @@ export class CompositeNoteIndex1745378064470 { transaction = isConcurrentIndexMigrationEnabled() ? false : undefined; async up(queryRunner) { - const mayConcurrently = isConcurrentIndexMigrationEnabled() ? 'CONCURRENTLY' : ''; - await queryRunner.query(`CREATE INDEX ${mayConcurrently} "IDX_724b311e6f883751f261ebe378" ON "note" ("userId", "id" DESC)`); + const concurrently = isConcurrentIndexMigrationEnabled(); + + if (concurrently) { + const hasValidIndex = await queryRunner.query(`SELECT indisvalid FROM pg_index INNER JOIN pg_class ON pg_index.indexrelid = pg_class.oid WHERE pg_class.relname = 'IDX_724b311e6f883751f261ebe378'`); + if (!hasValidIndex || hasValidIndex[0].indisvalid !== true) { + await queryRunner.query(`DROP INDEX IF EXISTS "IDX_724b311e6f883751f261ebe378"`); + await queryRunner.query(`CREATE INDEX CONCURRENTLY "IDX_724b311e6f883751f261ebe378" ON "note" ("userId", "id" DESC)`); + } + } else { + await queryRunner.query(`CREATE INDEX IF NOT EXISTS "IDX_724b311e6f883751f261ebe378" ON "note" ("userId", "id" DESC)`); + } + await queryRunner.query(`DROP INDEX IF EXISTS "IDX_5b87d9d19127bd5d92026017a7"`); // Flush all cached Linear Scan Plans and redo statistics for composite index // this is important for Postgres to learn that even in highly complex queries, using this index first can reduce the result set significantly From 111e5a8e609eff2098774b46f8ccf44ab21bb182 Mon Sep 17 00:00:00 2001 From: anatawa12 Date: Fri, 2 May 2025 13:52:44 +0900 Subject: [PATCH 8/9] =?UTF-8?q?docs(changelog):=202025.4.1=20=E3=81=A7?= =?UTF-8?q?=E8=BF=BD=E5=8A=A0=E3=81=95=E3=82=8C=E3=81=9F=E3=82=A4=E3=83=B3?= =?UTF-8?q?=E3=83=87=E3=83=83=E3=82=AF=E3=82=B9=E3=81=AE=E5=86=8D=E7=94=9F?= =?UTF-8?q?=E6=88=90=E3=82=92=E3=83=8E=E3=83=BC=E3=83=88=E3=81=AE=E8=BF=BD?= =?UTF-8?q?=E5=8A=A0=E3=81=97=E3=81=AA=E3=81=8C=E3=82=89=E8=A1=8C=E3=81=88?= =?UTF-8?q?=E3=82=8B=E3=82=88=E3=81=86=E3=81=AB=E3=81=AA=E3=82=8A=E3=81=BE?= =?UTF-8?q?=E3=81=97=E3=81=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index aba43b9e3a..70895522f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,11 @@ ### Server - Enhance: 凍結されたユーザのノートが各種タイムラインで表示されないように `#15775` - Enhance: 連合先のソフトウェア及びバージョン名により配信停止を行えるように `#15727` +- Enhance: 2025.4.1 で追加されたインデックスの再生成をノートの追加しながら行えるようになりました。 `#15915` + - `MISSKEY_MIGRATION_CREATE_INDEX_CONCURRENTLY` 環境変数を `1` にセットしていると、巨大なテーブルの既存のカラムに関するインデックス再生成が`CREATE INDEX CONCURRENTLY`を使用するようになりました。 + - 複数のインスタンスを持つサーバーにおいて、一部のインスタンスが起動している状態でこのオプションを有効にしてマイグレーションすることにより、ダウンタイムを削減することができます。 + - ただし、このオプションを有効にする場合、インデックスの作成にかかる時間が倍~3倍以上になることがあります。 + - また、大きなインスタンスである場合にはインデックスの作成に失敗し、複数回再試行する必要がある可能性があります。 ## 2025.4.1 From 189485ced3a85cbb554a9cf1268e61f5142eec58 Mon Sep 17 00:00:00 2001 From: anatawa12 Date: Fri, 2 May 2025 13:54:38 +0900 Subject: [PATCH 9/9] =?UTF-8?q?=E3=81=A1=E3=82=87=E3=81=A3=E3=81=A8?= =?UTF-8?q?=E8=A1=A8=E7=8F=BE=E3=82=92=E5=A4=89=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 70895522f0..d3956f003f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ - Enhance: 連合先のソフトウェア及びバージョン名により配信停止を行えるように `#15727` - Enhance: 2025.4.1 で追加されたインデックスの再生成をノートの追加しながら行えるようになりました。 `#15915` - `MISSKEY_MIGRATION_CREATE_INDEX_CONCURRENTLY` 環境変数を `1` にセットしていると、巨大なテーブルの既存のカラムに関するインデックス再生成が`CREATE INDEX CONCURRENTLY`を使用するようになりました。 - - 複数のインスタンスを持つサーバーにおいて、一部のインスタンスが起動している状態でこのオプションを有効にしてマイグレーションすることにより、ダウンタイムを削減することができます。 + - 複数のサーバープロセスをクラスタリングしているサーバーにおいて、一部のプロセスが起動している状態でこのオプションを有効にしてマイグレーションすることにより、ダウンタイムを削減することができます。 - ただし、このオプションを有効にする場合、インデックスの作成にかかる時間が倍~3倍以上になることがあります。 - また、大きなインスタンスである場合にはインデックスの作成に失敗し、複数回再試行する必要がある可能性があります。