diff --git a/.editorconfig b/.editorconfig index def7baa1a8..ccf388f06e 100644 --- a/.editorconfig +++ b/.editorconfig @@ -13,3 +13,7 @@ trim_trailing_whitespace = false [*.{yml,yaml}] indent_style = space + +[packages/backend/migration/*.js] +indent_style = space +indent_size = 4 diff --git a/.github/misskey/test.yml b/.github/misskey/test.yml index 3c807e8b9e..513bfb1ac0 100644 --- a/.github/misskey/test.yml +++ b/.github/misskey/test.yml @@ -15,3 +15,5 @@ redis: host: 127.0.0.1 port: 56312 id: aidx + +proxyRemoteFiles: true diff --git a/.github/workflows/test-backend.yml b/.github/workflows/test-backend.yml index 0c40f2c52a..e4b139ef63 100644 --- a/.github/workflows/test-backend.yml +++ b/.github/workflows/test-backend.yml @@ -152,3 +152,47 @@ jobs: with: token: ${{ secrets.CODECOV_TOKEN }} files: ./packages/backend/coverage/coverage-final.json + + migration: + name: Migration tests (backend) + runs-on: ubuntu-latest + strategy: + matrix: + node-version-file: + - .node-version + #- .github/min.node-version + + services: + postgres: + image: postgres:15 + ports: + - 54312:5432 + env: + POSTGRES_DB: test-misskey + POSTGRES_HOST_AUTH_METHOD: trust + + steps: + - uses: actions/checkout@v4.2.2 + with: + submodules: true + - name: Setup pnpm + uses: pnpm/action-setup@v4.1.0 + - name: Get current date + id: current-date + run: echo "today=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT + - name: Use Node.js + uses: actions/setup-node@v4.4.0 + with: + node-version-file: ${{ matrix.node-version-file }} + cache: 'pnpm' + - run: pnpm i --frozen-lockfile + - name: Check pnpm-lock.yaml + run: git diff --exit-code pnpm-lock.yaml + - name: Copy Configure + run: cp .github/misskey/test.yml .config + - name: Build + run: pnpm build + - name: Run migrations + run: MISSKEY_CONFIG_YML=test.yml pnpm --filter backend migrate + - name: Check no migrations are remaining + run: MISSKEY_CONFIG_YML=test.yml pnpm --filter backend check-migrations diff --git a/Dockerfile b/Dockerfile index 77277db8cb..62b737c084 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,6 +18,7 @@ WORKDIR /misskey COPY --link ["pnpm-lock.yaml", "pnpm-workspace.yaml", "package.json", "./"] COPY --link ["scripts", "./scripts"] +COPY --link ["patches", "./patches"] COPY --link ["packages/backend/package.json", "./packages/backend/"] COPY --link ["packages/frontend-shared/package.json", "./packages/frontend-shared/"] COPY --link ["packages/frontend/package.json", "./packages/frontend/"] @@ -53,6 +54,7 @@ WORKDIR /misskey COPY --link ["pnpm-lock.yaml", "pnpm-workspace.yaml", "package.json", "./"] COPY --link ["scripts", "./scripts"] +COPY --link ["patches", "./patches"] COPY --link ["packages/backend/package.json", "./packages/backend/"] COPY --link ["packages/misskey-js/package.json", "./packages/misskey-js/"] COPY --link ["packages/misskey-reversi/package.json", "./packages/misskey-reversi/"] diff --git a/cypress/e2e/basic.cy.ts b/cypress/e2e/basic.cy.ts index 2ce39737ed..4f2f700146 100644 --- a/cypress/e2e/basic.cy.ts +++ b/cypress/e2e/basic.cy.ts @@ -78,6 +78,8 @@ describe('After setup instance', () => { cy.get('[data-cy-signup-password] input').type('alice1234'); cy.get('[data-cy-signup-submit]').should('be.disabled'); cy.get('[data-cy-signup-password-retype] input').type('alice1234'); + cy.get('[data-cy-signup-submit]').should('be.disabled'); + cy.get('[data-cy-signup-invitation-code] input').type('test-invitation-code'); cy.get('[data-cy-signup-submit]').should('not.be.disabled'); cy.get('[data-cy-signup-submit]').click(); diff --git a/package.json b/package.json index 72ccb3be67..b6516ee5e1 100644 --- a/package.json +++ b/package.json @@ -83,6 +83,9 @@ "pnpm": { "overrides": { "@aiscript-dev/aiscript-languageserver": "-" + }, + "patchedDependencies": { + "typeorm": "patches/typeorm.patch" } } } diff --git a/packages/backend/migration/1752509043847-migration-cleanup.js b/packages/backend/migration/1752509043847-migration-cleanup.js new file mode 100644 index 0000000000..450e22af0c --- /dev/null +++ b/packages/backend/migration/1752509043847-migration-cleanup.js @@ -0,0 +1,79 @@ +/* + * SPDX-FileCopyrightText: syuilo and misskey-project + * SPDX-License-Identifier: AGPL-3.0-only + */ + +export class MigrationCleanup1752509043847 { + name = 'MigrationCleanup1752509043847' + + async up(queryRunner) { + // 1745378064470-composite-note-index.js created a index ON "note" ("userId", "id" DESC) as IDX_724b311e6f883751f261ebe378 but should be named IDX_a6f649630f55af3888e5a42919 + await queryRunner.query(`ALTER INDEX "IDX_724b311e6f883751f261ebe378" RENAME TO "IDX_a6f649630f55af3888e5a42919"`); + + // 1713656541000-abuse-report-notification.js generated system_webhook with hand-written SQL with CURRENT_TIMESTAMP as the default value, but its representation in TypeORM is `now()` + // see https://github.com/typeorm/typeorm/blob/f351757a15b9d2bd9d4222c69dcfd2316f46b5d1/src/driver/postgres/PostgresDriver.ts#L1575 + await queryRunner.query(`ALTER TABLE "system_webhook" ALTER COLUMN "updatedAt" SET DEFAULT now()`); + + // 1702718871541-ffVisibility.js defined a enum type "user_profile_followersVisibility_enum" but it should be "user_profile_followersvisibility_enum" (lowercase 'v') in typeorm + await queryRunner.query(`ALTER TYPE "public"."user_profile_followersVisibility_enum" RENAME TO "user_profile_followersvisibility_enum"`); + + // 1713656541000-abuse-report-notification.js generated abuse_report_notification_recipient with hand-written SQL with CURRENT_TIMESTAMP as the default value, but its representation in TypeORM is `now()` + await queryRunner.query(`ALTER TABLE "abuse_report_notification_recipient" ALTER COLUMN "updatedAt" SET DEFAULT now()`); + + // 1690796169261-play-visibility.js added visibility column to flash table but it forgot to set NOT NULL constraint + await queryRunner.query(`ALTER TABLE "flash" ALTER COLUMN "visibility" SET NOT NULL`); + + // 1736686850345-createNoteDraft.js created note_draft with hand-written SQL but several types and comments are not correctly defined + await queryRunner.query(`CREATE TYPE "public"."note_draft_visibility_enum" AS ENUM('public', 'home', 'followers', 'specified')`); + await queryRunner.query(`ALTER TABLE "note_draft" ALTER COLUMN "id" SET DATA TYPE character varying(32)`); + await queryRunner.query(`ALTER TABLE "note_draft" ALTER COLUMN "replyId" SET DATA TYPE character varying(32)`); + await queryRunner.query(`ALTER TABLE "note_draft" ALTER COLUMN "renoteId" SET DATA TYPE character varying(32)`); + await queryRunner.query(`ALTER TABLE "note_draft" ALTER COLUMN "userId" SET DATA TYPE character varying(32)`); + await queryRunner.query(`ALTER TABLE "note_draft" ALTER COLUMN "channelId" SET DATA TYPE character varying(32)`); + await queryRunner.query(`ALTER TABLE "note_draft" ALTER COLUMN "fileIds" SET DATA TYPE character varying(32) array`); + await queryRunner.query(`ALTER TABLE "note_draft" ALTER COLUMN "visibleUserIds" SET DATA TYPE character varying(32) array`); + await queryRunner.query(`ALTER TABLE "note_draft" ALTER COLUMN "visibility" SET DATA TYPE "public"."note_draft_visibility_enum" USING visibility::note_draft_visibility_enum`); + await queryRunner.query(`COMMENT ON COLUMN "note_draft"."replyId" IS 'The ID of reply target.'`); + await queryRunner.query(`COMMENT ON COLUMN "note_draft"."renoteId" IS 'The ID of renote target.'`); + await queryRunner.query(`COMMENT ON COLUMN "note_draft"."userId" IS 'The ID of author.'`); + await queryRunner.query(`COMMENT ON COLUMN "note_draft"."channelId" IS 'The ID of source channel.'`); + await queryRunner.query(`ALTER TABLE "note_draft" ALTER COLUMN "fileIds" SET NOT NULL`); + await queryRunner.query(`ALTER TABLE "note_draft" ALTER COLUMN "hasPoll" SET NOT NULL`); + await queryRunner.query(`ALTER TABLE "note_draft" ALTER COLUMN "pollChoices" SET NOT NULL`); + await queryRunner.query(`ALTER TABLE "note_draft" ALTER COLUMN "pollMultiple" SET NOT NULL`); + await queryRunner.query(`ALTER TABLE "note_draft" ALTER COLUMN "localOnly" SET NOT NULL`); + await queryRunner.query(`ALTER TABLE "note_draft" ALTER COLUMN "visibleUserIds" SET NOT NULL`); + } + + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "note_draft" ALTER COLUMN "visibleUserIds" DROP NOT NULL`); + await queryRunner.query(`ALTER TABLE "note_draft" ALTER COLUMN "localOnly" DROP NOT NULL`); + await queryRunner.query(`ALTER TABLE "note_draft" ALTER COLUMN "pollMultiple" DROP NOT NULL`); + await queryRunner.query(`ALTER TABLE "note_draft" ALTER COLUMN "pollChoices" DROP NOT NULL`); + await queryRunner.query(`ALTER TABLE "note_draft" ALTER COLUMN "hasPoll" DROP NOT NULL`); + await queryRunner.query(`ALTER TABLE "note_draft" ALTER COLUMN "fileIds" DROP NOT NULL`); + await queryRunner.query(`COMMENT ON COLUMN "note_draft"."channelId" IS NULL`); + await queryRunner.query(`COMMENT ON COLUMN "note_draft"."userId" IS 'The ID of author.'`); + await queryRunner.query(`COMMENT ON COLUMN "note_draft"."renoteId" IS NULL`); + await queryRunner.query(`COMMENT ON COLUMN "note_draft"."replyId" IS NULL`); + await queryRunner.query(`ALTER TABLE "note_draft" ALTER COLUMN "visibility" SET DATA TYPE varchar`); + await queryRunner.query(`ALTER TABLE "note_draft" ALTER COLUMN "visibleUserIds" SET DATA TYPE varchar[]`); + await queryRunner.query(`ALTER TABLE "note_draft" ALTER COLUMN "fileIds" SET DATA TYPE varchar[]`); + await queryRunner.query(`ALTER TABLE "note_draft" ALTER COLUMN "channelId" SET DATA TYPE varchar`); + await queryRunner.query(`ALTER TABLE "note_draft" ALTER COLUMN "userId" SET DATA TYPE varchar`); + await queryRunner.query(`ALTER TABLE "note_draft" ALTER COLUMN "renoteId" SET DATA TYPE varchar`); + await queryRunner.query(`ALTER TABLE "note_draft" ALTER COLUMN "replyId" SET DATA TYPE varchar`); + await queryRunner.query(`ALTER TABLE "note_draft" ALTER COLUMN "id" SET DATA TYPE varchar`); + await queryRunner.query(`DROP TYPE "public"."note_draft_visibility_enum"`); + + await queryRunner.query(`ALTER TABLE "flash" ALTER COLUMN "visibility" DROP NOT NULL`); + + await queryRunner.query(`ALTER TABLE "abuse_report_notification_recipient" ALTER COLUMN "updatedAt" SET DEFAULT CURRENT_TIMESTAMP`); + + await queryRunner.query(`ALTER TYPE "public"."user_profile_followersvisibility_enum" RENAME TO "user_profile_followersVisibility_enum"`); + + await queryRunner.query(`ALTER TABLE "system_webhook" ALTER COLUMN "updatedAt" SET DEFAULT CURRENT_TIMESTAMP`); + + await queryRunner.query(`ALTER INDEX "IDX_a6f649630f55af3888e5a42919" RENAME TO "IDX_724b311e6f883751f261ebe378"`); + } +} diff --git a/packages/backend/package.json b/packages/backend/package.json index 2173ce71a5..99482e71bc 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -33,6 +33,7 @@ "test:fed": "pnpm jest:fed", "test-and-coverage": "pnpm jest-and-coverage", "test-and-coverage:e2e": "pnpm build && pnpm build:test && pnpm jest-and-coverage:e2e", + "check-migrations": "node scripts/check_migrations_clean.js", "generate-api-json": "node ./scripts/generate_api_json.js" }, "optionalDependencies": { diff --git a/packages/backend/scripts/check_migrations_clean.js b/packages/backend/scripts/check_migrations_clean.js new file mode 100644 index 0000000000..ce67b1cd81 --- /dev/null +++ b/packages/backend/scripts/check_migrations_clean.js @@ -0,0 +1,26 @@ +/* + * SPDX-FileCopyrightText: syuilo and misskey-project + * SPDX-License-Identifier: AGPL-3.0-only + */ + +// This script checks if the database migrations has been generated correctly. + +import dataSource from '../ormconfig.js'; + +await dataSource.initialize(); + +const sqlInMemory = await dataSource.driver.createSchemaBuilder().log(); + +if (sqlInMemory.upQueries.length > 0 || sqlInMemory.downQueries.length > 0) { + console.error('There are several pending migrations. Please make sure you have generated the migrations correctly, or configured entities class correctly.'); + for (const query of sqlInMemory.upQueries) { + console.error(`- ${query.query}`); + } + for (const query of sqlInMemory.downQueries) { + console.error(`- ${query.query}`); + } + process.exit(1); +} else { + console.log('All migrations are clean.'); + process.exit(0); +} diff --git a/packages/backend/src/models/AbuseReportNotificationRecipient.ts b/packages/backend/src/models/AbuseReportNotificationRecipient.ts index fbff880afc..17ec6abed5 100644 --- a/packages/backend/src/models/AbuseReportNotificationRecipient.ts +++ b/packages/backend/src/models/AbuseReportNotificationRecipient.ts @@ -22,7 +22,7 @@ export class MiAbuseReportNotificationRecipient { /** * 有効かどうか. */ - @Index() + @Index('IDX_abuse_report_notification_recipient_isActive') @Column('boolean', { default: true, }) @@ -47,7 +47,7 @@ export class MiAbuseReportNotificationRecipient { /** * 通知方法. */ - @Index() + @Index('IDX_abuse_report_notification_recipient_method') @Column('varchar', { length: 64, }) @@ -56,10 +56,11 @@ export class MiAbuseReportNotificationRecipient { /** * 通知先のユーザID. */ - @Index() + @Index('IDX_abuse_report_notification_recipient_userId') @Column({ ...id(), nullable: true, + default: null, }) public userId: MiUser['id'] | null; @@ -75,17 +76,20 @@ export class MiAbuseReportNotificationRecipient { /** * 通知先のユーザプロフィール. */ - @ManyToOne(type => MiUserProfile, {}) + @ManyToOne(type => MiUserProfile, { + onDelete: 'CASCADE', + }) @JoinColumn({ name: 'userId', referencedColumnName: 'userId', foreignKeyConstraintName: 'FK_abuse_report_notification_recipient_userId2' }) public userProfile: MiUserProfile | null; /** * 通知先のシステムWebhookId. */ - @Index() + @Index('IDX_abuse_report_notification_recipient_systemWebhookId') @Column({ ...id(), nullable: true, + default: null, }) public systemWebhookId: string | null; @@ -95,6 +99,6 @@ export class MiAbuseReportNotificationRecipient { @ManyToOne(type => MiSystemWebhook, { onDelete: 'CASCADE', }) - @JoinColumn() + @JoinColumn({ name: 'systemWebhookId', referencedColumnName: 'id', foreignKeyConstraintName: 'FK_abuse_report_notification_recipient_systemWebhookId' }) public systemWebhook: MiSystemWebhook | null; } diff --git a/packages/backend/src/models/Emoji.ts b/packages/backend/src/models/Emoji.ts index d62b6e9f6f..8dff8fd153 100644 --- a/packages/backend/src/models/Emoji.ts +++ b/packages/backend/src/models/Emoji.ts @@ -8,6 +8,7 @@ import { id } from './util/id.js'; @Entity('emoji') @Index(['name', 'host'], { unique: true }) +@Index('IDX_EMOJI_ROLE_IDS', { synchronize: false }) // GIN for roleIdsThatCanBeUsedThisEmojiAsReaction in production export class MiEmoji { @PrimaryColumn(id()) public id: string; @@ -32,6 +33,7 @@ export class MiEmoji { @Column('varchar', { length: 128, nullable: true, }) + @Index('IDX_EMOJI_CATEGORY') public category: string | null; @Column('varchar', { diff --git a/packages/backend/src/models/Meta.ts b/packages/backend/src/models/Meta.ts index 3ee6190d45..85c10ab666 100644 --- a/packages/backend/src/models/Meta.ts +++ b/packages/backend/src/models/Meta.ts @@ -59,7 +59,7 @@ export class MiMeta { public maintainerEmail: string | null; @Column('boolean', { - default: false, + default: true, }) public disableRegistration: boolean; @@ -570,7 +570,7 @@ export class MiMeta { public bannedEmailDomains: string[]; @Column('varchar', { - length: 1024, array: true, default: '{ "admin", "administrator", "root", "system", "maintainer", "host", "mod", "moderator", "owner", "superuser", "staff", "auth", "i", "me", "everyone", "all", "mention", "mentions", "example", "user", "users", "account", "accounts", "official", "help", "helps", "support", "supports", "info", "information", "informations", "announce", "announces", "announcement", "announcements", "notice", "notification", "notifications", "dev", "developer", "developers", "tech", "misskey" }', + length: 1024, array: true, default: ['admin', 'administrator', 'root', 'system', 'maintainer', 'host', 'mod', 'moderator', 'owner', 'superuser', 'staff', 'auth', 'i', 'me', 'everyone', 'all', 'mention', 'mentions', 'example', 'user', 'users', 'account', 'accounts', 'official', 'help', 'helps', 'support', 'supports', 'info', 'information', 'informations', 'announce', 'announces', 'announcement', 'announcements', 'notice', 'notification', 'notifications', 'dev', 'developer', 'developers', 'tech', 'misskey'], }) public preservedUsernames: string[]; @@ -635,7 +635,7 @@ export class MiMeta { public urlPreviewMaximumContentLength: number; @Column('boolean', { - default: true, + default: false, }) public urlPreviewRequireContentLength: boolean; @@ -648,6 +648,7 @@ export class MiMeta { @Column('varchar', { length: 1024, nullable: true, + default: null, }) public urlPreviewUserAgent: string | null; diff --git a/packages/backend/src/models/Note.ts b/packages/backend/src/models/Note.ts index 0560ee17c0..9822ec94e4 100644 --- a/packages/backend/src/models/Note.ts +++ b/packages/backend/src/models/Note.ts @@ -20,7 +20,8 @@ import type { MiDriveFile } from './DriveFile.js'; // You should not use `@Index({ concurrent: true })` decorator because database initialization for test will fail // because it will always run CREATE INDEX in transaction based on decorators. // Not appending `{ concurrent: true }` to `@Index` will not cause any problem in production, -@Index(['userId', 'id']) + +@Index(['userId', 'id']) // Note: this index is ("userId", "id" DESC) in production, but not in test. @Entity('note') export class MiNote { @PrimaryColumn(id()) diff --git a/packages/backend/src/models/NoteDraft.ts b/packages/backend/src/models/NoteDraft.ts index 39d85e53dc..6483748bc2 100644 --- a/packages/backend/src/models/NoteDraft.ts +++ b/packages/backend/src/models/NoteDraft.ts @@ -12,11 +12,13 @@ import { MiNote } from './Note.js'; import type { MiDriveFile } from './DriveFile.js'; @Entity('note_draft') +@Index('IDX_NOTE_DRAFT_FILE_IDS', { synchronize: false }) // GIN for fileIds in production +@Index('IDX_NOTE_DRAFT_VISIBLE_USER_IDS', { synchronize: false }) // GIN for visibleUserIds in production export class MiNoteDraft { @PrimaryColumn(id()) public id: string; - @Index() + @Index('IDX_NOTE_DRAFT_REPLY_ID') @Column({ ...id(), nullable: true, @@ -31,7 +33,7 @@ export class MiNoteDraft { @JoinColumn() public reply: MiNote | null; - @Index() + @Index('IDX_NOTE_DRAFT_RENOTE_ID') @Column({ ...id(), nullable: true, @@ -57,7 +59,7 @@ export class MiNoteDraft { }) public cw: string | null; - @Index() + @Index('IDX_NOTE_DRAFT_USER_ID') @Column({ ...id(), comment: 'The ID of author.', @@ -108,7 +110,7 @@ export class MiNoteDraft { }) public hashtag: string | null; - @Index() + @Index('IDX_NOTE_DRAFT_CHANNEL_ID') @Column({ ...id(), nullable: true, diff --git a/packages/backend/src/models/UserProfile.ts b/packages/backend/src/models/UserProfile.ts index c4c1fa5ec9..501b539210 100644 --- a/packages/backend/src/models/UserProfile.ts +++ b/packages/backend/src/models/UserProfile.ts @@ -29,7 +29,7 @@ export class MiUserProfile { }) public location: string | null; - @Index() + // Note: There's index named IDX_de22cd2b445eee31ae51cdbe99 for SUBSTR("birthday", 6, 5) @Column('char', { length: 10, nullable: true, comment: 'The birthday (YYYY-MM-DD) of the User.', diff --git a/packages/backend/src/server/api/SignupApiService.ts b/packages/backend/src/server/api/SignupApiService.ts index 3ec5e5d3e6..53336a087d 100644 --- a/packages/backend/src/server/api/SignupApiService.ts +++ b/packages/backend/src/server/api/SignupApiService.ts @@ -129,7 +129,8 @@ export class SignupApiService { let ticket: MiRegistrationTicket | null = null; - if (this.meta.disableRegistration) { + // テスト時はこの機構は障害となるため無効にする + if (process.env.NODE_ENV !== 'test' && this.meta.disableRegistration) { if (invitationCode == null || typeof invitationCode !== 'string') { reply.code(400); return; diff --git a/packages/backend/test-federation/compose.tpl.yml b/packages/backend/test-federation/compose.tpl.yml index e4483acd7a..3d2ed21337 100644 --- a/packages/backend/test-federation/compose.tpl.yml +++ b/packages/backend/test-federation/compose.tpl.yml @@ -74,6 +74,10 @@ services: source: ../../../pnpm-workspace.yaml target: /misskey/pnpm-workspace.yaml read_only: true + - type: bind + source: ../../../patches + target: /misskey/patches + read_only: true - type: bind source: ./certificates/rootCA.crt target: /usr/local/share/ca-certificates/rootCA.crt diff --git a/packages/backend/test-federation/compose.yml b/packages/backend/test-federation/compose.yml index bd0ac15a31..330cc33854 100644 --- a/packages/backend/test-federation/compose.yml +++ b/packages/backend/test-federation/compose.yml @@ -74,6 +74,10 @@ services: source: ../../../pnpm-workspace.yaml target: /misskey/pnpm-workspace.yaml read_only: true + - type: bind + source: ../../../patches + target: /misskey/patches + read_only: true - type: bind source: ./certificates/rootCA.crt target: /usr/local/share/ca-certificates/rootCA.crt @@ -118,6 +122,10 @@ services: source: ../../../pnpm-workspace.yaml target: /misskey/pnpm-workspace.yaml read_only: true + - type: bind + source: ../../../patches + target: /misskey/patches + read_only: true working_dir: /misskey command: > bash -c " diff --git a/packages/frontend/src/components/MkSignupDialog.form.vue b/packages/frontend/src/components/MkSignupDialog.form.vue index aebec7a8f6..0f8713d4af 100644 --- a/packages/frontend/src/components/MkSignupDialog.form.vue +++ b/packages/frontend/src/components/MkSignupDialog.form.vue @@ -10,7 +10,7 @@ SPDX-License-Identifier: AGPL-3.0-only