From 71edc65d0d90c2152c91277ee4d45af2c27423a5 Mon Sep 17 00:00:00 2001 From: syuilo Date: Sat, 7 Oct 2023 12:05:17 +0900 Subject: [PATCH 01/13] enhance(backend): improve hashtags/trend performance --- CHANGELOG.md | 1 + packages/backend/src/core/FeaturedService.ts | 11 ++ packages/backend/src/core/HashtagService.ts | 101 +++++++++++++++- .../server/api/endpoints/hashtags/trend.ts | 110 ++---------------- 4 files changed, 121 insertions(+), 102 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f9ccaa9f1e..c4da82e0a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,7 @@ ### Server - Enhance: タイムライン取得時のパフォーマンスを大幅に向上 - Enhance: ハイライト取得時のパフォーマンスを大幅に向上 +- Enhance: トレンドハッシュタグ取得時のパフォーマンスを大幅に向上 - Enhance: 不要なPostgreSQLのインデックスを削除しパフォーマンスを向上 ## 2023.9.3 diff --git a/packages/backend/src/core/FeaturedService.ts b/packages/backend/src/core/FeaturedService.ts index 945c23b0e2..62b50ed38d 100644 --- a/packages/backend/src/core/FeaturedService.ts +++ b/packages/backend/src/core/FeaturedService.ts @@ -11,6 +11,7 @@ import { bindThis } from '@/decorators.js'; const GLOBAL_NOTES_RANKING_WINDOW = 1000 * 60 * 60 * 24 * 3; // 3日ごと const PER_USER_NOTES_RANKING_WINDOW = 1000 * 60 * 60 * 24 * 7; // 1週間ごと +const HASHTAG_RANKING_WINDOW = 1000 * 60 * 60; // 1時間ごと @Injectable() export class FeaturedService { @@ -88,6 +89,11 @@ export class FeaturedService { return this.updateRankingOf(`featuredPerUserNotesRanking:${userId}`, PER_USER_NOTES_RANKING_WINDOW, noteId, score); } + @bindThis + public updateHashtagsRanking(hashtag: string, score = 1): Promise { + return this.updateRankingOf('featuredHashtagsRanking', HASHTAG_RANKING_WINDOW, hashtag, score); + } + @bindThis public getGlobalNotesRanking(limit: number): Promise { return this.getRankingOf('featuredGlobalNotesRanking', GLOBAL_NOTES_RANKING_WINDOW, limit); @@ -102,4 +108,9 @@ export class FeaturedService { public getPerUserNotesRanking(userId: MiUser['id'], limit: number): Promise { return this.getRankingOf(`featuredPerUserNotesRanking:${userId}`, PER_USER_NOTES_RANKING_WINDOW, limit); } + + @bindThis + public getHashtagsRanking(limit: number): Promise { + return this.getRankingOf('featuredHashtagsRanking', HASHTAG_RANKING_WINDOW, limit); + } } diff --git a/packages/backend/src/core/HashtagService.ts b/packages/backend/src/core/HashtagService.ts index c72c7460ff..4900fa90a1 100644 --- a/packages/backend/src/core/HashtagService.ts +++ b/packages/backend/src/core/HashtagService.ts @@ -4,6 +4,8 @@ */ import { Inject, Injectable } from '@nestjs/common'; +import * as Redis from 'ioredis'; +import { getLineAndCharacterOfPosition } from 'typescript'; import { DI } from '@/di-symbols.js'; import type { MiUser } from '@/models/User.js'; import { normalizeForSearch } from '@/misc/normalize-for-search.js'; @@ -12,14 +14,19 @@ import type { MiHashtag } from '@/models/Hashtag.js'; import type { HashtagsRepository } from '@/models/_.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { bindThis } from '@/decorators.js'; +import { FeaturedService } from '@/core/FeaturedService.js'; @Injectable() export class HashtagService { constructor( + @Inject(DI.redis) + private redisClient: Redis.Redis, // TODO: 専用のRedisサーバーを設定できるようにする + @Inject(DI.hashtagsRepository) private hashtagsRepository: HashtagsRepository, private userEntityService: UserEntityService, + private featuredService: FeaturedService, private idService: IdService, ) { } @@ -46,6 +53,9 @@ export class HashtagService { public async updateHashtag(user: { id: MiUser['id']; host: MiUser['host']; }, tag: string, isUserAttached = false, inc = true) { tag = normalizeForSearch(tag); + // TODO: サンプリング + this.updateHashtagsRanking(tag, user.id); + const index = await this.hashtagsRepository.findOneBy({ name: tag }); if (index == null && !inc) return; @@ -85,7 +95,7 @@ export class HashtagService { } } } else { - // 自分が初めてこのタグを使ったなら + // 自分が初めてこのタグを使ったなら if (!index.mentionedUserIds.some(id => id === user.id)) { set.mentionedUserIds = () => `array_append("mentionedUserIds", '${user.id}')`; set.mentionedUsersCount = () => '"mentionedUsersCount" + 1'; @@ -144,4 +154,93 @@ export class HashtagService { } } } + + @bindThis + public async updateHashtagsRanking(hashtag: string, userId: MiUser['id']): Promise { + // TODO: instance.hiddenTagsの考慮 + + // YYYYMMDDHHmm (10分間隔) + const now = new Date(); + now.setMinutes(Math.floor(now.getMinutes() / 10) * 10, 0, 0); + const window = `${now.getUTCFullYear()}${(now.getUTCMonth() + 1).toString().padStart(2, '0')}${now.getUTCDate().toString().padStart(2, '0')}${now.getUTCHours().toString().padStart(2, '0')}${now.getUTCMinutes().toString().padStart(2, '0')}`; + + const exist = await this.redisClient.sismember(`hashtagUsers:${hashtag}`, userId); + if (exist === 1) return; + + this.featuredService.updateHashtagsRanking(hashtag, 1); + + const redisPipeline = this.redisClient.pipeline(); + + // TODO: これらの Set は Bloom Filter を使うようにしても良さそう + + // チャート用 + redisPipeline.sadd(`hashtagUsers:${hashtag}:${window}`, userId); + redisPipeline.expire(`hashtagUsers:${hashtag}:${window}`, + 60 * 60 * 24 * 3, // 3日間 + 'NX', // "NX -- Set expiry only when the key has no expiry" = 有効期限がないときだけ設定 + ); + + // ユニークカウント用 + redisPipeline.sadd(`hashtagUsers:${hashtag}`, userId); + redisPipeline.expire(`hashtagUsers:${hashtag}`, + 60 * 60, // 1時間 + 'NX', // "NX -- Set expiry only when the key has no expiry" = 有効期限がないときだけ設定 + ); + + redisPipeline.exec(); + } + + @bindThis + public async getChart(hashtag: string, range: number): Promise { + const now = new Date(); + now.setMinutes(Math.floor(now.getMinutes() / 10) * 10, 0, 0); + + const redisPipeline = this.redisClient.pipeline(); + + for (let i = 0; i < range; i++) { + const window = `${now.getUTCFullYear()}${(now.getUTCMonth() + 1).toString().padStart(2, '0')}${now.getUTCDate().toString().padStart(2, '0')}${now.getUTCHours().toString().padStart(2, '0')}${now.getUTCMinutes().toString().padStart(2, '0')}`; + redisPipeline.scard(`hashtagUsers:${hashtag}:${window}`); + now.setMinutes(now.getMinutes() - (i * 10), 0, 0); + } + + const result = await redisPipeline.exec(); + + if (result == null) return []; + + return result.map(x => x[1]) as number[]; + } + + @bindThis + public async getCharts(hashtags: string[], range: number): Promise> { + const now = new Date(); + now.setMinutes(Math.floor(now.getMinutes() / 10) * 10, 0, 0); + + const redisPipeline = this.redisClient.pipeline(); + + for (let i = 0; i < range; i++) { + const window = `${now.getUTCFullYear()}${(now.getUTCMonth() + 1).toString().padStart(2, '0')}${now.getUTCDate().toString().padStart(2, '0')}${now.getUTCHours().toString().padStart(2, '0')}${now.getUTCMinutes().toString().padStart(2, '0')}`; + for (const hashtag of hashtags) { + redisPipeline.scard(`hashtagUsers:${hashtag}:${window}`); + } + now.setMinutes(now.getMinutes() - (i * 10), 0, 0); + } + + const result = await redisPipeline.exec(); + + if (result == null) return {}; + + // key is hashtag + const charts = {} as Record; + for (const hashtag of hashtags) { + charts[hashtag] = []; + } + + for (let i = 0; i < range; i++) { + for (let j = 0; j < hashtags.length; j++) { + charts[hashtags[j]].push(result[(i * hashtags.length) + j][1] as number); + } + } + + return charts; + } } diff --git a/packages/backend/src/server/api/endpoints/hashtags/trend.ts b/packages/backend/src/server/api/endpoints/hashtags/trend.ts index 75d4fe3819..a69e007a40 100644 --- a/packages/backend/src/server/api/endpoints/hashtags/trend.ts +++ b/packages/backend/src/server/api/endpoints/hashtags/trend.ts @@ -3,29 +3,13 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import { Brackets } from 'typeorm'; import { Inject, Injectable } from '@nestjs/common'; import { Endpoint } from '@/server/api/endpoint-base.js'; -import type { NotesRepository } from '@/models/_.js'; -import type { MiNote } from '@/models/Note.js'; -import { safeForSql } from '@/misc/safe-for-sql.js'; import { normalizeForSearch } from '@/misc/normalize-for-search.js'; import { MetaService } from '@/core/MetaService.js'; import { DI } from '@/di-symbols.js'; - -/* -トレンドに載るためには「『直近a分間のユニーク投稿数が今からa分前～今からb分前の間のユニーク投稿数のn倍以上』のハッシュタグの上位5位以内に入る」ことが必要 -ユニーク投稿数とはそのハッシュタグと投稿ユーザーのペアのカウントで、例えば同じユーザーが複数回同じハッシュタグを投稿してもそのハッシュタグのユニーク投稿数は1とカウントされる - -..が理想だけどPostgreSQLでどうするのか分からないので単に「直近Aの内に投稿されたユニーク投稿数が多いハッシュタグ」で妥協する -*/ - -const rangeA = 1000 * 60 * 60; // 60分 -//const rangeB = 1000 * 60 * 120; // 2時間 -//const coefficient = 1.25; // 「n倍」の部分 -//const requiredUsers = 3; // 最低何人がそのタグを投稿している必要があるか - -const max = 5; +import { FeaturedService } from '@/core/FeaturedService.js'; +import { HashtagService } from '@/core/HashtagService.js'; export const meta = { tags: ['hashtags'], @@ -71,98 +55,22 @@ export const paramDef = { @Injectable() export default class extends Endpoint { // eslint-disable-line import/no-default-export constructor( - @Inject(DI.notesRepository) - private notesRepository: NotesRepository, - private metaService: MetaService, + private featuredService: FeaturedService, + private hashtagService: HashtagService, ) { super(meta, paramDef, async () => { const instance = await this.metaService.fetch(true); const hiddenTags = instance.hiddenTags.map(t => normalizeForSearch(t)); - const now = new Date(); // 5分単位で丸めた現在日時 - now.setMinutes(Math.round(now.getMinutes() / 5) * 5, 0, 0); + const ranking = await this.featuredService.getHashtagsRanking(10); - const tagNotes = await this.notesRepository.createQueryBuilder('note') - .where('note.createdAt > :date', { date: new Date(now.getTime() - rangeA) }) - .andWhere(new Brackets(qb => { qb - .where('note.visibility = \'public\'') - .orWhere('note.visibility = \'home\''); - })) - .andWhere('note.tags != \'{}\'') - .select(['note.tags', 'note.userId']) - .cache(60000) // 1 min - .getMany(); + const charts = ranking.length === 0 ? {} : await this.hashtagService.getCharts(ranking, 20); - if (tagNotes.length === 0) { - return []; - } - - const tags: { - name: string; - users: MiNote['userId'][]; - }[] = []; - - for (const note of tagNotes) { - for (const tag of note.tags) { - if (hiddenTags.includes(tag)) continue; - - const x = tags.find(x => x.name === tag); - if (x) { - if (!x.users.includes(note.userId)) { - x.users.push(note.userId); - } - } else { - tags.push({ - name: tag, - users: [note.userId], - }); - } - } - } - - // タグを人気順に並べ替え - const hots = tags - .sort((a, b) => b.users.length - a.users.length) - .map(tag => tag.name) - .slice(0, max); - - //#region 2(または3)で話題と判定されたタグそれぞれについて過去の投稿数グラフを取得する - const countPromises: Promise[] = []; - - const range = 20; - - // 10分 - const interval = 1000 * 60 * 10; - - for (let i = 0; i < range; i++) { - countPromises.push(Promise.all(hots.map(tag => this.notesRepository.createQueryBuilder('note') - .select('count(distinct note.userId)') - .where(`'{"${safeForSql(tag) ? tag : 'aichan_kawaii'}"}' <@ note.tags`) - .andWhere('note.createdAt < :lt', { lt: new Date(now.getTime() - (interval * i)) }) - .andWhere('note.createdAt > :gt', { gt: new Date(now.getTime() - (interval * (i + 1))) }) - .cache(60000) // 1 min - .getRawOne() - .then(x => parseInt(x.count, 10)), - ))); - } - - const countsLog = await Promise.all(countPromises); - //#endregion - - const totalCounts = await Promise.all(hots.map(tag => this.notesRepository.createQueryBuilder('note') - .select('count(distinct note.userId)') - .where(`'{"${safeForSql(tag) ? tag : 'aichan_kawaii'}"}' <@ note.tags`) - .andWhere('note.createdAt > :gt', { gt: new Date(now.getTime() - rangeA) }) - .cache(60000 * 60) // 60 min - .getRawOne() - .then(x => parseInt(x.count, 10)), - )); - - const stats = hots.map((tag, i) => ({ + const stats = ranking.map((tag, i) => ({ tag, - chart: countsLog.map(counts => counts[i]), - usersCount: totalCounts[i], + chart: charts[tag], + usersCount: Math.max(...charts[tag]), })); return stats; From c8d7a5ae766ae961eccad7bbb208d9a8ce800f79 Mon Sep 17 00:00:00 2001 From: MeiMei <30769358+mei23@users.noreply.github.com> Date: Sat, 7 Oct 2023 12:19:05 +0900 Subject: [PATCH 02/13] =?UTF-8?q?=E9=80=A3=E5=90=88=E3=81=AA=E3=81=97?= =?UTF-8?q?=E3=82=A2=E3=83=B3=E3=82=B1=E3=83=BC=E3=83=88=E3=81=AEUpdate?= =?UTF-8?q?=E3=81=8C=E3=83=AA=E3=83=A2=E3=83=BC=E3=83=88=E3=81=AB=E9=85=8D?= =?UTF-8?q?=E4=BF=A1=E3=81=95=E3=82=8C=E3=81=A6=E3=81=97=E3=81=BE=E3=81=86?= =?UTF-8?q?=E3=81=AE=E3=82=92=E4=BF=AE=E6=AD=A3=20(#11977)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/core/PollService.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/backend/src/core/PollService.ts b/packages/backend/src/core/PollService.ts index 940aa98347..570f2350f1 100644 --- a/packages/backend/src/core/PollService.ts +++ b/packages/backend/src/core/PollService.ts @@ -96,6 +96,8 @@ export class PollService { const note = await this.notesRepository.findOneBy({ id: noteId }); if (note == null) throw new Error('note not found'); + if (note.localOnly) return; + const user = await this.usersRepository.findOneBy({ id: note.userId }); if (user == null) throw new Error('note not found'); From 93bd34113c3198670772d71548a3258bb3e9f34c Mon Sep 17 00:00:00 2001 From: syuilo Date: Sat, 7 Oct 2023 12:19:26 +0900 Subject: [PATCH 03/13] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c4da82e0a1..0b901f047d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,6 +43,7 @@ - Enhance: ハイライト取得時のパフォーマンスを大幅に向上 - Enhance: トレンドハッシュタグ取得時のパフォーマンスを大幅に向上 - Enhance: 不要なPostgreSQLのインデックスを削除しパフォーマンスを向上 +- Fix: 連合なしアンケートに投票をするとUpdateがリモートに配信されてしまうのを修正 ## 2023.9.3 ### General From d6ef28d4caf347c7c42454a98359e1742e87a550 Mon Sep 17 00:00:00 2001 From: FineArchs <133759614+FineArchs@users.noreply.github.com> Date: Sat, 7 Oct 2023 12:25:16 +0900 Subject: [PATCH 04/13] =?UTF-8?q?=E3=83=90=E3=83=83=E3=82=AF=E3=82=A8?= =?UTF-8?q?=E3=83=B3=E3=83=89=E3=81=AE=E3=83=86=E3=82=B9=E3=83=88=E3=81=AE?= =?UTF-8?q?=E6=94=B9=E5=96=84=20(#11978)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update utils.ts * no async executer --- packages/backend/test/utils.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/packages/backend/test/utils.ts b/packages/backend/test/utils.ts index 121c4711d3..738635dae2 100644 --- a/packages/backend/test/utils.ts +++ b/packages/backend/test/utils.ts @@ -301,12 +301,14 @@ export const uploadFile = async (user?: UserToken, { path, name, blob }: UploadO }; export const uploadUrl = async (user: UserToken, url: string) => { - let file: any; + let resolve: unknown; + const file = new Promise(ok => resolve = ok); const marker = Math.random().toString(); const ws = await connectStream(user, 'main', (msg) => { if (msg.type === 'urlUploadFinished' && msg.body.marker === marker) { - file = msg.body.file; + ws.close(); + resolve(msg.body.file); } }); @@ -316,9 +318,6 @@ export const uploadUrl = async (user: UserToken, url: string) => { force: true, }, user); - await sleep(7000); - ws.close(); - return file; }; From 5e8c0deab3d62c2c2cfd1fda14dc63179f65257d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=81=8B=E3=81=A3=E3=81=93=E3=81=8B=E3=82=8A?= <67428053+kakkokari-gtyih@users.noreply.github.com> Date: Sat, 7 Oct 2023 13:13:13 +0900 Subject: [PATCH 05/13] =?UTF-8?q?=E3=83=97=E3=83=A9=E3=82=A4=E3=83=90?= =?UTF-8?q?=E3=82=B7=E3=83=BC=E3=83=9D=E3=83=AA=E3=82=B7=E3=83=BC=E3=83=BB?= =?UTF-8?q?=E9=81=8B=E5=96=B6=E8=80=85=E6=83=85=E5=A0=B1=E3=81=AE=E3=83=AA?= =?UTF-8?q?=E3=83=B3=E3=82=AF=E3=82=92=E8=BF=BD=E5=8A=A0=20(#11925)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 運営者情報・プライバシーポリシーリンクを追加 * Update Changelog * Run api extractor * プライバシーポリシー・利用規約の同意をまとめる * Update Changelog * fix lint * fix * api extractor * improve design * nodeinfoにプライバシーポリシー・運営者情報を追加 --- CHANGELOG.md | 2 + locales/index.d.ts | 6 +++ locales/ja-JP.yml | 6 +++ .../migration/1696003580220-AddSomeUrls.js | 17 ++++++++ packages/backend/src/models/Meta.ts | 12 ++++++ .../src/server/NodeinfoServerService.ts | 2 + .../src/server/api/endpoints/admin/meta.ts | 2 + .../server/api/endpoints/admin/update-meta.ts | 10 +++++ .../backend/src/server/api/endpoints/meta.ts | 2 + .../src/components/MkSignupDialog.rules.vue | 43 +++++++++++++------ .../src/components/MkVisitorDashboard.vue | 20 ++++++++- packages/frontend/src/pages/about.vue | 18 +++++--- .../frontend/src/pages/admin/moderation.vue | 8 ++++ .../frontend/src/pages/admin/settings.vue | 9 ++++ packages/frontend/src/ui/_common_/common.ts | 20 ++++++++- packages/misskey-js/etc/misskey-js.api.md | 4 +- packages/misskey-js/src/entities.ts | 2 + 17 files changed, 160 insertions(+), 23 deletions(-) create mode 100644 packages/backend/migration/1696003580220-AddSomeUrls.js diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b901f047d..d1f8f7b8d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,8 @@ - Feat: ユーザーごとに他ユーザーへの返信をタイムラインに含めるか設定可能になりました - Feat: ユーザーリスト内のメンバーごとに他ユーザーへの返信をユーザーリストタイムラインに含めるか設定可能になりました - Feat: ユーザーごとのハイライト +- Feat: プライバシーポリシー・運営者情報（Impressum）の指定が可能になりました + - プライバシーポリシーはサーバー登録時に同意確認が入ります - Enhance: ソフトワードミュートとハードワードミュートは統合されました - Enhance: モデレーションログ機能の強化 - Enhance: ローカリゼーションの更新 diff --git a/locales/index.d.ts b/locales/index.d.ts index 12ac0197f0..d90f8fa6f2 100644 --- a/locales/index.d.ts +++ b/locales/index.d.ts @@ -1132,6 +1132,12 @@ export interface Locale { "showRepliesToOthersInTimeline": string; "hideRepliesToOthersInTimeline": string; "externalServices": string; + "impressum": string; + "impressumUrl": string; + "impressumDescription": string; + "privacyPolicy": string; + "privacyPolicyUrl": string; + "tosAndPrivacyPolicy": string; "_announcement": { "forExistingUsers": string; "forExistingUsersDescription": string; diff --git a/locales/ja-JP.yml b/locales/ja-JP.yml index f2190022ce..c92d834366 100644 --- a/locales/ja-JP.yml +++ b/locales/ja-JP.yml @@ -1129,6 +1129,12 @@ fileAttachedOnly: "ファイル付きのみ" showRepliesToOthersInTimeline: "TLに他の人への返信を含める" hideRepliesToOthersInTimeline: "TLに他の人への返信を含めない" externalServices: "外部サービス" +impressum: "運営者情報" +impressumUrl: "運営者情報URL" +impressumDescription: "ドイツなどの一部の国と地域では表示が義務付けられています(Impressum)。" +privacyPolicy: "プライバシーポリシー" +privacyPolicyUrl: "プライバシーポリシーURL" +tosAndPrivacyPolicy: "利用規約・プライバシーポリシー" _announcement: forExistingUsers: "既存ユーザーのみ" diff --git a/packages/backend/migration/1696003580220-AddSomeUrls.js b/packages/backend/migration/1696003580220-AddSomeUrls.js new file mode 100644 index 0000000000..683aa5eeed --- /dev/null +++ b/packages/backend/migration/1696003580220-AddSomeUrls.js @@ -0,0 +1,17 @@ +/* + * SPDX-FileCopyrightText: syuilo and other misskey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +export class AddSomeUrls1696003580220 { + name = 'AddSomeUrls1696003580220' + + async up(queryRunner) { + await queryRunner.query(`ALTER TABLE "meta" ADD "impressumUrl" character varying(1024)`); + await queryRunner.query(`ALTER TABLE "meta" ADD "privacyPolicyUrl" character varying(1024)`); + } + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "impressumUrl"`); + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "privacyPolicyUrl"`); + } +} diff --git a/packages/backend/src/models/Meta.ts b/packages/backend/src/models/Meta.ts index 491d446723..60fdaab54a 100644 --- a/packages/backend/src/models/Meta.ts +++ b/packages/backend/src/models/Meta.ts @@ -335,6 +335,18 @@ export class MiMeta { }) public feedbackUrl: string | null; + @Column('varchar', { + length: 1024, + nullable: true, + }) + public impressumUrl: string | null; + + @Column('varchar', { + length: 1024, + nullable: true, + }) + public privacyPolicyUrl: string | null; + @Column('varchar', { length: 8192, nullable: true, diff --git a/packages/backend/src/server/NodeinfoServerService.ts b/packages/backend/src/server/NodeinfoServerService.ts index 79f130dabe..dd2b7882a2 100644 --- a/packages/backend/src/server/NodeinfoServerService.ts +++ b/packages/backend/src/server/NodeinfoServerService.ts @@ -102,6 +102,8 @@ export class NodeinfoServerService { }, langs: meta.langs, tosUrl: meta.termsOfServiceUrl, + privacyPolicyUrl: meta.privacyPolicyUrl, + impressumUrl: meta.impressumUrl, repositoryUrl: meta.repositoryUrl, feedbackUrl: meta.feedbackUrl, disableRegistration: meta.disableRegistration, diff --git a/packages/backend/src/server/api/endpoints/admin/meta.ts b/packages/backend/src/server/api/endpoints/admin/meta.ts index 53e3672784..ddca5b4709 100644 --- a/packages/backend/src/server/api/endpoints/admin/meta.ts +++ b/packages/backend/src/server/api/endpoints/admin/meta.ts @@ -331,6 +331,8 @@ export default class extends Endpoint { // eslint- tosUrl: instance.termsOfServiceUrl, repositoryUrl: instance.repositoryUrl, feedbackUrl: instance.feedbackUrl, + impressumUrl: instance.impressumUrl, + privacyPolicyUrl: instance.privacyPolicyUrl, disableRegistration: instance.disableRegistration, emailRequiredForSignup: instance.emailRequiredForSignup, enableHcaptcha: instance.enableHcaptcha, diff --git a/packages/backend/src/server/api/endpoints/admin/update-meta.ts b/packages/backend/src/server/api/endpoints/admin/update-meta.ts index 247d3ba4e0..ee7c564e23 100644 --- a/packages/backend/src/server/api/endpoints/admin/update-meta.ts +++ b/packages/backend/src/server/api/endpoints/admin/update-meta.ts @@ -86,6 +86,8 @@ export const paramDef = { tosUrl: { type: 'string', nullable: true }, repositoryUrl: { type: 'string' }, feedbackUrl: { type: 'string' }, + impressumUrl: { type: 'string' }, + privacyPolicyUrl: { type: 'string' }, useObjectStorage: { type: 'boolean' }, objectStorageBaseUrl: { type: 'string', nullable: true }, objectStorageBucket: { type: 'string', nullable: true }, @@ -345,6 +347,14 @@ export default class extends Endpoint { // eslint- set.feedbackUrl = ps.feedbackUrl; } + if (ps.impressumUrl !== undefined) { + set.impressumUrl = ps.impressumUrl; + } + + if (ps.privacyPolicyUrl !== undefined) { + set.privacyPolicyUrl = ps.privacyPolicyUrl; + } + if (ps.useObjectStorage !== undefined) { set.useObjectStorage = ps.useObjectStorage; } diff --git a/packages/backend/src/server/api/endpoints/meta.ts b/packages/backend/src/server/api/endpoints/meta.ts index 271b3f6fb2..e7c8d26827 100644 --- a/packages/backend/src/server/api/endpoints/meta.ts +++ b/packages/backend/src/server/api/endpoints/meta.ts @@ -299,6 +299,8 @@ export default class extends Endpoint { // eslint- tosUrl: instance.termsOfServiceUrl, repositoryUrl: instance.repositoryUrl, feedbackUrl: instance.feedbackUrl, + impressumUrl: instance.impressumUrl, + privacyPolicyUrl: instance.privacyPolicyUrl, disableRegistration: instance.disableRegistration, emailRequiredForSignup: instance.emailRequiredForSignup, enableHcaptcha: instance.enableHcaptcha, diff --git a/packages/frontend/src/components/MkSignupDialog.rules.vue b/packages/frontend/src/components/MkSignupDialog.rules.vue index aa4a184d7b..76163ab68b 100644 --- a/packages/frontend/src/components/MkSignupDialog.rules.vue +++ b/packages/frontend/src/components/MkSignupDialog.rules.vue @@ -30,13 +30,15 @@ SPDX-License-Identifier: AGPL-3.0-only {{ i18n.ts.agree }} - - - + + + +
+ + +
- {{ i18n.ts.termsOfService }} - - {{ i18n.ts.agree }} + {{ i18n.ts.agree }} @@ -70,14 +72,15 @@ import MkInfo from '@/components/MkInfo.vue'; import * as os from '@/os.js'; const availableServerRules = instance.serverRules.length > 0; -const availableTos = instance.tosUrl != null; +const availableTos = instance.tosUrl != null && instance.tosUrl !== ''; +const availablePrivacyPolicy = instance.privacyPolicyUrl != null && instance.privacyPolicyUrl !== ''; const agreeServerRules = ref(false); -const agreeTos = ref(false); +const agreeTosAndPrivacyPolicy = ref(false); const agreeNote = ref(false); const agreed = computed(() => { - return (!availableServerRules || agreeServerRules.value) && (!availableTos || agreeTos.value) && agreeNote.value; + return (!availableServerRules || agreeServerRules.value) && ((!availableTos && !availablePrivacyPolicy) || agreeTosAndPrivacyPolicy.value) && agreeNote.value; }); const emit = defineEmits<{ @@ -85,6 +88,18 @@ const emit = defineEmits<{ (ev: 'done'): void; }>(); +const tosPrivacyPolicyLabel = computed(() => { + if (availableTos && availablePrivacyPolicy) { + return i18n.ts.tosAndPrivacyPolicy; + } else if (availableTos) { + return i18n.ts.termsOfService; + } else if (availablePrivacyPolicy) { + return i18n.ts.privacyPolicy; + } else { + return ""; + } +}); + async function updateAgreeServerRules(v: boolean) { if (v) { const confirm = await os.confirm({ @@ -99,17 +114,19 @@ async function updateAgreeServerRules(v: boolean) { } } -async function updateAgreeTos(v: boolean) { +async function updateAgreeTosAndPrivacyPolicy(v: boolean) { if (v) { const confirm = await os.confirm({ type: 'question', title: i18n.ts.doYouAgree, - text: i18n.t('iHaveReadXCarefullyAndAgree', { x: i18n.ts.termsOfService }), + text: i18n.t('iHaveReadXCarefullyAndAgree', { + x: tosPrivacyPolicyLabel.value, + }), }); if (confirm.canceled) return; - agreeTos.value = true; + agreeTosAndPrivacyPolicy.value = true; } else { - agreeTos.value = false; + agreeTosAndPrivacyPolicy.value = false; } } diff --git a/packages/frontend/src/components/MkVisitorDashboard.vue b/packages/frontend/src/components/MkVisitorDashboard.vue index e4520bbb2d..40493a5d06 100644 --- a/packages/frontend/src/components/MkVisitorDashboard.vue +++ b/packages/frontend/src/components/MkVisitorDashboard.vue @@ -104,7 +104,25 @@ function showMenu(ev) { action: () => { os.pageWindow('/about-misskey'); }, - }, null, { + }, null, (instance.impressumUrl) ? { + text: i18n.ts.impressum, + icon: 'ti ti-file-invoice', + action: () => { + window.open(instance.impressumUrl, '_blank'); + }, + } : undefined, (instance.tosUrl) ? { + text: i18n.ts.termsOfService, + icon: 'ti ti-notebook', + action: () => { + window.open(instance.tosUrl, '_blank'); + }, + } : undefined, (instance.privacyPolicyUrl) ? { + text: i18n.ts.privacyPolicy, + icon: 'ti ti-shield-lock', + action: () => { + window.open(instance.privacyPolicyUrl, '_blank'); + }, + } : undefined, (!instance.impressumUrl && !instance.tosUrl && !instance.privacyPolicyUrl) ? undefined : null, { text: i18n.ts.help, icon: 'ti ti-help-circle', action: () => { diff --git a/packages/frontend/src/pages/about.vue b/packages/frontend/src/pages/about.vue index 02768b0774..ee4043f9a5 100644 --- a/packages/frontend/src/pages/about.vue +++ b/packages/frontend/src/pages/about.vue @@ -46,14 +46,18 @@ SPDX-License-Identifier: AGPL-3.0-only - - + {{ i18n.ts.impressum }} +
+ + -
    -
    2. -
- - {{ i18n.ts.termsOfService }} +
    +
    2. +
+ + {{ i18n.ts.termsOfService }} + {{ i18n.ts.privacyPolicy }} +
diff --git a/packages/frontend/src/pages/admin/moderation.vue b/packages/frontend/src/pages/admin/moderation.vue index 46f92729e8..8b160635f7 100644 --- a/packages/frontend/src/pages/admin/moderation.vue +++ b/packages/frontend/src/pages/admin/moderation.vue @@ -25,6 +25,11 @@ SPDX-License-Identifier: AGPL-3.0-only + + + + + @@ -69,6 +74,7 @@ let emailRequiredForSignup: boolean = $ref(false); let sensitiveWords: string = $ref(''); let preservedUsernames: string = $ref(''); let tosUrl: string | null = $ref(null); +let privacyPolicyUrl: string | null = $ref(null); async function init() { const meta = await os.api('admin/meta'); @@ -77,6 +83,7 @@ async function init() { sensitiveWords = meta.sensitiveWords.join('\n'); preservedUsernames = meta.preservedUsernames.join('\n'); tosUrl = meta.tosUrl; + privacyPolicyUrl = meta.privacyPolicyUrl; } function save() { @@ -84,6 +91,7 @@ function save() { disableRegistration: !enableRegistration, emailRequiredForSignup, tosUrl, + privacyPolicyUrl, sensitiveWords: sensitiveWords.split('\n'), preservedUsernames: preservedUsernames.split('\n'), }).then(() => { diff --git a/packages/frontend/src/pages/admin/settings.vue b/packages/frontend/src/pages/admin/settings.vue index 09a6cc7e2c..2e3f1611dc 100644 --- a/packages/frontend/src/pages/admin/settings.vue +++ b/packages/frontend/src/pages/admin/settings.vue @@ -34,6 +34,12 @@ SPDX-License-Identifier: AGPL-3.0-only + + + + + + @@ -135,6 +141,7 @@ let shortName: string | null = $ref(null); let description: string | null = $ref(null); let maintainerName: string | null = $ref(null); let maintainerEmail: string | null = $ref(null); +let impressumUrl: string | null = $ref(null); let pinnedUsers: string = $ref(''); let cacheRemoteFiles: boolean = $ref(false); let cacheRemoteSensitiveFiles: boolean = $ref(false); @@ -153,6 +160,7 @@ async function init(): Promise { description = meta.description; maintainerName = meta.maintainerName; maintainerEmail = meta.maintainerEmail; + impressumUrl = meta.impressumUrl; pinnedUsers = meta.pinnedUsers.join('\n'); cacheRemoteFiles = meta.cacheRemoteFiles; cacheRemoteSensitiveFiles = meta.cacheRemoteSensitiveFiles; @@ -172,6 +180,7 @@ function save(): void { description, maintainerName, maintainerEmail, + impressumUrl, pinnedUsers: pinnedUsers.split('\n'), cacheRemoteFiles, cacheRemoteSensitiveFiles, diff --git a/packages/frontend/src/ui/_common_/common.ts b/packages/frontend/src/ui/_common_/common.ts index ca4a71a67f..e075e05db3 100644 --- a/packages/frontend/src/ui/_common_/common.ts +++ b/packages/frontend/src/ui/_common_/common.ts @@ -68,7 +68,25 @@ export function openInstanceMenu(ev: MouseEvent) { text: i18n.ts.manageCustomEmojis, icon: 'ti ti-icons', } : undefined], - }, null, { + }, null, (instance.impressumUrl) ? { + text: i18n.ts.impressum, + icon: 'ti ti-file-invoice', + action: () => { + window.open(instance.impressumUrl, '_blank'); + }, + } : undefined, (instance.tosUrl) ? { + text: i18n.ts.termsOfService, + icon: 'ti ti-notebook', + action: () => { + window.open(instance.tosUrl, '_blank'); + }, + } : undefined, (instance.privacyPolicyUrl) ? { + text: i18n.ts.privacyPolicy, + icon: 'ti ti-shield-lock', + action: () => { + window.open(instance.privacyPolicyUrl, '_blank'); + }, + } : undefined, (!instance.impressumUrl && !instance.tosUrl && !instance.privacyPolicyUrl) ? undefined : null, { text: i18n.ts.help, icon: 'ti ti-help-circle', action: () => { diff --git a/packages/misskey-js/etc/misskey-js.api.md b/packages/misskey-js/etc/misskey-js.api.md index fa09fd94e6..fe913886c7 100644 --- a/packages/misskey-js/etc/misskey-js.api.md +++ b/packages/misskey-js/etc/misskey-js.api.md @@ -2408,6 +2408,8 @@ type LiteInstanceMetadata = { tosUrl: string | null; repositoryUrl: string; feedbackUrl: string; + impressumUrl: string | null; + privacyPolicyUrl: string | null; disableRegistration: boolean; disableLocalTimeline: boolean; disableGlobalTimeline: boolean; @@ -2978,7 +2980,7 @@ type UserSorting = '+follower' | '-follower' | '+createdAt' | '-createdAt' | '+u // src/api.types.ts:18:25 - (ae-forgotten-export) The symbol "NoParams" needs to be exported by the entry point index.d.ts // src/api.types.ts:630:18 - (ae-forgotten-export) The symbol "ShowUserReq" needs to be exported by the entry point index.d.ts // src/entities.ts:107:2 - (ae-forgotten-export) The symbol "notificationTypes_2" needs to be exported by the entry point index.d.ts -// src/entities.ts:594:2 - (ae-forgotten-export) The symbol "ModerationLogPayloads" needs to be exported by the entry point index.d.ts +// src/entities.ts:596:2 - (ae-forgotten-export) The symbol "ModerationLogPayloads" needs to be exported by the entry point index.d.ts // src/streaming.types.ts:33:4 - (ae-forgotten-export) The symbol "FIXME" needs to be exported by the entry point index.d.ts // (No @packageDocumentation comment for this package) diff --git a/packages/misskey-js/src/entities.ts b/packages/misskey-js/src/entities.ts index 8097618c33..b60056e21a 100644 --- a/packages/misskey-js/src/entities.ts +++ b/packages/misskey-js/src/entities.ts @@ -322,6 +322,8 @@ export type LiteInstanceMetadata = { tosUrl: string | null; repositoryUrl: string; feedbackUrl: string; + impressumUrl: string | null; + privacyPolicyUrl: string | null; disableRegistration: boolean; disableLocalTimeline: boolean; disableGlobalTimeline: boolean; From 0fe8c0134c32b060e318da14d4ad4d1510b69d29 Mon Sep 17 00:00:00 2001 From: syuilo Date: Sat, 7 Oct 2023 14:02:05 +0900 Subject: [PATCH 06/13] =?UTF-8?q?enhance(backend):=20notes/global-timeline?= =?UTF-8?q?=E5=BE=A9=E6=B4=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 1 - .../api/endpoints/notes/global-timeline.ts | 33 +++++++++++++++++-- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d1f8f7b8d4..7f2b8ac44d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,7 +20,6 @@ ### Changes - API: users/notes, notes/local-timeline で fileType 指定はできなくなりました -- API: notes/global-timeline は現在常に `[]` を返します - API: notes/featured でページネーションは他APIと同様 untilId を使って行うようになりました ### General diff --git a/packages/backend/src/server/api/endpoints/notes/global-timeline.ts b/packages/backend/src/server/api/endpoints/notes/global-timeline.ts index e5a86905d6..be7557c213 100644 --- a/packages/backend/src/server/api/endpoints/notes/global-timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/global-timeline.ts @@ -67,8 +67,37 @@ export default class extends Endpoint { // eslint- throw new ApiError(meta.errors.gtlDisabled); } - // TODO? - return []; + //#region Construct query + const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), + ps.sinceId, ps.untilId, ps.sinceDate, ps.untilDate) + .andWhere('note.visibility = \'public\'') + .andWhere('note.channelId IS NULL') + .innerJoinAndSelect('note.user', 'user') + .leftJoinAndSelect('note.reply', 'reply') + .leftJoinAndSelect('note.renote', 'renote') + .leftJoinAndSelect('reply.user', 'replyUser') + .leftJoinAndSelect('renote.user', 'renoteUser'); + + if (me) { + this.queryService.generateMutedUserQuery(query, me); + this.queryService.generateBlockedUserQuery(query, me); + this.queryService.generateMutedUserRenotesQueryForNotes(query, me); + } + + if (ps.withFiles) { + query.andWhere('note.fileIds != \'{}\''); + } + //#endregion + + const timeline = await query.limit(ps.limit).getMany(); + + process.nextTick(() => { + if (me) { + this.activeUsersChart.read(me); + } + }); + + return await this.noteEntityService.packMany(timeline, me); }); } } From dc435fb8eefdbfc1260f5fd0b8aea51ddaca2b59 Mon Sep 17 00:00:00 2001 From: syuilo Date: Sat, 7 Oct 2023 14:05:58 +0900 Subject: [PATCH 07/13] enhance(backend): tweak hashtag trend --- packages/backend/src/core/HashtagService.ts | 7 +++++-- .../backend/src/server/api/endpoints/hashtags/trend.ts | 6 ------ 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/packages/backend/src/core/HashtagService.ts b/packages/backend/src/core/HashtagService.ts index 4900fa90a1..1a06767da8 100644 --- a/packages/backend/src/core/HashtagService.ts +++ b/packages/backend/src/core/HashtagService.ts @@ -5,7 +5,6 @@ import { Inject, Injectable } from '@nestjs/common'; import * as Redis from 'ioredis'; -import { getLineAndCharacterOfPosition } from 'typescript'; import { DI } from '@/di-symbols.js'; import type { MiUser } from '@/models/User.js'; import { normalizeForSearch } from '@/misc/normalize-for-search.js'; @@ -15,6 +14,7 @@ import type { HashtagsRepository } from '@/models/_.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { bindThis } from '@/decorators.js'; import { FeaturedService } from '@/core/FeaturedService.js'; +import { MetaService } from '@/core/MetaService.js'; @Injectable() export class HashtagService { @@ -28,6 +28,7 @@ export class HashtagService { private userEntityService: UserEntityService, private featuredService: FeaturedService, private idService: IdService, + private metaService: MetaService, ) { } @@ -157,7 +158,9 @@ export class HashtagService { @bindThis public async updateHashtagsRanking(hashtag: string, userId: MiUser['id']): Promise { - // TODO: instance.hiddenTagsの考慮 + const instance = await this.metaService.fetch(); + const hiddenTags = instance.hiddenTags.map(t => normalizeForSearch(t)); + if (hiddenTags.includes(hashtag)) return; // YYYYMMDDHHmm (10分間隔) const now = new Date(); diff --git a/packages/backend/src/server/api/endpoints/hashtags/trend.ts b/packages/backend/src/server/api/endpoints/hashtags/trend.ts index a69e007a40..8f382eb96b 100644 --- a/packages/backend/src/server/api/endpoints/hashtags/trend.ts +++ b/packages/backend/src/server/api/endpoints/hashtags/trend.ts @@ -5,8 +5,6 @@ import { Inject, Injectable } from '@nestjs/common'; import { Endpoint } from '@/server/api/endpoint-base.js'; -import { normalizeForSearch } from '@/misc/normalize-for-search.js'; -import { MetaService } from '@/core/MetaService.js'; import { DI } from '@/di-symbols.js'; import { FeaturedService } from '@/core/FeaturedService.js'; import { HashtagService } from '@/core/HashtagService.js'; @@ -55,14 +53,10 @@ export const paramDef = { @Injectable() export default class extends Endpoint { // eslint-disable-line import/no-default-export constructor( - private metaService: MetaService, private featuredService: FeaturedService, private hashtagService: HashtagService, ) { super(meta, paramDef, async () => { - const instance = await this.metaService.fetch(true); - const hiddenTags = instance.hiddenTags.map(t => normalizeForSearch(t)); - const ranking = await this.featuredService.getHashtagsRanking(10); const charts = ranking.length === 0 ? {} : await this.hashtagService.getCharts(ranking, 20); From aae1034d621501a7121a73db660fcd7c6998d9ff Mon Sep 17 00:00:00 2001 From: syuilo Date: Sat, 7 Oct 2023 14:10:45 +0900 Subject: [PATCH 08/13] 2023.10.0-beta.6 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index a52c773dd8..df51a05f88 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "misskey", - "version": "2023.10.0-beta.5", + "version": "2023.10.0-beta.6", "codename": "nasubi", "repository": { "type": "git", From fb3338029bdb72543345c9a69cbe5318b5676233 Mon Sep 17 00:00:00 2001 From: syuilo Date: Sat, 7 Oct 2023 17:48:10 +0900 Subject: [PATCH 09/13] refactor --- .../server/api/endpoints/antennas/notes.ts | 12 +++---- .../api/endpoints/notes/hybrid-timeline.ts | 32 ++++++++----------- .../api/endpoints/notes/local-timeline.ts | 16 ++++------ .../server/api/endpoints/notes/timeline.ts | 16 ++++------ .../api/endpoints/notes/user-list-timeline.ts | 16 ++++------ .../src/server/api/endpoints/roles/notes.ts | 12 +++---- 6 files changed, 40 insertions(+), 64 deletions(-) diff --git a/packages/backend/src/server/api/endpoints/antennas/notes.ts b/packages/backend/src/server/api/endpoints/antennas/notes.ts index 63e542cb62..b563f704a8 100644 --- a/packages/backend/src/server/api/endpoints/antennas/notes.ts +++ b/packages/backend/src/server/api/endpoints/antennas/notes.ts @@ -86,17 +86,13 @@ export default class extends Endpoint { // eslint- }); const limit = ps.limit + (ps.untilId ? 1 : 0) + (ps.sinceId ? 1 : 0); // untilIdに指定したものも含まれるため+1 - const noteIdsRes = await this.redisForTimelines.xrevrange( + + const noteIds = await this.redisForTimelines.xrevrange( `antennaTimeline:${antenna.id}`, ps.untilId ? this.idService.parse(ps.untilId).date.getTime() : ps.untilDate ?? '+', ps.sinceId ? this.idService.parse(ps.sinceId).date.getTime() : ps.sinceDate ?? '-', - 'COUNT', limit); - - if (noteIdsRes.length === 0) { - return []; - } - - const noteIds = noteIdsRes.map(x => x[1][1]).filter(x => x !== ps.untilId && x !== ps.sinceId); + 'COUNT', limit, + ).then(res => res.map(x => x[1][1]).filter(x => x !== ps.untilId && x !== ps.sinceId)); if (noteIds.length === 0) { return []; diff --git a/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts b/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts index 8e7f2a2a98..33ffd42a76 100644 --- a/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts @@ -92,26 +92,22 @@ export default class extends Endpoint { // eslint- let timeline: MiNote[] = []; const limit = ps.limit + (ps.untilId ? 1 : 0) + (ps.sinceId ? 1 : 0); // untilIdに指定したものも含まれるため+1 - let htlNoteIdsRes: [string, string[]][] = []; - let ltlNoteIdsRes: [string, string[]][] = []; - if (!ps.sinceId && !ps.sinceDate) { - [htlNoteIdsRes, ltlNoteIdsRes] = await Promise.all([ - this.redisForTimelines.xrevrange( - ps.withFiles ? `homeTimelineWithFiles:${me.id}` : `homeTimeline:${me.id}`, - ps.untilId ? this.idService.parse(ps.untilId).date.getTime() : ps.untilDate ?? '+', - ps.sinceId ? this.idService.parse(ps.sinceId).date.getTime() : ps.sinceDate ?? '-', - 'COUNT', limit), - this.redisForTimelines.xrevrange( - ps.withFiles ? 'localTimelineWithFiles' : 'localTimeline', - ps.untilId ? this.idService.parse(ps.untilId).date.getTime() : ps.untilDate ?? '+', - ps.sinceId ? this.idService.parse(ps.sinceId).date.getTime() : ps.sinceDate ?? '-', - 'COUNT', limit), - ]); - } + const [htlNoteIds, ltlNoteIds] = await Promise.all([ + this.redisForTimelines.xrevrange( + ps.withFiles ? `homeTimelineWithFiles:${me.id}` : `homeTimeline:${me.id}`, + ps.untilId ? this.idService.parse(ps.untilId).date.getTime() : ps.untilDate ?? '+', + ps.sinceId ? this.idService.parse(ps.sinceId).date.getTime() : ps.sinceDate ?? '-', + 'COUNT', limit, + ).then(res => res.map(x => x[1][1]).filter(x => x !== ps.untilId && x !== ps.sinceId)), + this.redisForTimelines.xrevrange( + ps.withFiles ? 'localTimelineWithFiles' : 'localTimeline', + ps.untilId ? this.idService.parse(ps.untilId).date.getTime() : ps.untilDate ?? '+', + ps.sinceId ? this.idService.parse(ps.sinceId).date.getTime() : ps.sinceDate ?? '-', + 'COUNT', limit, + ).then(res => res.map(x => x[1][1]).filter(x => x !== ps.untilId && x !== ps.sinceId)), + ]); - const htlNoteIds = htlNoteIdsRes.map(x => x[1][1]).filter(x => x !== ps.untilId && x !== ps.sinceId); - const ltlNoteIds = ltlNoteIdsRes.map(x => x[1][1]).filter(x => x !== ps.untilId && x !== ps.sinceId); let noteIds = Array.from(new Set([...htlNoteIds, ...ltlNoteIds])); noteIds.sort((a, b) => a > b ? -1 : 1); noteIds = noteIds.slice(0, ps.limit); diff --git a/packages/backend/src/server/api/endpoints/notes/local-timeline.ts b/packages/backend/src/server/api/endpoints/notes/local-timeline.ts index ac8f8d610e..b9305e8e26 100644 --- a/packages/backend/src/server/api/endpoints/notes/local-timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/local-timeline.ts @@ -88,17 +88,13 @@ export default class extends Endpoint { // eslint- let timeline: MiNote[] = []; const limit = ps.limit + (ps.untilId ? 1 : 0) + (ps.sinceId ? 1 : 0); // untilIdに指定したものも含まれるため+1 - let noteIdsRes: [string, string[]][] = []; - if (!ps.sinceId && !ps.sinceDate) { - noteIdsRes = await this.redisForTimelines.xrevrange( - ps.withFiles ? 'localTimelineWithFiles' : 'localTimeline', - ps.untilId ? this.idService.parse(ps.untilId).date.getTime() : ps.untilDate ?? '+', - ps.sinceId ? this.idService.parse(ps.sinceId).date.getTime() : ps.sinceDate ?? '-', - 'COUNT', limit); - } - - const noteIds = noteIdsRes.map(x => x[1][1]).filter(x => x !== ps.untilId && x !== ps.sinceId); + const noteIds = await this.redisForTimelines.xrevrange( + ps.withFiles ? 'localTimelineWithFiles' : 'localTimeline', + ps.untilId ? this.idService.parse(ps.untilId).date.getTime() : ps.untilDate ?? '+', + ps.sinceId ? this.idService.parse(ps.sinceId).date.getTime() : ps.sinceDate ?? '-', + 'COUNT', limit, + ).then(res => res.map(x => x[1][1]).filter(x => x !== ps.untilId && x !== ps.sinceId)); if (noteIds.length === 0) { return []; diff --git a/packages/backend/src/server/api/endpoints/notes/timeline.ts b/packages/backend/src/server/api/endpoints/notes/timeline.ts index 7442356978..cd2ec8fe29 100644 --- a/packages/backend/src/server/api/endpoints/notes/timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/timeline.ts @@ -79,17 +79,13 @@ export default class extends Endpoint { // eslint- let timeline: MiNote[] = []; const limit = ps.limit + (ps.untilId ? 1 : 0) + (ps.sinceId ? 1 : 0); // untilIdに指定したものも含まれるため+1 - let noteIdsRes: [string, string[]][] = []; - if (!ps.sinceId && !ps.sinceDate) { - noteIdsRes = await this.redisForTimelines.xrevrange( - ps.withFiles ? `homeTimelineWithFiles:${me.id}` : `homeTimeline:${me.id}`, - ps.untilId ? this.idService.parse(ps.untilId).date.getTime() : ps.untilDate ?? '+', - ps.sinceId ? this.idService.parse(ps.sinceId).date.getTime() : ps.sinceDate ?? '-', - 'COUNT', limit); - } - - const noteIds = noteIdsRes.map(x => x[1][1]).filter(x => x !== ps.untilId && x !== ps.sinceId); + const noteIds = await this.redisForTimelines.xrevrange( + ps.withFiles ? `homeTimelineWithFiles:${me.id}` : `homeTimeline:${me.id}`, + ps.untilId ? this.idService.parse(ps.untilId).date.getTime() : ps.untilDate ?? '+', + ps.sinceId ? this.idService.parse(ps.sinceId).date.getTime() : ps.sinceDate ?? '-', + 'COUNT', limit, + ).then(res => res.map(x => x[1][1]).filter(x => x !== ps.untilId && x !== ps.sinceId)); if (noteIds.length === 0) { return []; diff --git a/packages/backend/src/server/api/endpoints/notes/user-list-timeline.ts b/packages/backend/src/server/api/endpoints/notes/user-list-timeline.ts index d11e9751b3..2eec6297c0 100644 --- a/packages/backend/src/server/api/endpoints/notes/user-list-timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/user-list-timeline.ts @@ -103,17 +103,13 @@ export default class extends Endpoint { // eslint- let timeline: MiNote[] = []; const limit = ps.limit + (ps.untilId ? 1 : 0) + (ps.sinceId ? 1 : 0); // untilIdに指定したものも含まれるため+1 - let noteIdsRes: [string, string[]][] = []; - if (!ps.sinceId && !ps.sinceDate) { - noteIdsRes = await this.redisForTimelines.xrevrange( - ps.withFiles ? `userListTimelineWithFiles:${list.id}` : `userListTimeline:${list.id}`, - ps.untilId ? this.idService.parse(ps.untilId).date.getTime() : ps.untilDate ?? '+', - ps.sinceId ? this.idService.parse(ps.sinceId).date.getTime() : ps.sinceDate ?? '-', - 'COUNT', limit); - } - - const noteIds = noteIdsRes.map(x => x[1][1]).filter(x => x !== ps.untilId && x !== ps.sinceId); + const noteIds = await this.redisForTimelines.xrevrange( + ps.withFiles ? `userListTimelineWithFiles:${list.id}` : `userListTimeline:${list.id}`, + ps.untilId ? this.idService.parse(ps.untilId).date.getTime() : ps.untilDate ?? '+', + ps.sinceId ? this.idService.parse(ps.sinceId).date.getTime() : ps.sinceDate ?? '-', + 'COUNT', limit, + ).then(res => res.map(x => x[1][1]).filter(x => x !== ps.untilId && x !== ps.sinceId)); if (noteIds.length === 0) { return []; diff --git a/packages/backend/src/server/api/endpoints/roles/notes.ts b/packages/backend/src/server/api/endpoints/roles/notes.ts index f2533efa36..3f2b31c02f 100644 --- a/packages/backend/src/server/api/endpoints/roles/notes.ts +++ b/packages/backend/src/server/api/endpoints/roles/notes.ts @@ -79,17 +79,13 @@ export default class extends Endpoint { // eslint- return []; } const limit = ps.limit + (ps.untilId ? 1 : 0) + (ps.sinceId ? 1 : 0); // untilIdに指定したものも含まれるため+1 - const noteIdsRes = await this.redisForTimelines.xrevrange( + + const noteIds = await this.redisForTimelines.xrevrange( `roleTimeline:${role.id}`, ps.untilId ? this.idService.parse(ps.untilId).date.getTime() : ps.untilDate ?? '+', ps.sinceId ? this.idService.parse(ps.sinceId).date.getTime() : ps.sinceDate ?? '-', - 'COUNT', limit); - - if (noteIdsRes.length === 0) { - return []; - } - - const noteIds = noteIdsRes.map(x => x[1][1]).filter(x => x !== ps.untilId && x !== ps.sinceId); + 'COUNT', limit, + ).then(res => res.map(x => x[1][1]).filter(x => x !== ps.untilId && x !== ps.sinceId)); if (noteIds.length === 0) { return []; From 69de8cad7cc0664db10f8459ebb752b75f669a81 Mon Sep 17 00:00:00 2001 From: syuilo Date: Sat, 7 Oct 2023 17:57:15 +0900 Subject: [PATCH 10/13] refactor --- .../src/server/api/endpoints/users/notes.ts | 52 +++++++++---------- 1 file changed, 25 insertions(+), 27 deletions(-) diff --git a/packages/backend/src/server/api/endpoints/users/notes.ts b/packages/backend/src/server/api/endpoints/users/notes.ts index 23c2811fb9..4c0f0cc588 100644 --- a/packages/backend/src/server/api/endpoints/users/notes.ts +++ b/packages/backend/src/server/api/endpoints/users/notes.ts @@ -81,38 +81,36 @@ export default class extends Endpoint { // eslint- let timeline: MiNote[] = []; const limit = ps.limit + (ps.untilId ? 1 : 0) + (ps.sinceId ? 1 : 0); // untilIdに指定したものも含まれるため+1 - let noteIdsRes: [string, string[]][] = []; - let repliesNoteIdsRes: [string, string[]][] = []; - let channelNoteIdsRes: [string, string[]][] = []; - if (!ps.sinceId && !ps.sinceDate) { - [noteIdsRes, repliesNoteIdsRes, channelNoteIdsRes] = await Promise.all([ - this.redisForTimelines.xrevrange( - ps.withFiles ? `userTimelineWithFiles:${ps.userId}` : `userTimeline:${ps.userId}`, + const [noteIdsRes, repliesNoteIdsRes, channelNoteIdsRes] = await Promise.all([ + this.redisForTimelines.xrevrange( + ps.withFiles ? `userTimelineWithFiles:${ps.userId}` : `userTimeline:${ps.userId}`, + ps.untilId ? this.idService.parse(ps.untilId).date.getTime() : ps.untilDate ?? '+', + ps.sinceId ? this.idService.parse(ps.sinceId).date.getTime() : ps.sinceDate ?? '-', + 'COUNT', limit, + ).then(res => res.map(x => x[1][1]).filter(x => x !== ps.untilId && x !== ps.sinceId)), + ps.withReplies + ? this.redisForTimelines.xrevrange( + `userTimelineWithReplies:${ps.userId}`, ps.untilId ? this.idService.parse(ps.untilId).date.getTime() : ps.untilDate ?? '+', ps.sinceId ? this.idService.parse(ps.sinceId).date.getTime() : ps.sinceDate ?? '-', - 'COUNT', limit), - ps.withReplies - ? this.redisForTimelines.xrevrange( - `userTimelineWithReplies:${ps.userId}`, - ps.untilId ? this.idService.parse(ps.untilId).date.getTime() : ps.untilDate ?? '+', - ps.sinceId ? this.idService.parse(ps.sinceId).date.getTime() : ps.sinceDate ?? '-', - 'COUNT', limit) - : Promise.resolve([]), - ps.withChannelNotes - ? this.redisForTimelines.xrevrange( - `userTimelineWithChannel:${ps.userId}`, - ps.untilId ? this.idService.parse(ps.untilId).date.getTime() : ps.untilDate ?? '+', - ps.sinceId ? this.idService.parse(ps.sinceId).date.getTime() : ps.sinceDate ?? '-', - 'COUNT', limit) - : Promise.resolve([]), - ]); - } + 'COUNT', limit, + ).then(res => res.map(x => x[1][1]).filter(x => x !== ps.untilId && x !== ps.sinceId)) + : Promise.resolve([]), + ps.withChannelNotes + ? this.redisForTimelines.xrevrange( + `userTimelineWithChannel:${ps.userId}`, + ps.untilId ? this.idService.parse(ps.untilId).date.getTime() : ps.untilDate ?? '+', + ps.sinceId ? this.idService.parse(ps.sinceId).date.getTime() : ps.sinceDate ?? '-', + 'COUNT', limit, + ).then(res => res.map(x => x[1][1]).filter(x => x !== ps.untilId && x !== ps.sinceId)) + : Promise.resolve([]), + ]); let noteIds = Array.from(new Set([ - ...noteIdsRes.map(x => x[1][1]).filter(x => x !== ps.untilId && x !== ps.sinceId), - ...repliesNoteIdsRes.map(x => x[1][1]).filter(x => x !== ps.untilId && x !== ps.sinceId), - ...channelNoteIdsRes.map(x => x[1][1]).filter(x => x !== ps.untilId && x !== ps.sinceId), + ...noteIdsRes, + ...repliesNoteIdsRes, + ...channelNoteIdsRes, ])); noteIds.sort((a, b) => a > b ? -1 : 1); noteIds = noteIds.slice(0, ps.limit); From 8c684d539181f030d03db1a6e0f408ae5d8a93ce Mon Sep 17 00:00:00 2001 From: syuilo Date: Sat, 7 Oct 2023 18:00:56 +0900 Subject: [PATCH 11/13] =?UTF-8?q?enhance(backend):=20User=20TL=E3=82=92Red?= =?UTF-8?q?is=E3=81=AB=E3=82=AD=E3=83=A3=E3=83=83=E3=82=B7=E3=83=A5?= =?UTF-8?q?=E3=81=95=E3=82=8C=E3=82=8B=E4=BB=A5=E5=89=8D=E3=81=BE=E3=81=A7?= =?UTF-8?q?=E9=81=A1=E3=82=8C=E3=82=8B=E3=82=88=E3=81=86=E3=81=AB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #11958 --- .../src/server/api/endpoints/users/notes.ts | 41 ++++++++++++++++++- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/server/api/endpoints/users/notes.ts b/packages/backend/src/server/api/endpoints/users/notes.ts index 4c0f0cc588..becc6debbb 100644 --- a/packages/backend/src/server/api/endpoints/users/notes.ts +++ b/packages/backend/src/server/api/endpoints/users/notes.ts @@ -10,10 +10,10 @@ import type { MiNote, NotesRepository } from '@/models/_.js'; import { Endpoint } from '@/server/api/endpoint-base.js'; import { NoteEntityService } from '@/core/entities/NoteEntityService.js'; import { DI } from '@/di-symbols.js'; -import { GetterService } from '@/server/api/GetterService.js'; import { CacheService } from '@/core/CacheService.js'; import { IdService } from '@/core/IdService.js'; import { isUserRelated } from '@/misc/is-user-related.js'; +import { QueryService } from '@/core/QueryService.js'; import { ApiError } from '../../error.js'; export const meta = { @@ -67,7 +67,7 @@ export default class extends Endpoint { // eslint- private notesRepository: NotesRepository, private noteEntityService: NoteEntityService, - private getterService: GetterService, + private queryService: QueryService, private cacheService: CacheService, private idService: IdService, ) { @@ -148,6 +148,43 @@ export default class extends Endpoint { // eslint- timeline.sort((a, b) => a.id > b.id ? -1 : 1); + // fallback to database + if (timeline.length === 0) { + //#region Construct query + const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), ps.sinceId, ps.untilId, ps.sinceDate, ps.untilDate) + .andWhere('note.userId = :userId', { userId: ps.userId }) + .innerJoinAndSelect('note.user', 'user') + .leftJoinAndSelect('note.reply', 'reply') + .leftJoinAndSelect('note.renote', 'renote') + .leftJoinAndSelect('note.channel', 'channel') + .leftJoinAndSelect('reply.user', 'replyUser') + .leftJoinAndSelect('renote.user', 'renoteUser'); + + query.andWhere(new Brackets(qb => { + qb.orWhere('note.channelId IS NULL'); + qb.orWhere('channel.isSensitive = false'); + })); + + this.queryService.generateVisibilityQuery(query, me); + + if (ps.withFiles) { + query.andWhere('note.fileIds != \'{}\''); + } + + if (ps.includeMyRenotes === false) { + query.andWhere(new Brackets(qb => { + qb.orWhere('note.userId != :userId', { userId: ps.userId }); + qb.orWhere('note.renoteId IS NULL'); + qb.orWhere('note.text IS NOT NULL'); + qb.orWhere('note.fileIds != \'{}\''); + qb.orWhere('0 < (SELECT COUNT(*) FROM poll WHERE poll."noteId" = note.id)'); + })); + } + //#endregion + + timeline = await query.limit(ps.limit).getMany(); + } + return await this.noteEntityService.packMany(timeline, me); }); } From 986623dbdca61473fb4e60ff746c6c77e3a9d442 Mon Sep 17 00:00:00 2001 From: syuilo Date: Sat, 7 Oct 2023 18:21:16 +0900 Subject: [PATCH 12/13] fix(backend): fix sql error when featured notes is zero --- .../src/server/api/endpoints/users/featured-notes.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/backend/src/server/api/endpoints/users/featured-notes.ts b/packages/backend/src/server/api/endpoints/users/featured-notes.ts index fdf36a6ae0..dec0b7a122 100644 --- a/packages/backend/src/server/api/endpoints/users/featured-notes.ts +++ b/packages/backend/src/server/api/endpoints/users/featured-notes.ts @@ -50,16 +50,16 @@ export default class extends Endpoint { // eslint- super(meta, paramDef, async (ps, me) => { let noteIds = await this.featuredService.getPerUserNotesRanking(ps.userId, 50); - if (noteIds.length === 0) { - return []; - } - noteIds.sort((a, b) => a > b ? -1 : 1); if (ps.untilId) { noteIds = noteIds.filter(id => id < ps.untilId!); } noteIds = noteIds.slice(0, ps.limit); + if (noteIds.length === 0) { + return []; + } + const query = this.notesRepository.createQueryBuilder('note') .where('note.id IN (:...noteIds)', { noteIds: noteIds }) .innerJoinAndSelect('note.user', 'user') From 6d5e18aa8dbfc5fbf15b12d0f711921df03e9372 Mon Sep 17 00:00:00 2001 From: syuilo Date: Sat, 7 Oct 2023 18:21:30 +0900 Subject: [PATCH 13/13] 2023.10.0-beta.7 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index df51a05f88..d298fbed90 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "misskey", - "version": "2023.10.0-beta.6", + "version": "2023.10.0-beta.7", "codename": "nasubi", "repository": { "type": "git",