feat: export clips

locales/index.d.ts

@@ -2256,6 +2256,7 @@ export interface Locale {
     "_exportOrImport": {
         "allNotes": string;
         "favoritedNotes": string;
+        "clips": string;
         "followingList": string;
         "muteList": string;
         "blockingList": string;

locales/ja-JP.yml

@@ -2159,6 +2159,7 @@ _profile:
 _exportOrImport:
   allNotes: "全てのノート"
   favoritedNotes: "お気に入りにしたノート"
+  clips: "クリップ"
   followingList: "フォロー"
   muteList: "ミュート"
   blockingList: "ブロック"

packages/backend/src/core/QueueService.ts

@@ -182,6 +182,16 @@ export class QueueService {
 		});
 	}
 
+	@bindThis
+	public createExportClipsJob(user: ThinUser) {
+		return this.dbQueue.add('exportClips', {
+			user: { id: user.id },
+		}, {
+			removeOnComplete: true,
+			removeOnFail: true,
+		});
+	}
+
 	@bindThis
 	public createExportFavoritesJob(user: ThinUser) {
 		return this.dbQueue.add('exportFavorites', {

packages/backend/src/queue/QueueProcessorModule.ts

@@ -24,6 +24,7 @@ import { ExportCustomEmojisProcessorService } from './processors/ExportCustomEmo
 import { ExportFollowingProcessorService } from './processors/ExportFollowingProcessorService.js';
 import { ExportMutingProcessorService } from './processors/ExportMutingProcessorService.js';
 import { ExportNotesProcessorService } from './processors/ExportNotesProcessorService.js';
+import { ExportClipsProcessorService } from './processors/ExportClipsProcessorService.js';
 import { ExportUserListsProcessorService } from './processors/ExportUserListsProcessorService.js';
 import { ExportAntennasProcessorService } from './processors/ExportAntennasProcessorService.js';
 import { ImportBlockingProcessorService } from './processors/ImportBlockingProcessorService.js';
@@ -53,6 +54,7 @@ import { RelationshipProcessorService } from './processors/RelationshipProcessor
 		DeleteDriveFilesProcessorService,
 		ExportCustomEmojisProcessorService,
 		ExportNotesProcessorService,
+		ExportClipsProcessorService,
 		ExportFavoritesProcessorService,
 		ExportFollowingProcessorService,
 		ExportMutingProcessorService,

packages/backend/src/queue/QueueProcessorService.ts

@@ -16,6 +16,7 @@ import { InboxProcessorService } from './processors/InboxProcessorService.js';
 import { DeleteDriveFilesProcessorService } from './processors/DeleteDriveFilesProcessorService.js';
 import { ExportCustomEmojisProcessorService } from './processors/ExportCustomEmojisProcessorService.js';
 import { ExportNotesProcessorService } from './processors/ExportNotesProcessorService.js';
+import { ExportClipsProcessorService } from './processors/ExportClipsProcessorService.js';
 import { ExportFollowingProcessorService } from './processors/ExportFollowingProcessorService.js';
 import { ExportMutingProcessorService } from './processors/ExportMutingProcessorService.js';
 import { ExportBlockingProcessorService } from './processors/ExportBlockingProcessorService.js';
@@ -91,6 +92,7 @@ export class QueueProcessorService implements OnApplicationShutdown {
 		private deleteDriveFilesProcessorService: DeleteDriveFilesProcessorService,
 		private exportCustomEmojisProcessorService: ExportCustomEmojisProcessorService,
 		private exportNotesProcessorService: ExportNotesProcessorService,
+		private exportClipsProcessorService: ExportClipsProcessorService,
 		private exportFavoritesProcessorService: ExportFavoritesProcessorService,
 		private exportFollowingProcessorService: ExportFollowingProcessorService,
 		private exportMutingProcessorService: ExportMutingProcessorService,
@@ -164,6 +166,7 @@ export class QueueProcessorService implements OnApplicationShutdown {
 				case 'deleteDriveFiles': return this.deleteDriveFilesProcessorService.process(job);
 				case 'exportCustomEmojis': return this.exportCustomEmojisProcessorService.process(job);
 				case 'exportNotes': return this.exportNotesProcessorService.process(job);
+				case 'exportClips': return this.exportClipsProcessorService.process(job);
 				case 'exportFavorites': return this.exportFavoritesProcessorService.process(job);
 				case 'exportFollowing': return this.exportFollowingProcessorService.process(job);
 				case 'exportMuting': return this.exportMutingProcessorService.process(job);

packages/backend/src/queue/processors/ExportClipsProcessorService.ts

@@ -0,0 +1,206 @@
+/*
+ * SPDX-FileCopyrightText: syuilo and other misskey contributors
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+import * as fs from 'node:fs';
+import { Writable } from 'node:stream';
+import { Inject, Injectable, StreamableFile } from '@nestjs/common';
+import { MoreThan } from 'typeorm';
+import { format as dateFormat } from 'date-fns';
+import { DI } from '@/di-symbols.js';
+import type { ClipNotesRepository, ClipsRepository, MiClip, MiClipNote, MiUser, NotesRepository, PollsRepository, UsersRepository } from '@/models/_.js';
+import type Logger from '@/logger.js';
+import { DriveService } from '@/core/DriveService.js';
+import { createTemp } from '@/misc/create-temp.js';
+import type { MiPoll } from '@/models/Poll.js';
+import type { MiNote } from '@/models/Note.js';
+import { bindThis } from '@/decorators.js';
+import { DriveFileEntityService } from '@/core/entities/DriveFileEntityService.js';
+import { Packed } from '@/misc/json-schema.js';
+import { IdService } from '@/core/IdService.js';
+import { QueueLoggerService } from '../QueueLoggerService.js';
+import type * as Bull from 'bullmq';
+import type { DbJobDataWithUser } from '../types.js';
+
+@Injectable()
+export class ExportClipsProcessorService {
+	private logger: Logger;
+
+	constructor(
+		@Inject(DI.usersRepository)
+		private usersRepository: UsersRepository,
+
+		@Inject(DI.pollsRepository)
+		private pollsRepository: PollsRepository,
+
+		@Inject(DI.clipsRepository)
+		private clipsRepository: ClipsRepository,
+
+		@Inject(DI.clipNotesRepository)
+		private clipNotesRepository: ClipNotesRepository,
+
+		private driveService: DriveService,
+		private queueLoggerService: QueueLoggerService,
+		private idService: IdService,
+	) {
+		this.logger = this.queueLoggerService.logger.createSubLogger('export-clips');
+	}
+
+	@bindThis
+	public async process(job: Bull.Job<DbJobDataWithUser>): Promise<void> {
+		this.logger.info(`Exporting clips of ${job.data.user.id} ...`);
+
+		const user = await this.usersRepository.findOneBy({ id: job.data.user.id });
+		if (user == null) {
+			return;
+		}
+
+		// Create temp file
+		const [path, cleanup] = await createTemp();
+
+		this.logger.info(`Temp file is ${path}`);
+
+		try {
+			const stream = Writable.toWeb(fs.createWriteStream(path, { flags: 'a' }));
+			const writer = stream.getWriter();
+			writer.closed.catch(this.logger.error);
+
+			await writer.write('[');
+
+			await this.processClips(writer, user, job);
+
+			await writer.write(']');
+			await writer.close();
+
+			this.logger.succ(`Exported to: ${path}`);
+
+			const fileName = 'clips-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.json';
+			const driveFile = await this.driveService.addFile({ user, path, name: fileName, force: true, ext: 'json' });
+
+			this.logger.succ(`Exported to: ${driveFile.id}`);
+		} finally {
+			cleanup();
+		}
+	}
+
+	async processClips(writer: WritableStreamDefaultWriter, user: MiUser, job: Bull.Job<DbJobDataWithUser>) {
+		let exportedClipsCount = 0;
+		let cursor: MiClip['id'] | null = null;
+
+		while (true) {
+			const clips = await this.clipsRepository.find({
+				where: {
+					userId: user.id,
+					...(cursor ? { id: MoreThan(cursor) } : {}),
+				},
+				take: 100,
+				order: {
+					id: 1,
+				},
+			});
+
+			if (clips.length === 0) {
+				job.updateProgress(100);
+				break;
+			}
+
+			cursor = clips.at(-1)?.id ?? null;
+
+			for (const clip of clips) {
+				// Stringify but remove the last `]}`
+				const content = JSON.stringify(this.serializeClip(clip)).slice(0, -2);
+				const isFirst = exportedClipsCount === 0;
+				await writer.write(isFirst ? content : ',\n' + content);
+
+				await this.processClipNotes(writer, clip.id);
+
+				await writer.write(']}');
+				exportedClipsCount++;
+			}
+
+			const total = await this.clipsRepository.countBy({
+				userId: user.id,
+			});
+
+			job.updateProgress(exportedClipsCount / total);
+		}
+	}
+
+	async processClipNotes(writer: WritableStreamDefaultWriter, clipId: string): Promise<void> {
+		let exportedClipNotesCount = 0;
+		let cursor: MiClipNote['id'] | null = null;
+
+		while (true) {
+			const clipNotes = await this.clipNotesRepository.find({
+				where: {
+					clipId,
+					...(cursor ? { id: MoreThan(cursor) } : {}),
+				},
+				take: 100,
+				order: {
+					id: 1,
+				},
+				relations: ['note', 'note.user'],
+			}) as (MiClipNote & { note: MiNote & { user: MiUser } })[];
+
+			if (clipNotes.length === 0) {
+				break;
+			}
+
+			cursor = clipNotes.at(-1)?.id ?? null;
+
+			for (const clipNote of clipNotes) {
+				let poll: MiPoll | undefined;
+				if (clipNote.note.hasPoll) {
+					poll = await this.pollsRepository.findOneByOrFail({ noteId: clipNote.note.id });
+				}
+				const content = JSON.stringify(this.serializeClipNote(clipNote, poll));
+				const isFirst = exportedClipNotesCount === 0;
+				await writer.write(isFirst ? content : ',\n' + content);
+
+				exportedClipNotesCount++;
+			}
+		}
+	}
+
+	private serializeClip(clip: MiClip): Record<string, unknown> {
+		return {
+			id: clip.id,
+			name: clip.name,
+			description: clip.description,
+			lastClippedAt: clip.lastClippedAt?.toISOString(),
+			clipNotes: [],
+		};
+	}
+
+	private serializeClipNote(clip: MiClipNote & { note: MiNote & { user: MiUser } }, poll: MiPoll | undefined): Record<string, unknown> {
+		return {
+			id: clip.id,
+			createdAt: this.idService.parse(clip.id).date.toISOString(),
+			note: {
+				id: clip.note.id,
+				text: clip.note.text,
+				createdAt: this.idService.parse(clip.note.id).date.toISOString(),
+				fileIds: clip.note.fileIds,
+				replyId: clip.note.replyId,
+				renoteId: clip.note.renoteId,
+				poll: poll,
+				cw: clip.note.cw,
+				visibility: clip.note.visibility,
+				visibleUserIds: clip.note.visibleUserIds,
+				localOnly: clip.note.localOnly,
+				reactionAcceptance: clip.note.reactionAcceptance,
+				uri: clip.note.uri,
+				url: clip.note.url,
+				user: {
+					id: clip.note.user.id,
+					name: clip.note.user.name,
+					username: clip.note.user.username,
+					host: clip.note.user.host,
+					uri: clip.note.user.uri,
+				},
+			},
+		};
+	}
+}

packages/backend/src/server/api/EndpointsModule.ts

@@ -208,6 +208,7 @@ import * as ep___i_exportBlocking from './endpoints/i/export-blocking.js';
 import * as ep___i_exportFollowing from './endpoints/i/export-following.js';
 import * as ep___i_exportMute from './endpoints/i/export-mute.js';
 import * as ep___i_exportNotes from './endpoints/i/export-notes.js';
+import * as ep___i_exportClips from './endpoints/i/export-clips.js';
 import * as ep___i_exportFavorites from './endpoints/i/export-favorites.js';
 import * as ep___i_exportUserLists from './endpoints/i/export-user-lists.js';
 import * as ep___i_exportAntennas from './endpoints/i/export-antennas.js';
@@ -569,6 +570,7 @@ const $i_exportBlocking: Provider = { provide: 'ep:i/export-blocking', useClass:
 const $i_exportFollowing: Provider = { provide: 'ep:i/export-following', useClass: ep___i_exportFollowing.default };
 const $i_exportMute: Provider = { provide: 'ep:i/export-mute', useClass: ep___i_exportMute.default };
 const $i_exportNotes: Provider = { provide: 'ep:i/export-notes', useClass: ep___i_exportNotes.default };
+const $i_exportClips: Provider = { provide: 'ep:i/export-clips', useClass: ep___i_exportClips.default };
 const $i_exportFavorites: Provider = { provide: 'ep:i/export-favorites', useClass: ep___i_exportFavorites.default };
 const $i_exportUserLists: Provider = { provide: 'ep:i/export-user-lists', useClass: ep___i_exportUserLists.default };
 const $i_exportAntennas: Provider = { provide: 'ep:i/export-antennas', useClass: ep___i_exportAntennas.default };
@@ -934,6 +936,7 @@ const $retention: Provider = { provide: 'ep:retention', useClass: ep___retention
 		$i_exportFollowing,
 		$i_exportMute,
 		$i_exportNotes,
+		$i_exportClips,
 		$i_exportFavorites,
 		$i_exportUserLists,
 		$i_exportAntennas,
@@ -1293,6 +1296,7 @@ const $retention: Provider = { provide: 'ep:retention', useClass: ep___retention
 		$i_exportFollowing,
 		$i_exportMute,
 		$i_exportNotes,
+		$i_exportClips,
 		$i_exportFavorites,
 		$i_exportUserLists,
 		$i_exportAntennas,

packages/backend/src/server/api/endpoints.ts

@@ -3,8 +3,8 @@
  * SPDX-License-Identifier: AGPL-3.0-only
  */
 
-import type { Schema } from '@/misc/json-schema.js';
 import { permissions } from 'misskey-js';
+import type { Schema } from '@/misc/json-schema.js';
 import { RolePolicies } from '@/core/RoleService.js';
 
 import * as ep___admin_meta from './endpoints/admin/meta.js';
@@ -209,6 +209,7 @@ import * as ep___i_exportBlocking from './endpoints/i/export-blocking.js';
 import * as ep___i_exportFollowing from './endpoints/i/export-following.js';
 import * as ep___i_exportMute from './endpoints/i/export-mute.js';
 import * as ep___i_exportNotes from './endpoints/i/export-notes.js';
+import * as ep___i_exportClips from './endpoints/i/export-clips.js';
 import * as ep___i_exportFavorites from './endpoints/i/export-favorites.js';
 import * as ep___i_exportUserLists from './endpoints/i/export-user-lists.js';
 import * as ep___i_exportAntennas from './endpoints/i/export-antennas.js';
@@ -568,6 +569,7 @@ const eps = [
 	['i/export-following', ep___i_exportFollowing],
 	['i/export-mute', ep___i_exportMute],
 	['i/export-notes', ep___i_exportNotes],
+	['i/export-clips', ep___i_exportClips],
 	['i/export-favorites', ep___i_exportFavorites],
 	['i/export-user-lists', ep___i_exportUserLists],
 	['i/export-antennas', ep___i_exportAntennas],

packages/backend/src/server/api/endpoints/i/export-clips.ts

@@ -0,0 +1,35 @@
+/*
+ * SPDX-FileCopyrightText: syuilo and other misskey contributors
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+import { Injectable } from '@nestjs/common';
+import ms from 'ms';
+import { Endpoint } from '@/server/api/endpoint-base.js';
+import { QueueService } from '@/core/QueueService.js';
+
+export const meta = {
+	secure: true,
+	requireCredential: true,
+	limit: {
+		duration: ms('1day'),
+		max: 1,
+	},
+} as const;
+
+export const paramDef = {
+	type: 'object',
+	properties: {},
+	required: [],
+} as const;
+
+@Injectable()
+export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-disable-line import/no-default-export
+	constructor(
+		private queueService: QueueService,
+	) {
+		super(meta, paramDef, async (ps, me) => {
+			this.queueService.createExportClipsJob(me);
+		});
+	}
+}

packages/backend/test/e2e/exports.ts

@@ -0,0 +1,194 @@
+/*
+ * SPDX-FileCopyrightText: syuilo and other misskey contributors
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+process.env.NODE_ENV = 'test';
+
+import * as assert from 'assert';
+import { signup, api, startServer, startJobQueue, port, post } from '../utils.js';
+import type { INestApplicationContext } from '@nestjs/common';
+import type * as misskey from 'misskey-js';
+
+describe('export-clips', () => {
+	let app: INestApplicationContext;
+	let alice: misskey.entities.SignupResponse;
+	let bob: misskey.entities.SignupResponse;
+
+	// XXX: Any better way to get the result?
+	async function pollFirstDriveFile() {
+		while (true) {
+			const files = (await api('/drive/files', {}, alice)).body;
+			if (!files.length) {
+				await new Promise(r => setTimeout(r, 100));
+				continue;
+			}
+			if (files.length > 1) {
+				throw new Error('Too many files?');
+			}
+			const file = (await api('/drive/files/show', { fileId: files[0].id }, alice)).body;
+			const res = await fetch(new URL(new URL(file.url).pathname, `http://127.0.0.1:${port}`));
+			return await res.json();
+		}
+	}
+
+	beforeAll(async () => {
+		app = await startServer();
+		await startJobQueue();
+		alice = await signup({ username: 'alice' });
+		bob = await signup({ username: 'bob' });
+	}, 1000 * 60 * 2);
+
+	afterAll(async () => {
+		await app.close();
+	});
+
+	beforeEach(async () => {
+		// Clean all clips and files of alice
+		const clips = (await api('/clips/list', {}, alice)).body;
+		for (const clip of clips) {
+			const res = await api('/clips/delete', { clipId: clip.id }, alice);
+			if (res.status !== 204) {
+				throw new Error('Failed to delete clip');
+			}
+		}
+		const files = (await api('/drive/files', {}, alice)).body;
+		for (const file of files) {
+			const res = await api('/drive/files/delete', { fileId: file.id }, alice);
+			if (res.status !== 204) {
+				throw new Error('Failed to delete file');
+			}
+		}
+	});
+
+	test('basic export', async () => {
+		let res = await api('/clips/create', {
+			name: 'foo',
+			description: 'bar',
+		}, alice);
+		assert.strictEqual(res.status, 200);
+
+		res = await api('/i/export-clips', {}, alice);
+		assert.strictEqual(res.status, 204);
+
+		const exported = await pollFirstDriveFile();
+		assert.strictEqual(exported[0].name, 'foo');
+		assert.strictEqual(exported[0].description, 'bar');
+		assert.strictEqual(exported[0].clipNotes.length, 0);
+	});
+
+	test('export with notes', async () => {
+		let res = await api('/clips/create', {
+			name: 'foo',
+			description: 'bar',
+		}, alice);
+		assert.strictEqual(res.status, 200);
+		const clip = res.body;
+
+		const note1 = await post(alice, {
+			text: 'baz1',
+		});
+
+		const note2 = await post(alice, {
+			text: 'baz2',
+			poll: {
+				choices: ['sakura', 'izumi', 'ako'],
+			},
+		});
+
+		for (const note of [note1, note2]) {
+			res = await api('/clips/add-note', {
+				clipId: clip.id,
+				noteId: note.id,
+			}, alice);
+			assert.strictEqual(res.status, 204);
+		}
+
+		res = await api('/i/export-clips', {}, alice);
+		assert.strictEqual(res.status, 204);
+
+		const exported = await pollFirstDriveFile();
+		assert.strictEqual(exported[0].name, 'foo');
+		assert.strictEqual(exported[0].description, 'bar');
+		assert.strictEqual(exported[0].clipNotes.length, 2);
+		assert.strictEqual(exported[0].clipNotes[0].note.text, 'baz1');
+		assert.strictEqual(exported[0].clipNotes[1].note.text, 'baz2');
+		assert.deepStrictEqual(exported[0].clipNotes[1].note.poll.choices[0], 'sakura');
+	});
+
+	test('multiple clips', async () => {
+		let res = await api('/clips/create', {
+			name: 'kawaii',
+			description: 'kawaii',
+		}, alice);
+		assert.strictEqual(res.status, 200);
+		const clip1 = res.body;
+
+		res = await api('/clips/create', {
+			name: 'yuri',
+			description: 'yuri',
+		}, alice);
+		assert.strictEqual(res.status, 200);
+		const clip2 = res.body;
+
+		const note1 = await post(alice, {
+			text: 'baz1',
+		});
+
+		const note2 = await post(alice, {
+			text: 'baz2',
+		});
+
+		res = await api('/clips/add-note', {
+			clipId: clip1.id,
+			noteId: note1.id,
+		}, alice);
+		assert.strictEqual(res.status, 204);
+
+		res = await api('/clips/add-note', {
+			clipId: clip2.id,
+			noteId: note2.id,
+		}, alice);
+		assert.strictEqual(res.status, 204);
+
+		res = await api('/i/export-clips', {}, alice);
+		assert.strictEqual(res.status, 204);
+
+		const exported = await pollFirstDriveFile();
+		assert.strictEqual(exported[0].name, 'kawaii');
+		assert.strictEqual(exported[0].clipNotes.length, 1);
+		assert.strictEqual(exported[0].clipNotes[0].note.text, 'baz1');
+		assert.strictEqual(exported[1].name, 'yuri');
+		assert.strictEqual(exported[1].clipNotes.length, 1);
+		assert.strictEqual(exported[1].clipNotes[0].note.text, 'baz2');
+	});
+
+	test('Clipping other user\'s note', async () => {
+		let res = await api('/clips/create', {
+			name: 'kawaii',
+			description: 'kawaii',
+		}, alice);
+		assert.strictEqual(res.status, 200);
+		const clip = res.body;
+
+		const note = await post(bob, {
+			text: 'baz',
+			visibility: 'followers',
+		});
+
+		res = await api('/clips/add-note', {
+			clipId: clip.id,
+			noteId: note.id,
+		}, alice);
+		assert.strictEqual(res.status, 204);
+
+		res = await api('/i/export-clips', {}, alice);
+		assert.strictEqual(res.status, 204);
+
+		const exported = await pollFirstDriveFile();
+		assert.strictEqual(exported[0].name, 'kawaii');
+		assert.strictEqual(exported[0].clipNotes.length, 1);
+		assert.strictEqual(exported[0].clipNotes[0].note.text, 'baz');
+		assert.strictEqual(exported[0].clipNotes[0].note.user.username, 'bob');
+	});
+});

packages/backend/test/utils.ts

@@ -17,7 +17,7 @@ import { entities } from '../src/postgres.js';
 import { loadConfig } from '../src/config.js';
 import type * as misskey from 'misskey-js';
 
-export { server as startServer } from '@/boot/common.js';
+export { server as startServer, jobQueue as startJobQueue } from '@/boot/common.js';
 
 interface UserToken {
 	token: string;

packages/frontend/src/pages/settings/import-export.vue

@@ -21,6 +21,14 @@ SPDX-License-Identifier: AGPL-3.0-only
 			<MkButton primary :class="$style.button" inline @click="exportFavorites()"><i class="ti ti-download"></i> {{ i18n.ts.export }}</MkButton>
 		</MkFolder>
 	</FormSection>
+	<FormSection>
+		<template #label><i class="ti ti-star"></i> {{ i18n.ts._exportOrImport.clips }}</template>
+		<MkFolder>
+			<template #label>{{ i18n.ts.export }}</template>
+			<template #icon><i class="ti ti-download"></i></template>
+			<MkButton primary :class="$style.button" inline @click="exportClips()"><i class="ti ti-download"></i> {{ i18n.ts.export }}</MkButton>
+		</MkFolder>
+	</FormSection>
 	<FormSection>
 		<template #label><i class="ti ti-users"></i> {{ i18n.ts._exportOrImport.followingList }}</template>
 		<div class="_gaps_s">
@@ -157,6 +165,10 @@ const exportFavorites = () => {
 	misskeyApi('i/export-favorites', {}).then(onExportSuccess).catch(onError);
 };
 
+const exportClips = () => {
+	misskeyApi('i/export-clips', {}).then(onExportSuccess).catch(onError);
+};
+
 const exportFollowing = () => {
 	misskeyApi('i/export-following', {
 		excludeMuting: excludeMutingUsers.value,