feat: Add Badge Image to Push Notification (#8012)

* fix

* nanka iroiro

* wip

* wip

* fix lint

* fix loginId

* fix

* refactor

* refactor

* remove follow action

* clean up

* Revert "remove follow action"

This reverts commit defbb41648.

* Revert "clean up"

This reverts commit f94919cb9c.

* remove fetch specification

* renoteの条件追加

* apiFetch => cli

* bypass fetch?

* fix

* refactor: use path alias

* temp: add submodule

* remove submodule

* enhane: unison-reloadに指定したパスに移動できるように

* null

* null

* feat: ログインするアカウントのIDをクエリ文字列で指定する機能

* null

* await?

* rename

* rename

* Update read.ts

* merge

* get-note-summary

* fix

* swパッケージに

* add missing packages

* fix getNoteSummary

* add webpack-cli

* ✌️

* remove plugins

* sw-inject分離したがテストしてない

* fix notification.vue

* remove a blank line

* disconnect intersection observer

* disconnect2

* fix notification.vue

* remove a blank line

* disconnect intersection observer

* disconnect2

* fix

* ✌️

* clean up config

* typesを戻した

* backend/src/web/index.ts

* notification-badges

* add scripts

* change create-notification.ts

* Update packages/client/src/components/notification.vue

Co-authored-by: Acid Chicken (硫酸鶏) <root@acid-chicken.com>

* disconnect

* oops

* Failed to load the script unexpectedly回避
sw.jsとlib.tsを分離してみた

* truncate notification

* Update packages/client/src/ui/_common_/common.vue

Co-authored-by: syuilo <Syuilotan@yahoo.co.jp>

* clean up

* clean up

* refactor

* キャッシュ対策

* Truncate push notification message

* fix

* クライアントがあったらストリームに接続しているということなので通知しない判定の位置を修正

* components/drive-file-thumbnail.vue

* components/drive-select-dialog.vue

* components/drive-window.vue

* merge

* fix

* Service Workerのビルドにesbuildを使うようにする

* return createEmptyNotification()

* fix

* fix

* i18n.ts

* update

* ✌️

* remove ts-loader

* fix

* fix

* enhance: Service Workerを常に登録するように

* pollEnded

* pollEnded

* URLをsw.jsに戻す

* clean up

* fix lint

* changelog

* alpha-test

* also with twemoji

* add isMimeImage function

* catch

* Colour => Color

* char2file => char2filePath

* Update autocomplete.vue

* remove clone?

Co-authored-by: Acid Chicken (硫酸鶏) <root@acid-chicken.com>
Co-authored-by: syuilo <Syuilotan@yahoo.co.jp>

CHANGELOG.md

@@ -15,6 +15,7 @@ You should also include the user name that made the change.
 - Server: Add rate limit to i/notifications @tamaina
 - Client: Improve files page of control panel @syuilo
 - Improve player detection in URL preview @mei23
+- Add Badge Image to Push Notification #8012 @tamaina
 
 ### Bugfixes
 - Server: Fix GenerateVideoThumbnail failed @mei23

packages/backend/assets/notification-badges/LICENSE

@@ -0,0 +1,5 @@
+Font Awesome Icons
+-------------------------
+
+Ⓒ Font Awesome
+CC BY 4.0 (https://creativecommons.org/licenses/by/4.0/)

packages/backend/src/misc/is-mime-image.ts

@@ -0,0 +1,8 @@
+import { FILE_TYPE_BROWSERSAFE } from '@/const.js';
+
+const dictionary = {
+	'safe-file': FILE_TYPE_BROWSERSAFE,
+	'sharp-convertible-image': ['image/jpeg', 'image/png', 'image/gif', 'image/apng', 'image/vnd.mozilla.apng', 'image/webp', 'image/svg+xml'],
+};
+
+export const isMimeImage = (mime: string, type: keyof typeof dictionary): boolean => dictionary[type].includes(mime);

packages/backend/src/server/proxy/proxy-media.ts

@@ -1,13 +1,16 @@
 import * as fs from 'node:fs';
 import Koa from 'koa';
-import { serverLogger } from '../index.js';
+import sharp from 'sharp';
 import { IImage, convertToWebp } from '@/services/drive/image-processor.js';
 import { createTemp } from '@/misc/create-temp.js';
 import { downloadUrl } from '@/misc/download-url.js';
 import { detectType } from '@/misc/get-file-info.js';
 import { StatusError } from '@/misc/fetch.js';
 import { FILE_TYPE_BROWSERSAFE } from '@/const.js';
+import { serverLogger } from '../index.js';
+import { isMimeImage } from '@/misc/is-mime-image.js';
 
+// eslint-disable-next-line @typescript-eslint/explicit-function-return-type
 export async function proxyMedia(ctx: Koa.Context) {
 	const url = 'url' in ctx.query ? ctx.query.url : 'https://' + ctx.params.url;
 
@@ -23,14 +26,50 @@ export async function proxyMedia(ctx: Koa.Context) {
 		await downloadUrl(url, path);
 
 		const { mime, ext } = await detectType(path);
+		const isConvertibleImage = isMimeImage(mime, 'sharp-convertible-image');
 
 		let image: IImage;
 
-		if ('static' in ctx.query && ['image/png', 'image/gif', 'image/apng', 'image/vnd.mozilla.apng', 'image/webp', 'image/svg+xml'].includes(mime)) {
+		if ('static' in ctx.query && isConvertibleImage) {
 			image = await convertToWebp(path, 498, 280);
-		} else if ('preview' in ctx.query && ['image/jpeg', 'image/png', 'image/gif', 'image/apng', 'image/vnd.mozilla.apng', 'image/svg+xml'].includes(mime)) {
+		} else if ('preview' in ctx.query && isConvertibleImage) {
 			image = await convertToWebp(path, 200, 200);
-		}	else if (['image/svg+xml'].includes(mime)) {
+		} else if ('badge' in ctx.query) {
+			if (!isConvertibleImage) {
+				// 画像でないなら404でお茶を濁す
+				throw new StatusError('Unexpected mime', 404);
+			}
+
+			const mask = sharp(path)
+				.resize(96, 96, {
+					fit: 'inside',
+					withoutEnlargement: false,
+				})
+				.greyscale()
+				.normalise()
+				.linear(1.75, -(128 * 1.75) + 128) // 1.75x contrast
+				.flatten({ background: '#000' })
+				.toColorspace('b-w');
+
+			const stats = await mask.clone().stats();
+
+			if (stats.entropy < 0.1) {
+				// エントロピーがあまりない場合は404にする
+				throw new StatusError('Skip to provide badge', 404);
+			}
+
+			const data = sharp({
+				create: { width: 96, height: 96, channels: 4, background: { r: 0, g: 0, b: 0, alpha: 0 } },
+			})
+				.pipelineColorspace('b-w')
+				.boolean(await mask.png().toBuffer(), 'eor');
+
+			image = {
+				data: await data.png().toBuffer(),
+				ext: 'png',
+				type: 'image/png',
+			};
+		}	else if (mime === 'image/svg+xml') {
 			image = await convertToWebp(path, 2048, 2048, 1);
 		} else if (!mime.startsWith('image/') || !FILE_TYPE_BROWSERSAFE.includes(mime)) {
 			throw new StatusError('Rejected type', 403, 'Rejected type');
@@ -48,7 +87,7 @@ export async function proxyMedia(ctx: Koa.Context) {
 	} catch (e) {
 		serverLogger.error(`${e}`);
 
-		if (e instanceof StatusError && e.isClientError) {
+		if (e instanceof StatusError && (e.statusCode === 302 || e.isClientError)) {
 			ctx.status = e.statusCode;
 		} else {
 			ctx.status = 500;

packages/backend/src/server/web/index.ts

@@ -11,6 +11,7 @@ import Router from '@koa/router';
 import send from 'koa-send';
 import favicon from 'koa-favicon';
 import views from 'koa-views';
+import sharp from 'sharp';
 import { createBullBoard } from '@bull-board/api';
 import { BullAdapter } from '@bull-board/api/bullAdapter.js';
 import { KoaAdapter } from '@bull-board/koa';
@@ -140,6 +141,49 @@ router.get('/twemoji/(.*)', async ctx => {
 	});
 });
 
+router.get('/twemoji-badge/(.*)', async ctx => {
+	const path = ctx.path.replace('/twemoji-badge/', '');
+
+	if (!path.match(/^[0-9a-f-]+\.png$/)) {
+		ctx.status = 404;
+		return;
+	}
+
+	const mask = await sharp(
+		`${_dirname}/../../../node_modules/@discordapp/twemoji/dist/svg/${path.replace('.png', '')}.svg`,
+		{ density: 1000 },
+	)
+		.resize(488, 488)
+		.greyscale()
+		.normalise()
+		.linear(1.75, -(128 * 1.75) + 128) // 1.75x contrast
+		.flatten({ background: '#000' })
+		.extend({
+			top: 12,
+			bottom: 12,
+			left: 12,
+			right: 12,
+			background: '#000',
+		})
+		.toColorspace('b-w')
+		.png()
+		.toBuffer();
+
+	const buffer = await sharp({
+		create: { width: 512, height: 512, channels: 4, background: { r: 0, g: 0, b: 0, alpha: 0 } },
+	})
+		.pipelineColorspace('b-w')
+		.boolean(mask, 'eor')
+		.resize(96, 96)
+		.png()
+		.toBuffer();
+
+	ctx.set('Content-Security-Policy', 'default-src \'none\'; style-src \'unsafe-inline\'');
+	ctx.set('Cache-Control', 'max-age=2592000');
+	ctx.set('Content-Type', 'image/png');
+	ctx.body = buffer;
+});
+
 // ServiceWorker
 router.get(`/sw.js`, async ctx => {
 	await send(ctx as any, `/sw.js`, {

packages/client/src/components/autocomplete.vue

@@ -35,6 +35,7 @@
 <script lang="ts">
 import { markRaw, ref, onUpdated, onMounted, onBeforeUnmount, nextTick, watch } from 'vue';
 import contains from '@/scripts/contains';
+import { char2filePath } from '@/scripts/twemoji-base';
 import { getStaticImageUrl } from '@/scripts/get-static-image-url';
 import { acct } from '@/filters/user';
 import * as os from '@/os';
@@ -42,7 +43,6 @@ import { MFM_TAGS } from '@/scripts/mfm-tags';
 import { defaultStore } from '@/store';
 import { emojilist } from '@/scripts/emojilist';
 import { instance } from '@/instance';
-import { twemojiSvgBase } from '@/scripts/twemoji-base';
 import { i18n } from '@/i18n';
 
 type EmojiDef = {
@@ -55,16 +55,10 @@ type EmojiDef = {
 
 const lib = emojilist.filter(x => x.category !== 'flags');
 
-const char2file = (char: string) => {
-	let codes = Array.from(char).map(x => x.codePointAt(0)?.toString(16));
-	if (!codes.includes('200d')) codes = codes.filter(x => x !== 'fe0f');
-	return codes.filter(x => x && x.length).join('-');
-};
-
 const emjdb: EmojiDef[] = lib.map(x => ({
 	emoji: x.char,
 	name: x.name,
-	url: `${twemojiSvgBase}/${char2file(x.char)}.svg`
+	url: char2filePath(x.char),
 }));
 
 for (const x of lib) {
@@ -74,7 +68,7 @@ for (const x of lib) {
 				emoji: x.char,
 				name: k,
 				aliasOf: x.name,
-				url: `${twemojiSvgBase}/${char2file(x.char)}.svg`
+				url: char2filePath(x.char),
 			});
 		}
 	}

packages/client/src/components/global/emoji.vue

@@ -1,4 +1,4 @@
-<template>
+char2filePath<template>
 <img v-if="customEmoji" class="mk-emoji custom" :class="{ normal, noStyle }" :src="url" :alt="alt" :title="alt" decoding="async"/>
 <img v-else-if="char && !useOsNativeEmojis" class="mk-emoji" :src="url" :alt="alt" :title="alt" decoding="async"/>
 <span v-else-if="char && useOsNativeEmojis">{{ char }}</span>
@@ -8,7 +8,7 @@
 <script lang="ts">
 import { computed, defineComponent, ref, watch } from 'vue';
 import { getStaticImageUrl } from '@/scripts/get-static-image-url';
-import { twemojiSvgBase } from '@/scripts/twemoji-base';
+import { char2filePath } from '@/scripts/twemoji-base';
 import { defaultStore } from '@/store';
 import { instance } from '@/instance';
 
@@ -45,10 +45,7 @@ export default defineComponent({
 		const customEmoji = computed(() => isCustom.value ? ce.value.find(x => x.name === props.emoji.substr(1, props.emoji.length - 2)) : null);
 		const url = computed(() => {
 			if (char.value) {
-				let codes = Array.from(char.value).map(x => x.codePointAt(0).toString(16));
+				return char2filePath(char.value);
-				if (!codes.includes('200d')) codes = codes.filter(x => x !== 'fe0f');
-				codes = codes.filter(x => x && x.length);
-				return `${twemojiSvgBase}/${codes.join('-')}.svg`;
 			} else {
 				return defaultStore.state.disableShowingAnimatedImages
 					? getStaticImageUrl(customEmoji.value.url)

packages/client/src/scripts/twemoji-base.ts

@@ -1 +1,12 @@
 export const twemojiSvgBase = '/twemoji';
+
+export function char2fileName(char: string): string {
+	let codes = Array.from(char).map(x => x.codePointAt(0)?.toString(16));
+	if (!codes.includes('200d')) codes = codes.filter(x => x !== 'fe0f');
+	codes = codes.filter(x => x && x.length);
+	return codes.join('-');
+}
+
+export function char2filePath(char: string): string {
+	return `${twemojiSvgBase}/${char2fileName(char)}.svg`;
+}

packages/sw/src/scripts/create-notification.ts

@@ -9,6 +9,10 @@ import { pushNotificationDataMap } from '@/types';
 import getUserName from '@/scripts/get-user-name';
 import { I18n } from '@/scripts/i18n';
 import { getAccountFromId } from '@/scripts/get-account-from-id';
+import { char2fileName } from '@/scripts/twemoji-base';
+import * as url from '@/scripts/url';
+
+const iconUrl = (name: string) => `/static-assets/notification-badges/${name}.png`;
 
 export async function createNotification<K extends keyof pushNotificationDataMap>(data: pushNotificationDataMap[K]) {
 	const n = await composeNotification(data);
@@ -44,6 +48,7 @@ async function composeNotification<K extends keyof pushNotificationDataMap>(data
 					return [t('_notification.youWereFollowed'), {
 						body: getUserName(data.body.user),
 						icon: data.body.user.avatarUrl,
+						badge: iconUrl('user-plus'),
 						data,
 						actions: userDetail.isFollowing ? [] : [
 							{
@@ -57,6 +62,7 @@ async function composeNotification<K extends keyof pushNotificationDataMap>(data
 					return [t('_notification.youGotMention', { name: getUserName(data.body.user) }), {
 						body: data.body.note.text || '',
 						icon: data.body.user.avatarUrl,
+						badge: iconUrl('at'),
 						data,
 						actions: [
 							{
@@ -70,6 +76,7 @@ async function composeNotification<K extends keyof pushNotificationDataMap>(data
 					return [t('_notification.youGotReply', { name: getUserName(data.body.user) }), {
 						body: data.body.note.text || '',
 						icon: data.body.user.avatarUrl,
+						badge: iconUrl('reply'),
 						data,
 						actions: [
 							{
@@ -83,6 +90,7 @@ async function composeNotification<K extends keyof pushNotificationDataMap>(data
 					return [t('_notification.youRenoted', { name: getUserName(data.body.user) }), {
 						body: data.body.note.text || '',
 						icon: data.body.user.avatarUrl,
+						badge: iconUrl('retweet'),
 						data,
 						actions: [
 							{
@@ -96,6 +104,7 @@ async function composeNotification<K extends keyof pushNotificationDataMap>(data
 					return [t('_notification.youGotQuote', { name: getUserName(data.body.user) }), {
 						body: data.body.note.text || '',
 						icon: data.body.user.avatarUrl,
+						badge: iconUrl('quote-right'),
 						data,
 						actions: [
 							{
@@ -112,9 +121,44 @@ async function composeNotification<K extends keyof pushNotificationDataMap>(data
 					}];
 
 				case 'reaction':
-					return [`${data.body.reaction} ${getUserName(data.body.user)}`, {
+					let reaction = data.body.reaction;
+					let badge: string | undefined;
+
+					if (reaction.startsWith(':')) {
+						// カスタム絵文字の場合
+						const customEmoji = data.body.note.emojis.find(x => x.name === reaction.substr(1, reaction.length - 2));
+						if (customEmoji) {
+							if (reaction.includes('@')) {
+								reaction = `:${reaction.substr(1, reaction.indexOf('@') - 1)}:`;
+							}
+
+							const u = new URL(customEmoji.url);
+							if (u.href.startsWith(`${origin}/proxy/`)) {
+								// もう既にproxyっぽそうだったらsearchParams付けるだけ
+								u.searchParams.set('badge', '1');
+								badge = u.href;
+							} else {
+								const dummy = `${u.host}${u.pathname}`;	// 拡張子がないとキャッシュしてくれないCDNがあるので
+								badge = `${origin}/proxy/${dummy}?${url.query({
+									url: u.href,
+									badge: '1'
+								})}`;
+							}
+						}
+					} else {
+						// Unicode絵文字の場合
+						badge = `/twemoji-badge/${char2fileName(reaction)}.png`;
+					}
+
+
+					if (badge ? await fetch(badge).then(res => res.status !== 200).catch(() => true) : true) {
+						badge = iconUrl('plus');
+					}
+
+					return [`${reaction} ${getUserName(data.body.user)}`, {
 						body: data.body.note.text || '',
 						icon: data.body.user.avatarUrl,
+						badge,
 						data,
 						actions: [
 							{
@@ -128,12 +172,14 @@ async function composeNotification<K extends keyof pushNotificationDataMap>(data
 					return [t('_notification.youGotPoll', { name: getUserName(data.body.user) }), {
 						body: data.body.note.text || '',
 						icon: data.body.user.avatarUrl,
+						badge: iconUrl('poll-h'),
 						data,
 					}];
 
 				case 'pollEnded':
 					return [t('_notification.pollEnded'), {
 						body: data.body.note.text || '',
+						badge: iconUrl('clipboard-check-solid'),
 						data,
 					}];
 
@@ -141,6 +187,7 @@ async function composeNotification<K extends keyof pushNotificationDataMap>(data
 					return [t('_notification.youReceivedFollowRequest'), {
 						body: getUserName(data.body.user),
 						icon: data.body.user.avatarUrl,
+						badge: iconUrl('clock'),
 						data,
 						actions: [
 							{
@@ -158,12 +205,14 @@ async function composeNotification<K extends keyof pushNotificationDataMap>(data
 					return [t('_notification.yourFollowRequestAccepted'), {
 						body: getUserName(data.body.user),
 						icon: data.body.user.avatarUrl,
+						badge: iconUrl('check'),
 						data,
 					}];
 
 				case 'groupInvited':
 					return [t('_notification.youWereInvitedToGroup', { userName: getUserName(data.body.user) }), {
 						body: data.body.invitation.group.name,
+						badge: iconUrl('id-card-alt'),
 						data,
 						actions: [
 							{
@@ -191,6 +240,7 @@ async function composeNotification<K extends keyof pushNotificationDataMap>(data
 			if (data.body.groupId === null) {
 				return [t('_notification.youGotMessagingMessageFromUser', { name: getUserName(data.body.user) }), {
 					icon: data.body.user.avatarUrl,
+					badge: iconUrl('comments'),
 					tag: `messaging:user:${data.body.userId}`,
 					data,
 					renotify: true,
@@ -198,6 +248,7 @@ async function composeNotification<K extends keyof pushNotificationDataMap>(data
 			}
 			return [t('_notification.youGotMessagingMessageFromGroup', { name: data.body.group.name }), {
 				icon: data.body.user.avatarUrl,
+				badge: iconUrl('comments'),
 				tag: `messaging:group:${data.body.groupId}`,
 				data,
 				renotify: true,
@@ -217,6 +268,7 @@ export async function createEmptyNotification() {
 			t('_notification.emptyPushNotificationMessage'),
 			{
 				silent: true,
+				badge: iconUrl('null'),
 				tag: 'read_notification',
 			}
 		);

packages/sw/src/scripts/twemoji-base.ts

@@ -0,0 +1,12 @@
+export const twemojiSvgBase = '/twemoji';
+
+export function char2fileName(char: string): string {
+	let codes = Array.from(char).map(x => x.codePointAt(0)?.toString(16));
+	if (!codes.includes('200d')) codes = codes.filter(x => x !== 'fe0f');
+	codes = codes.filter(x => x && x.length);
+	return codes.join('-');
+}
+
+export function char2filePath(char: string): string {
+	return `${twemojiSvgBase}/${char2fileName(char)}.svg`;
+}

packages/sw/src/scripts/url.ts

@@ -0,0 +1,13 @@
+export function query(obj: {}): string {
+	const params = Object.entries(obj)
+		.filter(([, v]) => Array.isArray(v) ? v.length : v !== undefined)
+		.reduce((a, [k, v]) => (a[k] = v, a), {} as Record<string, any>);
+
+	return Object.entries(params)
+		.map((e) => `${e[0]}=${encodeURIComponent(e[1])}`)
+		.join('&');
+}
+
+export function appendQuery(url: string, query: string): string {
+	return `${url}${/\?/.test(url) ? url.endsWith('?') ? '' : '&' : '?'}${query}`;
+}