usbharu
/
misskey
mirror of https://github.com/misskey-dev/misskey.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

wip

This commit is contained in:
syuilo 2025-05-30 15:00:52 +09:00
parent 1e9faff0db
commit 0e37048e1e
4 changed files with 142 additions and 142 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

155
packages/frontend/src/utility/image-effector/ImageEffector.ts
View File

@ -3,7 +3,7 @@
* SPDX-License-Identifier: AGPL-3.0-only * SPDX-License-Identifier: AGPL-3.0-only
*/ */
import { createTexture } from './utilts.js'; import { getProxiedImageUrl } from '../media-proxy.js';
type ParamTypeToPrimitive = { type ParamTypeToPrimitive = {
'number': number; 'number': number;
@ -11,7 +11,7 @@ type ParamTypeToPrimitive = {
'boolean': boolean; 'boolean': boolean;
'align': { x: 'left' | 'center' | 'right'; y: 'top' | 'center' | 'bottom'; }; 'align': { x: 'left' | 'center' | 'right'; y: 'top' | 'center' | 'bottom'; };
'seed': number; 'seed': number;
'texture': string | null; 'texture': { type: 'text'; text: string | null; } | { type: 'url'; url: string | null; } | null;
}; };
type ImageEffectorFxParamDefs = Record<string, { type ImageEffectorFxParamDefs = Record<string, {
@ -19,17 +19,16 @@ type ImageEffectorFxParamDefs = Record<string, {
default: any; default: any;
}>; }>;
export function defineImageEffectorFx<ID extends string, PS extends ImageEffectorFxParamDefs, US extends string[], TS extends string[]>(fx: ImageEffectorFx<ID, PS, US, TS>) { export function defineImageEffectorFx<ID extends string, PS extends ImageEffectorFxParamDefs, US extends string[]>(fx: ImageEffectorFx<ID, PS, US>) {
return fx; return fx;
} }
export type ImageEffectorFx<ID extends string = string, PS extends ImageEffectorFxParamDefs = any, US extends string[] = string[], TS extends string[] = string[]> = { export type ImageEffectorFx<ID extends string = string, PS extends ImageEffectorFxParamDefs = ImageEffectorFxParamDefs, US extends string[] = string[]> = {
id: ID; id: ID;
name: string; name: string;
shader: string; shader: string;
uniforms: US; uniforms: US;
params: PS, params: PS,
textures?: TS;
main: (ctx: { main: (ctx: {
gl: WebGL2RenderingContext; gl: WebGL2RenderingContext;
program: WebGLProgram; program: WebGLProgram;
@ -39,7 +38,7 @@ export type ImageEffectorFx<ID extends string = string, PS extends ImageEffector
u: Record<US[number], WebGLUniformLocation>; u: Record<US[number], WebGLUniformLocation>;
width: number; width: number;
height: number; height: number;
textures: Record<TS[number], { textures: Record<string, {
texture: WebGLTexture; texture: WebGLTexture;
width: number; width: number;
height: number; height: number;
@ -51,13 +50,10 @@ export type ImageEffectorLayer = {
id: string; id: string;
fxId: string; fxId: string;
params: Record<string, any>; params: Record<string, any>;
textures?: Record<string, ExternalTextureId | null>;
}; };
type ExternalTextureId = string;
export class ImageEffector { export class ImageEffector {
public gl: WebGL2RenderingContext; private gl: WebGL2RenderingContext;
private canvas: HTMLCanvasElement | null = null; private canvas: HTMLCanvasElement | null = null;
private renderTextureProgram!: WebGLProgram; private renderTextureProgram!: WebGLProgram;
private renderInvertedTextureProgram!: WebGLProgram; private renderInvertedTextureProgram!: WebGLProgram;
@ -70,7 +66,7 @@ export class ImageEffector {
private perLayerResultTextures: Map<string, WebGLTexture> = new Map(); private perLayerResultTextures: Map<string, WebGLTexture> = new Map();
private perLayerResultFrameBuffers: Map<string, WebGLFramebuffer> = new Map(); private perLayerResultFrameBuffers: Map<string, WebGLFramebuffer> = new Map();
private fxs: ImageEffectorFx[]; private fxs: ImageEffectorFx[];
private externalTextures: Map<ExternalTextureId, { texture: WebGLTexture; width: number; height: number; }> = new Map(); private paramTextures: Map<string, { texture: WebGLTexture; width: number; height: number; }> = new Map();
constructor(options: { constructor(options: {
canvas: HTMLCanvasElement; canvas: HTMLCanvasElement;
@ -239,11 +235,12 @@ export class ImageEffector {
width: this.renderWidth, width: this.renderWidth,
height: this.renderHeight, height: this.renderHeight,
textures: Object.fromEntries( textures: Object.fromEntries(
Object.entries(layer.textures ?? {}).map(([key, textureId]) => { Object.entries(fx.params).map(([k, v]) => {
if (textureId == null) return [key, null]; if (v.type !== 'texture') return [k, null];
const externalTexture = this.externalTextures.get(textureId); const param = layer.params[k];
if (externalTexture == null) return [key, null]; if (param == null) return [k, null];
return [key, externalTexture]; const texture = this.paramTextures.get(this.getTextureKeyForParam(param));
return [k, texture];
})), })),
}); });
@ -311,18 +308,44 @@ export class ImageEffector {
public async setLayers(layers: ImageEffectorLayer[]) { public async setLayers(layers: ImageEffectorLayer[]) {
this.layers = layers; this.layers = layers;
const unused = new Set(this.paramTextures.keys());
for (const layer of layers) {
const fx = this.fxs.find(fx => fx.id === layer.fxId);
if (fx == null) continue;
for (const [k, v] of Object.entries(layer.params)) {
const paramDef = fx.params[k];
if (paramDef == null) continue;
if (paramDef.type !== 'texture') continue;
if (v == null) continue;
const textureKey = this.getTextureKeyForParam(v);
unused.delete(textureKey);
if (this.paramTextures.has(textureKey)) continue;
console.log(`Baking texture of <${textureKey}>...`);
const texture = v.type === 'text' ? await createTextureFromText(this.gl, v.text) : v.type === 'url' ? await createTextureFromUrl(this.gl, v.url) : null;
if (texture == null) continue;
this.paramTextures.set(textureKey, texture);
}
}
for (const k of unused) {
console.log(`Dispose unused texture <${k}>...`);
this.gl.deleteTexture(this.paramTextures.get(k)!.texture);
this.paramTextures.delete(k);
}
this.render(); this.render();
} }
public registerExternalTexture(id: string, texture: WebGLTexture, width: number, height: number) { private getTextureKeyForParam(v: ParamTypeToPrimitive['texture']) {
this.externalTextures.set(id, { texture, width, height }); if (v == null) return '';
} return v.type === 'text' ? `text:${v.text}` : v.type === 'url' ? `url:${v.url}` : '';
public disposeExternalTextures() {
for (const bakedTexture of this.externalTextures.values()) {
this.gl.deleteTexture(bakedTexture.texture);
}
this.externalTextures.clear();
} }
public destroy() { public destroy() {
@ -341,9 +364,89 @@ export class ImageEffector {
} }
this.perLayerResultFrameBuffers.clear(); this.perLayerResultFrameBuffers.clear();
this.disposeExternalTextures(); for (const texture of this.paramTextures.values()) {
this.gl.deleteTexture(texture.texture);
}
this.paramTextures.clear();
this.gl.deleteProgram(this.renderTextureProgram); this.gl.deleteProgram(this.renderTextureProgram);
this.gl.deleteProgram(this.renderInvertedTextureProgram); this.gl.deleteProgram(this.renderInvertedTextureProgram);
this.gl.deleteTexture(this.originalImageTexture); this.gl.deleteTexture(this.originalImageTexture);
} }
} }
function createTexture(gl: WebGL2RenderingContext): WebGLTexture {
const texture = gl.createTexture();
gl.bindTexture(gl.TEXTURE_2D, texture);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.REPEAT);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.REPEAT);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
gl.bindTexture(gl.TEXTURE_2D, null);
return texture;
}
async function createTextureFromUrl(gl: WebGL2RenderingContext, imageUrl: string | null): Promise<{ texture: WebGLTexture, width: number, height: number } | null> {
if (imageUrl == null || imageUrl.trim() === '') return null;
const image = await new Promise<HTMLImageElement>((resolve, reject) => {
const img = new Image();
img.onload = () => resolve(img);
img.onerror = reject;
img.src = getProxiedImageUrl(imageUrl); // CORS対策
});
const texture = createTexture(gl);
gl.activeTexture(gl.TEXTURE0);
gl.bindTexture(gl.TEXTURE_2D, texture);
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, image.width, image.height, 0, gl.RGBA, gl.UNSIGNED_BYTE, image);
gl.bindTexture(gl.TEXTURE_2D, null);
return {
texture,
width: image.width,
height: image.height,
};
}
async function createTextureFromText(gl: WebGL2RenderingContext, text: string | null, resolution = 2048): Promise<{ texture: WebGLTexture, width: number, height: number } | null> {
if (text == null || text.trim() === '') return null;
const ctx = window.document.createElement('canvas').getContext('2d')!;
ctx.canvas.width = resolution;
ctx.canvas.height = resolution / 4;
const fontSize = resolution / 32;
const margin = fontSize / 2;
ctx.shadowColor = '#000000';
ctx.shadowBlur = fontSize / 4;
//ctx.fillStyle = '#00ff00';
//ctx.fillRect(0, 0, ctx.canvas.width, ctx.canvas.height);
ctx.fillStyle = '#ffffff';
ctx.font = `bold ${fontSize}px sans-serif`;
ctx.textBaseline = 'middle';
ctx.fillText(text, margin, ctx.canvas.height / 2);
const textMetrics = ctx.measureText(text);
const cropWidth = (Math.ceil(textMetrics.actualBoundingBoxRight + textMetrics.actualBoundingBoxLeft) + margin + margin);
const cropHeight = (Math.ceil(textMetrics.actualBoundingBoxAscent + textMetrics.actualBoundingBoxDescent) + margin + margin);
const data = ctx.getImageData(0, (ctx.canvas.height / 2) - (cropHeight / 2), ctx.canvas.width, ctx.canvas.height);
const texture = createTexture(gl);
gl.activeTexture(gl.TEXTURE0);
gl.bindTexture(gl.TEXTURE_2D, texture);
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, cropWidth, cropHeight, 0, gl.RGBA, gl.UNSIGNED_BYTE, data);
gl.bindTexture(gl.TEXTURE_2D, null);
const info = {
texture: texture,
width: cropWidth,
height: cropHeight,
};
ctx.canvas.remove();
return info;
}

5
packages/frontend/src/utility/image-effector/fxs/watermarkPlacement.ts
View File

@ -98,8 +98,11 @@ export const FX_watermarkPlacement = defineImageEffectorFx({
max: 1.0, max: 1.0,
step: 0.01, step: 0.01,
}, },
watermark: {
type: 'texture' as const,
default: null,
},
}, },
textures: ['watermark'] as const,
main: ({ gl, u, params, textures }) => { main: ({ gl, u, params, textures }) => {
if (textures.watermark == null) { if (textures.watermark == null) {
return; return;

78
packages/frontend/src/utility/image-effector/utilts.ts
View File

@ -1,78 +0,0 @@
/*
* SPDX-FileCopyrightText: syuilo and misskey-project
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { getProxiedImageUrl } from '../media-proxy.js';
export function createTexture(gl: WebGL2RenderingContext): WebGLTexture {
const texture = gl.createTexture();
gl.bindTexture(gl.TEXTURE_2D, texture);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.REPEAT);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.REPEAT);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
gl.bindTexture(gl.TEXTURE_2D, null);
return texture;
}
export async function createTextureFromUrl(gl: WebGL2RenderingContext, imageUrl: string): Promise<{ texture: WebGLTexture, width: number, height: number }> {
const image = await new Promise<HTMLImageElement>((resolve, reject) => {
const img = new Image();
img.onload = () => resolve(img);
img.onerror = reject;
img.src = getProxiedImageUrl(imageUrl); // CORS対策
});
const texture = createTexture(gl);
gl.activeTexture(gl.TEXTURE0);
gl.bindTexture(gl.TEXTURE_2D, texture);
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, image.width, image.height, 0, gl.RGBA, gl.UNSIGNED_BYTE, image);
gl.bindTexture(gl.TEXTURE_2D, null);
return {
texture,
width: image.width,
height: image.height,
};
}
export async function createTextureFromText(gl: WebGL2RenderingContext, text: string, resolution = 2048): Promise<{ texture: WebGLTexture, width: number, height: number }> {
const ctx = window.document.createElement('canvas').getContext('2d')!;
ctx.canvas.width = resolution;
ctx.canvas.height = resolution / 4;
const fontSize = resolution / 32;
const margin = fontSize / 2;
ctx.shadowColor = '#000000';
ctx.shadowBlur = fontSize / 4;
//ctx.fillStyle = '#00ff00';
//ctx.fillRect(0, 0, ctx.canvas.width, ctx.canvas.height);
ctx.fillStyle = '#ffffff';
ctx.font = `bold ${fontSize}px sans-serif`;
ctx.textBaseline = 'middle';
ctx.fillText(text, margin, ctx.canvas.height / 2);
const textMetrics = ctx.measureText(text);
const cropWidth = (Math.ceil(textMetrics.actualBoundingBoxRight + textMetrics.actualBoundingBoxLeft) + margin + margin);
const cropHeight = (Math.ceil(textMetrics.actualBoundingBoxAscent + textMetrics.actualBoundingBoxDescent) + margin + margin);
const data = ctx.getImageData(0, (ctx.canvas.height / 2) - (cropHeight / 2), ctx.canvas.width, ctx.canvas.height);
const texture = createTexture(gl);
gl.activeTexture(gl.TEXTURE0);
gl.bindTexture(gl.TEXTURE_2D, texture);
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, cropWidth, cropHeight, 0, gl.RGBA, gl.UNSIGNED_BYTE, data);
gl.bindTexture(gl.TEXTURE_2D, null);
const info = {
texture: texture,
width: cropWidth,
height: cropHeight,
};
ctx.canvas.remove();
return info;
}

46
packages/frontend/src/utility/watermark.ts
View File

@ -4,7 +4,6 @@
*/ */
import { FX_watermarkPlacement } from './image-effector/fxs/watermarkPlacement.js'; import { FX_watermarkPlacement } from './image-effector/fxs/watermarkPlacement.js';
import { createTextureFromText, createTextureFromUrl } from './image-effector/utilts.js';
import type { ImageEffectorLayer } from '@/utility/image-effector/ImageEffector.js'; import type { ImageEffectorLayer } from '@/utility/image-effector/ImageEffector.js';
import { ImageEffector } from '@/utility/image-effector/ImageEffector.js'; import { ImageEffector } from '@/utility/image-effector/ImageEffector.js';
@ -35,7 +34,6 @@ export type WatermarkPreset = {
export class WatermarkRenderer { export class WatermarkRenderer {
private effector: ImageEffector; private effector: ImageEffector;
private layers: WatermarkPreset['layers'] = []; private layers: WatermarkPreset['layers'] = [];
private texturesKey = '';
constructor(options: { constructor(options: {
canvas: HTMLCanvasElement, canvas: HTMLCanvasElement,
@ -52,30 +50,6 @@ export class WatermarkRenderer {
}); });
} }
private calcTexturesKey() {
return this.layers.map(layer => {
if (layer.type === 'image' && layer.imageUrl != null) {
return layer.imageUrl;
} else if (layer.type === 'text' && layer.text != null) {
return layer.text;
}
return '';
}).join(';');
}
private async bakeTextures(): Promise<void> {
this.effector.disposeExternalTextures();
for (const layer of this.layers) {
if (layer.type === 'text' && layer.text != null) {
const { texture, width, height } = await createTextureFromText(this.effector.gl, layer.text);
this.effector.registerExternalTexture(layer.id, texture, width, height);
} else if (layer.type === 'image' && layer.imageUrl != null) {
const { texture, width, height } = await createTextureFromUrl(this.effector.gl, layer.imageUrl);
this.effector.registerExternalTexture(layer.id, texture, width, height);
}
}
}
private makeImageEffectorLayers(): ImageEffectorLayer[] { private makeImageEffectorLayers(): ImageEffectorLayer[] {
return this.layers.map(layer => { return this.layers.map(layer => {
if (layer.type === 'text') { if (layer.type === 'text') {
@ -88,8 +62,11 @@ export class WatermarkRenderer {
align: layer.align, align: layer.align,
opacity: layer.opacity, opacity: layer.opacity,
cover: false, cover: false,
watermark: {
type: 'text',
text: layer.text,
},
}, },
textures: { watermark: layer.id },
}; };
} else { } else {
return { return {
@ -101,8 +78,11 @@ export class WatermarkRenderer {
align: layer.align, align: layer.align,
opacity: layer.opacity, opacity: layer.opacity,
cover: layer.cover, cover: layer.cover,
watermark: {
type: 'url',
url: layer.imageUrl,
},
}, },
textures: { watermark: layer.id },
}; };
} }
}); });
@ -110,15 +90,7 @@ export class WatermarkRenderer {
public async setLayers(layers: WatermarkPreset['layers']) { public async setLayers(layers: WatermarkPreset['layers']) {
this.layers = layers; this.layers = layers;
await this.effector.setLayers(this.makeImageEffectorLayers());
const newTexturesKey = this.calcTexturesKey();
if (newTexturesKey !== this.texturesKey) {
this.texturesKey = newTexturesKey;
await this.bakeTextures();
}
this.effector.setLayers(this.makeImageEffectorLayers());
this.render(); this.render();
} }