From 0e37048e1e57e1298b09ef53d839ced0d452ebd6 Mon Sep 17 00:00:00 2001 From: syuilo <4439005+syuilo@users.noreply.github.com> Date: Fri, 30 May 2025 15:00:52 +0900 Subject: [PATCH] wip --- .../utility/image-effector/ImageEffector.ts | 155 +++++++++++++++--- .../image-effector/fxs/watermarkPlacement.ts | 5 +- .../src/utility/image-effector/utilts.ts | 78 --------- packages/frontend/src/utility/watermark.ts | 46 +----- 4 files changed, 142 insertions(+), 142 deletions(-) delete mode 100644 packages/frontend/src/utility/image-effector/utilts.ts diff --git a/packages/frontend/src/utility/image-effector/ImageEffector.ts b/packages/frontend/src/utility/image-effector/ImageEffector.ts index a921b5dcc9..1550f1f0ff 100644 --- a/packages/frontend/src/utility/image-effector/ImageEffector.ts +++ b/packages/frontend/src/utility/image-effector/ImageEffector.ts @@ -3,7 +3,7 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import { createTexture } from './utilts.js'; +import { getProxiedImageUrl } from '../media-proxy.js'; type ParamTypeToPrimitive = { 'number': number; @@ -11,7 +11,7 @@ type ParamTypeToPrimitive = { 'boolean': boolean; 'align': { x: 'left' | 'center' | 'right'; y: 'top' | 'center' | 'bottom'; }; 'seed': number; - 'texture': string | null; + 'texture': { type: 'text'; text: string | null; } | { type: 'url'; url: string | null; } | null; }; type ImageEffectorFxParamDefs = Record; -export function defineImageEffectorFx(fx: ImageEffectorFx) { +export function defineImageEffectorFx(fx: ImageEffectorFx) { return fx; } -export type ImageEffectorFx = { +export type ImageEffectorFx = { id: ID; name: string; shader: string; uniforms: US; params: PS, - textures?: TS; main: (ctx: { gl: WebGL2RenderingContext; program: WebGLProgram; @@ -39,7 +38,7 @@ export type ImageEffectorFx; width: number; height: number; - textures: Record; - textures?: Record; }; -type ExternalTextureId = string; - export class ImageEffector { - public gl: WebGL2RenderingContext; + private gl: WebGL2RenderingContext; private canvas: HTMLCanvasElement | null = null; private renderTextureProgram!: WebGLProgram; private renderInvertedTextureProgram!: WebGLProgram; @@ -70,7 +66,7 @@ export class ImageEffector { private perLayerResultTextures: Map = new Map(); private perLayerResultFrameBuffers: Map = new Map(); private fxs: ImageEffectorFx[]; - private externalTextures: Map = new Map(); + private paramTextures: Map = new Map(); constructor(options: { canvas: HTMLCanvasElement; @@ -239,11 +235,12 @@ export class ImageEffector { width: this.renderWidth, height: this.renderHeight, textures: Object.fromEntries( - Object.entries(layer.textures ?? {}).map(([key, textureId]) => { - if (textureId == null) return [key, null]; - const externalTexture = this.externalTextures.get(textureId); - if (externalTexture == null) return [key, null]; - return [key, externalTexture]; + Object.entries(fx.params).map(([k, v]) => { + if (v.type !== 'texture') return [k, null]; + const param = layer.params[k]; + if (param == null) return [k, null]; + const texture = this.paramTextures.get(this.getTextureKeyForParam(param)); + return [k, texture]; })), }); @@ -311,18 +308,44 @@ export class ImageEffector { public async setLayers(layers: ImageEffectorLayer[]) { this.layers = layers; + + const unused = new Set(this.paramTextures.keys()); + + for (const layer of layers) { + const fx = this.fxs.find(fx => fx.id === layer.fxId); + if (fx == null) continue; + + for (const [k, v] of Object.entries(layer.params)) { + const paramDef = fx.params[k]; + if (paramDef == null) continue; + if (paramDef.type !== 'texture') continue; + if (v == null) continue; + + const textureKey = this.getTextureKeyForParam(v); + unused.delete(textureKey); + if (this.paramTextures.has(textureKey)) continue; + + console.log(`Baking texture of <${textureKey}>...`); + + const texture = v.type === 'text' ? await createTextureFromText(this.gl, v.text) : v.type === 'url' ? await createTextureFromUrl(this.gl, v.url) : null; + if (texture == null) continue; + + this.paramTextures.set(textureKey, texture); + } + } + + for (const k of unused) { + console.log(`Dispose unused texture <${k}>...`); + this.gl.deleteTexture(this.paramTextures.get(k)!.texture); + this.paramTextures.delete(k); + } + this.render(); } - public registerExternalTexture(id: string, texture: WebGLTexture, width: number, height: number) { - this.externalTextures.set(id, { texture, width, height }); - } - - public disposeExternalTextures() { - for (const bakedTexture of this.externalTextures.values()) { - this.gl.deleteTexture(bakedTexture.texture); - } - this.externalTextures.clear(); + private getTextureKeyForParam(v: ParamTypeToPrimitive['texture']) { + if (v == null) return ''; + return v.type === 'text' ? `text:${v.text}` : v.type === 'url' ? `url:${v.url}` : ''; } public destroy() { @@ -341,9 +364,89 @@ export class ImageEffector { } this.perLayerResultFrameBuffers.clear(); - this.disposeExternalTextures(); + for (const texture of this.paramTextures.values()) { + this.gl.deleteTexture(texture.texture); + } + this.paramTextures.clear(); + this.gl.deleteProgram(this.renderTextureProgram); this.gl.deleteProgram(this.renderInvertedTextureProgram); this.gl.deleteTexture(this.originalImageTexture); } } + +function createTexture(gl: WebGL2RenderingContext): WebGLTexture { + const texture = gl.createTexture(); + gl.bindTexture(gl.TEXTURE_2D, texture); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.REPEAT); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.REPEAT); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST); + gl.bindTexture(gl.TEXTURE_2D, null); + return texture; +} + +async function createTextureFromUrl(gl: WebGL2RenderingContext, imageUrl: string | null): Promise<{ texture: WebGLTexture, width: number, height: number } | null> { + if (imageUrl == null || imageUrl.trim() === '') return null; + + const image = await new Promise((resolve, reject) => { + const img = new Image(); + img.onload = () => resolve(img); + img.onerror = reject; + img.src = getProxiedImageUrl(imageUrl); // CORS対策 + }); + + const texture = createTexture(gl); + gl.activeTexture(gl.TEXTURE0); + gl.bindTexture(gl.TEXTURE_2D, texture); + gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, image.width, image.height, 0, gl.RGBA, gl.UNSIGNED_BYTE, image); + gl.bindTexture(gl.TEXTURE_2D, null); + + return { + texture, + width: image.width, + height: image.height, + }; +} + +async function createTextureFromText(gl: WebGL2RenderingContext, text: string | null, resolution = 2048): Promise<{ texture: WebGLTexture, width: number, height: number } | null> { + if (text == null || text.trim() === '') return null; + + const ctx = window.document.createElement('canvas').getContext('2d')!; + ctx.canvas.width = resolution; + ctx.canvas.height = resolution / 4; + const fontSize = resolution / 32; + const margin = fontSize / 2; + ctx.shadowColor = '#000000'; + ctx.shadowBlur = fontSize / 4; + + //ctx.fillStyle = '#00ff00'; + //ctx.fillRect(0, 0, ctx.canvas.width, ctx.canvas.height); + + ctx.fillStyle = '#ffffff'; + ctx.font = `bold ${fontSize}px sans-serif`; + ctx.textBaseline = 'middle'; + + ctx.fillText(text, margin, ctx.canvas.height / 2); + + const textMetrics = ctx.measureText(text); + const cropWidth = (Math.ceil(textMetrics.actualBoundingBoxRight + textMetrics.actualBoundingBoxLeft) + margin + margin); + const cropHeight = (Math.ceil(textMetrics.actualBoundingBoxAscent + textMetrics.actualBoundingBoxDescent) + margin + margin); + const data = ctx.getImageData(0, (ctx.canvas.height / 2) - (cropHeight / 2), ctx.canvas.width, ctx.canvas.height); + + const texture = createTexture(gl); + gl.activeTexture(gl.TEXTURE0); + gl.bindTexture(gl.TEXTURE_2D, texture); + gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, cropWidth, cropHeight, 0, gl.RGBA, gl.UNSIGNED_BYTE, data); + gl.bindTexture(gl.TEXTURE_2D, null); + + const info = { + texture: texture, + width: cropWidth, + height: cropHeight, + }; + + ctx.canvas.remove(); + + return info; +} diff --git a/packages/frontend/src/utility/image-effector/fxs/watermarkPlacement.ts b/packages/frontend/src/utility/image-effector/fxs/watermarkPlacement.ts index 866878f227..c1c5e8edf4 100644 --- a/packages/frontend/src/utility/image-effector/fxs/watermarkPlacement.ts +++ b/packages/frontend/src/utility/image-effector/fxs/watermarkPlacement.ts @@ -98,8 +98,11 @@ export const FX_watermarkPlacement = defineImageEffectorFx({ max: 1.0, step: 0.01, }, + watermark: { + type: 'texture' as const, + default: null, + }, }, - textures: ['watermark'] as const, main: ({ gl, u, params, textures }) => { if (textures.watermark == null) { return; diff --git a/packages/frontend/src/utility/image-effector/utilts.ts b/packages/frontend/src/utility/image-effector/utilts.ts deleted file mode 100644 index 064e4deeb0..0000000000 --- a/packages/frontend/src/utility/image-effector/utilts.ts +++ /dev/null @@ -1,78 +0,0 @@ -/* - * SPDX-FileCopyrightText: syuilo and misskey-project - * SPDX-License-Identifier: AGPL-3.0-only - */ - -import { getProxiedImageUrl } from '../media-proxy.js'; - -export function createTexture(gl: WebGL2RenderingContext): WebGLTexture { - const texture = gl.createTexture(); - gl.bindTexture(gl.TEXTURE_2D, texture); - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.REPEAT); - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.REPEAT); - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST); - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST); - gl.bindTexture(gl.TEXTURE_2D, null); - return texture; -} - -export async function createTextureFromUrl(gl: WebGL2RenderingContext, imageUrl: string): Promise<{ texture: WebGLTexture, width: number, height: number }> { - const image = await new Promise((resolve, reject) => { - const img = new Image(); - img.onload = () => resolve(img); - img.onerror = reject; - img.src = getProxiedImageUrl(imageUrl); // CORS対策 - }); - - const texture = createTexture(gl); - gl.activeTexture(gl.TEXTURE0); - gl.bindTexture(gl.TEXTURE_2D, texture); - gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, image.width, image.height, 0, gl.RGBA, gl.UNSIGNED_BYTE, image); - gl.bindTexture(gl.TEXTURE_2D, null); - - return { - texture, - width: image.width, - height: image.height, - }; -} - -export async function createTextureFromText(gl: WebGL2RenderingContext, text: string, resolution = 2048): Promise<{ texture: WebGLTexture, width: number, height: number }> { - const ctx = window.document.createElement('canvas').getContext('2d')!; - ctx.canvas.width = resolution; - ctx.canvas.height = resolution / 4; - const fontSize = resolution / 32; - const margin = fontSize / 2; - ctx.shadowColor = '#000000'; - ctx.shadowBlur = fontSize / 4; - - //ctx.fillStyle = '#00ff00'; - //ctx.fillRect(0, 0, ctx.canvas.width, ctx.canvas.height); - - ctx.fillStyle = '#ffffff'; - ctx.font = `bold ${fontSize}px sans-serif`; - ctx.textBaseline = 'middle'; - - ctx.fillText(text, margin, ctx.canvas.height / 2); - - const textMetrics = ctx.measureText(text); - const cropWidth = (Math.ceil(textMetrics.actualBoundingBoxRight + textMetrics.actualBoundingBoxLeft) + margin + margin); - const cropHeight = (Math.ceil(textMetrics.actualBoundingBoxAscent + textMetrics.actualBoundingBoxDescent) + margin + margin); - const data = ctx.getImageData(0, (ctx.canvas.height / 2) - (cropHeight / 2), ctx.canvas.width, ctx.canvas.height); - - const texture = createTexture(gl); - gl.activeTexture(gl.TEXTURE0); - gl.bindTexture(gl.TEXTURE_2D, texture); - gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, cropWidth, cropHeight, 0, gl.RGBA, gl.UNSIGNED_BYTE, data); - gl.bindTexture(gl.TEXTURE_2D, null); - - const info = { - texture: texture, - width: cropWidth, - height: cropHeight, - }; - - ctx.canvas.remove(); - - return info; -} diff --git a/packages/frontend/src/utility/watermark.ts b/packages/frontend/src/utility/watermark.ts index 6871ca3676..b9975fd4fc 100644 --- a/packages/frontend/src/utility/watermark.ts +++ b/packages/frontend/src/utility/watermark.ts @@ -4,7 +4,6 @@ */ import { FX_watermarkPlacement } from './image-effector/fxs/watermarkPlacement.js'; -import { createTextureFromText, createTextureFromUrl } from './image-effector/utilts.js'; import type { ImageEffectorLayer } from '@/utility/image-effector/ImageEffector.js'; import { ImageEffector } from '@/utility/image-effector/ImageEffector.js'; @@ -35,7 +34,6 @@ export type WatermarkPreset = { export class WatermarkRenderer { private effector: ImageEffector; private layers: WatermarkPreset['layers'] = []; - private texturesKey = ''; constructor(options: { canvas: HTMLCanvasElement, @@ -52,30 +50,6 @@ export class WatermarkRenderer { }); } - private calcTexturesKey() { - return this.layers.map(layer => { - if (layer.type === 'image' && layer.imageUrl != null) { - return layer.imageUrl; - } else if (layer.type === 'text' && layer.text != null) { - return layer.text; - } - return ''; - }).join(';'); - } - - private async bakeTextures(): Promise { - this.effector.disposeExternalTextures(); - for (const layer of this.layers) { - if (layer.type === 'text' && layer.text != null) { - const { texture, width, height } = await createTextureFromText(this.effector.gl, layer.text); - this.effector.registerExternalTexture(layer.id, texture, width, height); - } else if (layer.type === 'image' && layer.imageUrl != null) { - const { texture, width, height } = await createTextureFromUrl(this.effector.gl, layer.imageUrl); - this.effector.registerExternalTexture(layer.id, texture, width, height); - } - } - } - private makeImageEffectorLayers(): ImageEffectorLayer[] { return this.layers.map(layer => { if (layer.type === 'text') { @@ -88,8 +62,11 @@ export class WatermarkRenderer { align: layer.align, opacity: layer.opacity, cover: false, + watermark: { + type: 'text', + text: layer.text, + }, }, - textures: { watermark: layer.id }, }; } else { return { @@ -101,8 +78,11 @@ export class WatermarkRenderer { align: layer.align, opacity: layer.opacity, cover: layer.cover, + watermark: { + type: 'url', + url: layer.imageUrl, + }, }, - textures: { watermark: layer.id }, }; } }); @@ -110,15 +90,7 @@ export class WatermarkRenderer { public async setLayers(layers: WatermarkPreset['layers']) { this.layers = layers; - - const newTexturesKey = this.calcTexturesKey(); - if (newTexturesKey !== this.texturesKey) { - this.texturesKey = newTexturesKey; - await this.bakeTextures(); - } - - this.effector.setLayers(this.makeImageEffectorLayers()); - + await this.effector.setLayers(this.makeImageEffectorLayers()); this.render(); }