diff --git a/packages/frontend/src/components/MkImageFrameEditorDialog.vue b/packages/frontend/src/components/MkImageFrameEditorDialog.vue index ac3e05ed98..af44a6f162 100644 --- a/packages/frontend/src/components/MkImageFrameEditorDialog.vue +++ b/packages/frontend/src/components/MkImageFrameEditorDialog.vue @@ -132,8 +132,8 @@ SPDX-License-Identifier: AGPL-3.0-only import { ref, useTemplateRef, watch, onMounted, onUnmounted, reactive, nextTick } from 'vue'; import ExifReader from 'exifreader'; import { throttle } from 'throttle-debounce'; -import type { ImageFrameParams } from '@/utility/image-frame-renderer.js'; -import { ImageFrameRenderer } from '@/utility/image-frame-renderer.js'; +import type { ImageFrameParams } from '@/utility/image-frame-renderer/image-frame-renderer.js'; +import { ImageFrameRenderer } from '@/utility/image-frame-renderer/image-frame-renderer.js'; import { i18n } from '@/i18n.js'; import MkModalWindow from '@/components/MkModalWindow.vue'; import MkSelect from '@/components/MkSelect.vue'; diff --git a/packages/frontend/src/composables/use-uploader.ts b/packages/frontend/src/composables/use-uploader.ts index 2c452d44ec..5b3a1cc8f6 100644 --- a/packages/frontend/src/composables/use-uploader.ts +++ b/packages/frontend/src/composables/use-uploader.ts @@ -11,7 +11,7 @@ import { computed, markRaw, onMounted, onUnmounted, ref, triggerRef } from 'vue' import ExifReader from 'exifreader'; import type { MenuItem } from '@/types/menu.js'; import type { WatermarkPreset } from '@/utility/watermark.js'; -import type { ImageFrameParams, ImageFramePreset } from '@/utility/image-frame-renderer.js'; +import type { ImageFrameParams, ImageFramePreset } from '@/utility/image-frame-renderer/image-frame-renderer.js'; import { genId } from '@/utility/id.js'; import { i18n } from '@/i18n.js'; import { prefer } from '@/preferences.js'; @@ -20,7 +20,7 @@ import { uploadFile, UploadAbortedError } from '@/utility/drive.js'; import * as os from '@/os.js'; import { ensureSignin } from '@/i.js'; import { WatermarkRenderer } from '@/utility/watermark.js'; -import { ImageFrameRenderer } from '@/utility/image-frame-renderer.js'; +import { ImageFrameRenderer } from '@/utility/image-frame-renderer/image-frame-renderer.js'; export type UploaderFeatures = { imageEditing?: boolean; diff --git a/packages/frontend/src/preferences/def.ts b/packages/frontend/src/preferences/def.ts index 4054e39afe..6ad84b64e0 100644 --- a/packages/frontend/src/preferences/def.ts +++ b/packages/frontend/src/preferences/def.ts @@ -13,7 +13,7 @@ import type { Plugin } from '@/plugin.js'; import type { DeviceKind } from '@/utility/device-kind.js'; import type { DeckProfile } from '@/deck.js'; import type { WatermarkPreset } from '@/utility/watermark.js'; -import type { ImageFramePreset } from '@/utility/image-frame-renderer.js'; +import type { ImageFramePreset } from '@/utility/image-frame-renderer/image-frame-renderer.js'; import { genId } from '@/utility/id.js'; import { DEFAULT_DEVICE_KIND } from '@/utility/device-kind.js'; import { deepEqual } from '@/utility/deep-equal.js'; diff --git a/packages/frontend/src/utility/image-effector/ImageCompositor.ts b/packages/frontend/src/utility/image-effector/ImageCompositor.ts index fa57c591fd..b0a32d8962 100644 --- a/packages/frontend/src/utility/image-effector/ImageCompositor.ts +++ b/packages/frontend/src/utility/image-effector/ImageCompositor.ts @@ -5,35 +5,32 @@ import { createTexture, initShaderProgram } from '../webgl.js'; -export type ImageCompositorProgramParamDefs = Record; +export type ImageCompositorFunctionParams = Record; -export function defineImageCompositorFx(program: ImageCompositorProgram) { - return program; -} - -export type ImageCompositorProgram = { - id: string; +export type ImageCompositorFunction = { shader: string; - uniforms: US; - params: PS, main: (ctx: { gl: WebGL2RenderingContext; program: WebGLProgram; params: PS; - u: Record; + u: Record; width: number; height: number; textures: Map; }) => void; }; -export type ImageCompositorNode = { +export type ImageCompositorLayer = { id: string; - programId: string; + functionId: string; params: Record; }; -// TODO: per node cache +export function defineImageCompositorFunction(fn: ImageCompositorFunction) { + return fn; +} + +// TODO: per layer cache export class ImageCompositor { private gl: WebGL2RenderingContext; @@ -46,7 +43,7 @@ export class ImageCompositor { private perLayerResultFrameBuffers: Map = new Map(); private nopProgram: WebGLProgram; private registeredTextures: Map = new Map(); - private programs: ImageCompositorProgram[] = []; + private registeredFunctions: Map = new Map(); constructor(options: { canvas: HTMLCanvasElement; @@ -116,13 +113,23 @@ export class ImageCompositor { gl.enableVertexAttribArray(positionLocation); } - private renderNode(node: ImageCompositorNode, preTexture: WebGLTexture, invert = false) { + private extractUniformNamesFromShader(shader: string): string[] { + const uniformRegex = /uniform\s+\w+\s+(\w+)\s*;/g; + const uniforms: string[] = []; + let match; + while ((match = uniformRegex.exec(shader)) !== null) { + uniforms.push(match[1].replace(/^u_/, '')); + } + return uniforms; + } + + private renderLayer(layer: ImageCompositorLayer, preTexture: WebGLTexture, invert = false) { const gl = this.gl; - const program = this.programs.find(p => p.id === node.programId); - if (program == null) return; + const fn = this.registeredFunctions.get(layer.functionId); + if (fn == null) return; - const cachedShader = this.shaderCache.get(program.id); + const cachedShader = this.shaderCache.get(fn.id); const shaderProgram = cachedShader ?? initShaderProgram(this.gl, `#version 300 es in vec2 position; uniform bool u_invert; @@ -132,9 +139,9 @@ export class ImageCompositor { in_uv = (position + 1.0) / 2.0; gl_Position = u_invert ? vec4(position * vec2(1.0, -1.0), 0.0, 1.0) : vec4(position, 0.0, 1.0); } - `, program.shader); + `, fn.shader); if (cachedShader == null) { - this.shaderCache.set(program.id, shaderProgram); + this.shaderCache.set(fn.id, shaderProgram); } gl.useProgram(shaderProgram); @@ -150,11 +157,11 @@ export class ImageCompositor { const in_texture = gl.getUniformLocation(shaderProgram, 'in_texture'); gl.uniform1i(in_texture, 0); - program.main({ + fn.main({ gl: gl, program: shaderProgram, - params: node.params, - u: Object.fromEntries(program.uniforms.map(u => [u, gl.getUniformLocation(shaderProgram, 'u_' + u)!])), + params: layer.params, + u: Object.fromEntries(fn.uniforms.map(u => [u, gl.getUniformLocation(shaderProgram, 'u_' + u)!])), width: this.renderWidth, height: this.renderHeight, textures: this.registeredTextures, @@ -163,11 +170,11 @@ export class ImageCompositor { gl.drawArrays(gl.TRIANGLES, 0, 6); } - public render(nodes: ImageCompositorNode[]) { + public render(layers: ImageCompositorLayer[]) { const gl = this.gl; // 入力をそのまま出力 - if (nodes.length === 0) { + if (layers.length === 0) { gl.activeTexture(gl.TEXTURE0); gl.bindTexture(gl.TEXTURE_2D, this.baseTexture); @@ -180,8 +187,8 @@ export class ImageCompositor { let preTexture = this.baseTexture; - for (const layer of nodes) { - const isLast = layer === nodes.at(-1); + for (const layer of layers) { + const isLast = layer === layers.at(-1); const cachedResultTexture = this.perLayerResultTextures.get(layer.id); const resultTexture = cachedResultTexture ?? createTexture(gl); @@ -204,14 +211,15 @@ export class ImageCompositor { gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, resultTexture, 0); } - this.renderNode(layer, preTexture, isLast); + this.renderLayer(layer, preTexture, isLast); preTexture = resultTexture; } } - public registerProgram(program: ImageCompositorProgram) { - this.programs.push(program); + public registerFunction(id: string, fn: ImageCompositorFunction) { + const uniforms = this.extractUniformNamesFromShader(fn.shader); + this.registeredFunctions.set(id, { ...fn, id, uniforms }); } public registerTexture(key: string, image: ImageData | ImageBitmap | HTMLImageElement | HTMLCanvasElement) { @@ -236,6 +244,24 @@ export class ImageCompositor { }); } + public unregisterTexture(key: string) { + const gl = this.gl; + + const existing = this.registeredTextures.get(key); + if (existing != null) { + gl.deleteTexture(existing.texture); + this.registeredTextures.delete(key); + } + } + + public hasTexture(key: string) { + return this.registeredTextures.has(key); + } + + public getKeysOfRegisteredTextures() { + return this.registeredTextures.keys(); + } + public changeResolution(width: number, height: number) { if (this.renderWidth === width && this.renderHeight === height) return; diff --git a/packages/frontend/src/utility/image-effector/ImageEffector.ts b/packages/frontend/src/utility/image-effector/ImageEffector.ts index 5bd80feac7..c2df927080 100644 --- a/packages/frontend/src/utility/image-effector/ImageEffector.ts +++ b/packages/frontend/src/utility/image-effector/ImageEffector.ts @@ -6,7 +6,8 @@ import QRCodeStyling from 'qr-code-styling'; import { url, host } from '@@/js/config.js'; import { getProxiedImageUrl } from '../media-proxy.js'; -import { createTexture, initShaderProgram } from '../webgl.js'; +import { ImageCompositor } from './ImageCompositor.js'; +import type { ImageCompositorFunction, ImageCompositorLayer } from './ImageCompositor.js'; import { ensureSignin } from '@/i.js'; export type ImageEffectorRGB = [r: number, g: number, b: number]; @@ -94,243 +95,72 @@ export type ParamsRecordTypeToDefRecord = { [K in keyof PS]: GetParamType; }; -export function defineImageEffectorFx(fx: ImageEffectorFx) { - return fx; -} - -export type ImageEffectorFx = { - id: ID; +export type ImageEffectorFxDefinition = { + id: string; name: string; - shader: string; - uniforms: US; params: PS, - main: (ctx: { - gl: WebGL2RenderingContext; - program: WebGLProgram; - params: ParamsRecordTypeToDefRecord; - u: Record; - width: number; - height: number; - textures: Record; - }) => void; + shader: string; + main: ImageCompositorFunction['main']; +}; + +export type ImageEffectorFx = { + id: string; + name: string; + fn: ImageCompositorFunction; + params: PS, }; export type ImageEffectorLayer = { id: string; fxId: string; - params: Record; + params: ImageCompositorLayer['params']; }; +export function defineImageEffectorFx(fx: ImageEffectorFxDefinition): ImageEffectorFx { + return { + id: fx.id, + name: fx.name, + fn: { + shader: fx.shader, + main: fx.main, + }, + params: fx.params, + }; +} + function getValue(params: Record, k: string): ParamTypeToPrimitive[T] { return params[k]; } -export class ImageEffector>> { - private gl: WebGL2RenderingContext; +export class ImageEffector { private canvas: HTMLCanvasElement | null = null; - private renderWidth: number; - private renderHeight: number; - private layers: ImageEffectorLayer[] = []; - private baseTexture: WebGLTexture; - private shaderCache: Map = new Map(); - private perLayerResultTextures: Map = new Map(); - private perLayerResultFrameBuffers: Map = new Map(); - private nopProgram: WebGLProgram; - private fxs: [...IEX]; - private paramTextures: Map = new Map(); - private registeredTextures: Map = new Map(); + private fxs: ImageEffectorFx[]; + private compositor: ImageCompositor; constructor(options: { canvas: HTMLCanvasElement; renderWidth: number; renderHeight: number; image: ImageData | ImageBitmap | HTMLImageElement | HTMLCanvasElement | null; - fxs: [...IEX]; + fxs: ImageEffectorFx[]; }) { this.canvas = options.canvas; - this.renderWidth = options.renderWidth; - this.renderHeight = options.renderHeight; this.fxs = options.fxs; - this.canvas.width = this.renderWidth; - this.canvas.height = this.renderHeight; - - const gl = this.canvas.getContext('webgl2', { - preserveDrawingBuffer: false, - alpha: true, - premultipliedAlpha: false, + this.compositor = new ImageCompositor({ + canvas: this.canvas, + renderWidth: options.renderWidth, + renderHeight: options.renderHeight, + image: options.image, }); - if (gl == null) throw new Error('Failed to initialize WebGL2 context'); - - this.gl = gl; - - gl.viewport(0, 0, gl.drawingBufferWidth, gl.drawingBufferHeight); - - const VERTICES = new Float32Array([-1, -1, -1, 1, 1, 1, -1, -1, 1, 1, 1, -1]); - const vertexBuffer = gl.createBuffer(); - gl.bindBuffer(gl.ARRAY_BUFFER, vertexBuffer); - gl.bufferData(gl.ARRAY_BUFFER, VERTICES, gl.STATIC_DRAW); - - if (options.image != null) { - this.baseTexture = createTexture(gl); - gl.activeTexture(gl.TEXTURE0); - gl.bindTexture(gl.TEXTURE_2D, this.baseTexture); - gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, options.image.width, options.image.height, 0, gl.RGBA, gl.UNSIGNED_BYTE, options.image); - gl.bindTexture(gl.TEXTURE_2D, null); - } else { - this.baseTexture = createTexture(gl); - gl.activeTexture(gl.TEXTURE0); - } - - this.nopProgram = initShaderProgram(this.gl, `#version 300 es - in vec2 position; - out vec2 in_uv; - - void main() { - in_uv = (position + 1.0) / 2.0; - gl_Position = vec4(position * vec2(1.0, -1.0), 0.0, 1.0); - } - `, `#version 300 es - precision mediump float; - - in vec2 in_uv; - uniform sampler2D u_texture; - out vec4 out_color; - - void main() { - out_color = texture(u_texture, in_uv); - } - `); - - // レジスタ番号はシェーダープログラムに属しているわけではなく、独立の存在なので、とりあえず nopProgram を使って設定する(その後は効果が持続する) - // ref. https://qiita.com/emadurandal/items/5966c8374f03d4de3266 - const positionLocation = gl.getAttribLocation(this.nopProgram, 'position'); - gl.vertexAttribPointer(positionLocation, 2, gl.FLOAT, false, 0, 0); - gl.enableVertexAttribArray(positionLocation); - } - - private renderLayer(layer: ImageEffectorLayer, preTexture: WebGLTexture, invert = false) { - const gl = this.gl; - - const fx = this.fxs.find(fx => fx.id === layer.fxId); - if (fx == null) return; - - const cachedShader = this.shaderCache.get(fx.id); - const shaderProgram = cachedShader ?? initShaderProgram(this.gl, `#version 300 es - in vec2 position; - uniform bool u_invert; - out vec2 in_uv; - - void main() { - in_uv = (position + 1.0) / 2.0; - gl_Position = u_invert ? vec4(position * vec2(1.0, -1.0), 0.0, 1.0) : vec4(position, 0.0, 1.0); - } - `, fx.shader); - if (cachedShader == null) { - this.shaderCache.set(fx.id, shaderProgram); - } - - gl.useProgram(shaderProgram); - - const in_resolution = gl.getUniformLocation(shaderProgram, 'in_resolution'); - gl.uniform2fv(in_resolution, [this.renderWidth, this.renderHeight]); - - const u_invert = gl.getUniformLocation(shaderProgram, 'u_invert'); - gl.uniform1i(u_invert, invert ? 1 : 0); - - gl.activeTexture(gl.TEXTURE0); - gl.bindTexture(gl.TEXTURE_2D, preTexture); - const in_texture = gl.getUniformLocation(shaderProgram, 'in_texture'); - gl.uniform1i(in_texture, 0); - - fx.main({ - gl: gl, - program: shaderProgram, - params: Object.fromEntries( - Object.entries(fx.params as ImageEffectorFxParamDefs).map(([key, param]) => { - return [key, layer.params[key] ?? param.default]; - }), - ), - u: Object.fromEntries(fx.uniforms.map(u => [u, gl.getUniformLocation(shaderProgram, 'u_' + u)!])), - width: this.renderWidth, - height: this.renderHeight, - textures: Object.fromEntries( - Object.entries(fx.params as ImageEffectorFxParamDefs).map(([k, v]) => { - if (v.type === 'textureRef') { - const param = getValue(layer.params, k); - if (param == null) return [k, null]; - const texture = this.registeredTextures.get(param) ?? null; - return [k, texture]; - } else if (v.type === 'texture') { - const param = getValue(layer.params, k); - if (param == null) return [k, null]; - const texture = this.paramTextures.get(this.getTextureKeyForParam(param)) ?? null; - return [k, texture]; - } else { - return [k, null]; - } - })), - }); - - gl.drawArrays(gl.TRIANGLES, 0, 6); - } - - public render() { - const gl = this.gl; - - // 入力をそのまま出力 - if (this.layers.length === 0) { - gl.activeTexture(gl.TEXTURE0); - gl.bindTexture(gl.TEXTURE_2D, this.baseTexture); - - gl.useProgram(this.nopProgram); - gl.uniform1i(gl.getUniformLocation(this.nopProgram, 'u_texture')!, 0); - - gl.drawArrays(gl.TRIANGLES, 0, 6); - return; - } - - let preTexture = this.baseTexture; - - for (const layer of this.layers) { - const isLast = layer === this.layers.at(-1); - - const cachedResultTexture = this.perLayerResultTextures.get(layer.id); - const resultTexture = cachedResultTexture ?? createTexture(gl); - if (cachedResultTexture == null) { - this.perLayerResultTextures.set(layer.id, resultTexture); - } - gl.bindTexture(gl.TEXTURE_2D, resultTexture); - gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, this.renderWidth, this.renderHeight, 0, gl.RGBA, gl.UNSIGNED_BYTE, null); - gl.bindTexture(gl.TEXTURE_2D, null); - - if (isLast) { - gl.bindFramebuffer(gl.FRAMEBUFFER, null); - } else { - const cachedResultFrameBuffer = this.perLayerResultFrameBuffers.get(layer.id); - const resultFrameBuffer = cachedResultFrameBuffer ?? gl.createFramebuffer()!; - if (cachedResultFrameBuffer == null) { - this.perLayerResultFrameBuffers.set(layer.id, resultFrameBuffer); - } - gl.bindFramebuffer(gl.FRAMEBUFFER, resultFrameBuffer); - gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, resultTexture, 0); - } - - this.renderLayer(layer, preTexture, isLast); - - preTexture = resultTexture; + for (const fx of this.fxs) { + this.compositor.registerFunction(fx.id, fx.fn); } } public async setLayersAndRender(layers: ImageEffectorLayer[]) { - this.layers = layers; - - const unused = new Set(this.paramTextures.keys()); + const unused = new Set(this.compositor.getKeysOfRegisteredTextures()); for (const layer of layers) { const fx = this.fxs.find(fx => fx.id === layer.fxId); @@ -345,62 +175,35 @@ export class ImageEffector...`); - const texture = - v.type === 'text' ? await createTextureFromText(this.gl, v.text) : - v.type === 'url' ? await createTextureFromUrl(this.gl, v.url) : - v.type === 'qr' ? await createTextureFromQr(this.gl, { data: v.data }) : + const image = + v.type === 'text' ? await createTextureFromText(v.text) : + v.type === 'url' ? await createTextureFromUrl(v.url) : + v.type === 'qr' ? await createTextureFromQr({ data: v.data }) : null; - if (texture == null) continue; + if (image == null) continue; - this.paramTextures.set(textureKey, texture); + this.compositor.registerTexture(textureKey, image); } } for (const k of unused) { if (_DEV_) console.log(`Dispose unused texture <${k}>...`); - this.gl.deleteTexture(this.paramTextures.get(k)!.texture); - this.paramTextures.delete(k); + this.compositor.unregisterTexture(k); } - this.render(); - } - - public registerTexture(key: string, image: ImageData | ImageBitmap | HTMLImageElement | HTMLCanvasElement) { - const gl = this.gl; - - if (this.registeredTextures.has(key)) { - const existing = this.registeredTextures.get(key)!; - gl.deleteTexture(existing.texture); - this.registeredTextures.delete(key); - } - - const texture = createTexture(gl); - gl.activeTexture(gl.TEXTURE0); - gl.bindTexture(gl.TEXTURE_2D, texture); - gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, image.width, image.height, 0, gl.RGBA, gl.UNSIGNED_BYTE, image); - gl.bindTexture(gl.TEXTURE_2D, null); - - this.registeredTextures.set(key, { - texture: texture, - width: image.width, - height: image.height, - }); + this.compositor.render(layers.map(layer => ({ + id: layer.id, + functionId: layer.fxId, + params: layer.params, + }))); } public changeResolution(width: number, height: number) { - if (this.renderWidth === width && this.renderHeight === height) return; - - this.renderWidth = width; - this.renderHeight = height; - if (this.canvas) { - this.canvas.width = this.renderWidth; - this.canvas.height = this.renderHeight; - } - this.gl.viewport(0, 0, this.renderWidth, this.renderHeight); + this.compositor.changeResolution(width, height); } private getTextureKeyForParam(v: ParamTypeToPrimitive['texture']) { @@ -417,43 +220,11 @@ export class ImageEffector { +async function createTextureFromUrl(imageUrl: string | null) { if (imageUrl == null || imageUrl.trim() === '') return null; const image = await new Promise((resolve, reject) => { @@ -465,20 +236,10 @@ async function createTextureFromUrl(gl: WebGL2RenderingContext, imageUrl: string if (image == null) return null; - const texture = createTexture(gl); - gl.activeTexture(gl.TEXTURE0); - gl.bindTexture(gl.TEXTURE_2D, texture); - gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, image.width, image.height, 0, gl.RGBA, gl.UNSIGNED_BYTE, image); - gl.bindTexture(gl.TEXTURE_2D, null); - - return { - texture, - width: image.width, - height: image.height, - }; + return image; } -async function createTextureFromText(gl: WebGL2RenderingContext, text: string | null, resolution = 2048): Promise<{ texture: WebGLTexture, width: number, height: number } | null> { +async function createTextureFromText(text: string | null, resolution = 2048) { if (text == null || text.trim() === '') return null; const ctx = window.document.createElement('canvas').getContext('2d')!; @@ -503,24 +264,12 @@ async function createTextureFromText(gl: WebGL2RenderingContext, text: string | const cropHeight = (Math.ceil(textMetrics.actualBoundingBoxAscent + textMetrics.actualBoundingBoxDescent) + margin + margin); const data = ctx.getImageData(0, (ctx.canvas.height / 2) - (cropHeight / 2), ctx.canvas.width, ctx.canvas.height); - const texture = createTexture(gl); - gl.activeTexture(gl.TEXTURE0); - gl.bindTexture(gl.TEXTURE_2D, texture); - gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, cropWidth, cropHeight, 0, gl.RGBA, gl.UNSIGNED_BYTE, data); - gl.bindTexture(gl.TEXTURE_2D, null); - - const info = { - texture: texture, - width: cropWidth, - height: cropHeight, - }; - ctx.canvas.remove(); - return info; + return data; } -async function createTextureFromQr(gl: WebGL2RenderingContext, options: { data: string | null }, resolution = 512): Promise<{ texture: WebGLTexture, width: number, height: number } | null> { +async function createTextureFromQr(options: { data: string | null }, resolution = 512) { const $i = ensureSignin(); const qrCodeInstance = new QRCodeStyling({ @@ -557,15 +306,5 @@ async function createTextureFromQr(gl: WebGL2RenderingContext, options: { data: const image = await window.createImageBitmap(blob); - const texture = createTexture(gl); - gl.activeTexture(gl.TEXTURE0); - gl.bindTexture(gl.TEXTURE_2D, texture); - gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, resolution, resolution, 0, gl.RGBA, gl.UNSIGNED_BYTE, image); - gl.bindTexture(gl.TEXTURE_2D, null); - - return { - texture, - width: resolution, - height: resolution, - }; + return image; } diff --git a/packages/frontend/src/utility/image-effector/fxs/blockNoise.ts b/packages/frontend/src/utility/image-effector/fxs/blockNoise.ts index 355ab4536c..b825f0e515 100644 --- a/packages/frontend/src/utility/image-effector/fxs/blockNoise.ts +++ b/packages/frontend/src/utility/image-effector/fxs/blockNoise.ts @@ -4,15 +4,14 @@ */ import seedrandom from 'seedrandom'; -import shader from './blockNoise.glsl'; import { defineImageEffectorFx } from '../ImageEffector.js'; +import shader from './blockNoise.glsl'; import { i18n } from '@/i18n.js'; export const FX_blockNoise = defineImageEffectorFx({ id: 'blockNoise', name: i18n.ts._imageEffector._fxs.glitch + ': ' + i18n.ts._imageEffector._fxs.blockNoise, shader, - uniforms: ['amount', 'channelShift'] as const, params: { amount: { label: i18n.ts._imageEffector._fxProps.amount, diff --git a/packages/frontend/src/utility/image-effector/fxs/blur.ts b/packages/frontend/src/utility/image-effector/fxs/blur.ts index 40f51fa646..c7f240d858 100644 --- a/packages/frontend/src/utility/image-effector/fxs/blur.ts +++ b/packages/frontend/src/utility/image-effector/fxs/blur.ts @@ -11,7 +11,6 @@ export const FX_blur = defineImageEffectorFx({ id: 'blur', name: i18n.ts._imageEffector._fxs.blur, shader, - uniforms: ['offset', 'scale', 'ellipse', 'angle', 'radius', 'samples'] as const, params: { offsetX: { label: i18n.ts._imageEffector._fxProps.offset + ' X', diff --git a/packages/frontend/src/utility/image-effector/fxs/checker.ts b/packages/frontend/src/utility/image-effector/fxs/checker.ts index 7d1938eeb7..37d5146392 100644 --- a/packages/frontend/src/utility/image-effector/fxs/checker.ts +++ b/packages/frontend/src/utility/image-effector/fxs/checker.ts @@ -11,7 +11,6 @@ export const FX_checker = defineImageEffectorFx({ id: 'checker', name: i18n.ts._imageEffector._fxs.checker, shader, - uniforms: ['angle', 'scale', 'color', 'opacity'] as const, params: { angle: { label: i18n.ts._imageEffector._fxProps.angle, diff --git a/packages/frontend/src/utility/image-effector/fxs/chromaticAberration.ts b/packages/frontend/src/utility/image-effector/fxs/chromaticAberration.ts index ed4d134251..6b435c60d5 100644 --- a/packages/frontend/src/utility/image-effector/fxs/chromaticAberration.ts +++ b/packages/frontend/src/utility/image-effector/fxs/chromaticAberration.ts @@ -11,7 +11,6 @@ export const FX_chromaticAberration = defineImageEffectorFx({ id: 'chromaticAberration', name: i18n.ts._imageEffector._fxs.chromaticAberration, shader, - uniforms: ['amount', 'start', 'normalize'] as const, params: { normalize: { label: i18n.ts._imageEffector._fxProps.normalize, diff --git a/packages/frontend/src/utility/image-effector/fxs/colorAdjust.ts b/packages/frontend/src/utility/image-effector/fxs/colorAdjust.ts index 989ca79a2c..7e281c91ac 100644 --- a/packages/frontend/src/utility/image-effector/fxs/colorAdjust.ts +++ b/packages/frontend/src/utility/image-effector/fxs/colorAdjust.ts @@ -11,7 +11,6 @@ export const FX_colorAdjust = defineImageEffectorFx({ id: 'colorAdjust', name: i18n.ts._imageEffector._fxs.colorAdjust, shader, - uniforms: ['lightness', 'contrast', 'hue', 'brightness', 'saturation'] as const, params: { lightness: { label: i18n.ts._imageEffector._fxProps.lightness, diff --git a/packages/frontend/src/utility/image-effector/fxs/colorClamp.ts b/packages/frontend/src/utility/image-effector/fxs/colorClamp.ts index f3513011fa..58b978e941 100644 --- a/packages/frontend/src/utility/image-effector/fxs/colorClamp.ts +++ b/packages/frontend/src/utility/image-effector/fxs/colorClamp.ts @@ -11,7 +11,6 @@ export const FX_colorClamp = defineImageEffectorFx({ id: 'colorClamp', name: i18n.ts._imageEffector._fxs.colorClamp, shader, - uniforms: ['rMax', 'rMin', 'gMax', 'gMin', 'bMax', 'bMin'] as const, params: { max: { label: i18n.ts._imageEffector._fxProps.max, diff --git a/packages/frontend/src/utility/image-effector/fxs/colorClampAdvanced.ts b/packages/frontend/src/utility/image-effector/fxs/colorClampAdvanced.ts index 397e16c1ba..afd684740b 100644 --- a/packages/frontend/src/utility/image-effector/fxs/colorClampAdvanced.ts +++ b/packages/frontend/src/utility/image-effector/fxs/colorClampAdvanced.ts @@ -11,7 +11,6 @@ export const FX_colorClampAdvanced = defineImageEffectorFx({ id: 'colorClampAdvanced', name: i18n.ts._imageEffector._fxs.colorClampAdvanced, shader, - uniforms: ['rMax', 'rMin', 'gMax', 'gMin', 'bMax', 'bMin'] as const, params: { rMax: { label: `${i18n.ts._imageEffector._fxProps.max} (${i18n.ts._imageEffector._fxProps.redComponent})`, diff --git a/packages/frontend/src/utility/image-effector/fxs/distort.ts b/packages/frontend/src/utility/image-effector/fxs/distort.ts index 3ea93a0266..ff1a9fab8d 100644 --- a/packages/frontend/src/utility/image-effector/fxs/distort.ts +++ b/packages/frontend/src/utility/image-effector/fxs/distort.ts @@ -11,7 +11,6 @@ export const FX_distort = defineImageEffectorFx({ id: 'distort', name: i18n.ts._imageEffector._fxs.distort, shader, - uniforms: ['phase', 'frequency', 'strength', 'direction'] as const, params: { direction: { label: i18n.ts._imageEffector._fxProps.direction, diff --git a/packages/frontend/src/utility/image-effector/fxs/fill.ts b/packages/frontend/src/utility/image-effector/fxs/fill.ts index 772cd76cf7..e8677135ab 100644 --- a/packages/frontend/src/utility/image-effector/fxs/fill.ts +++ b/packages/frontend/src/utility/image-effector/fxs/fill.ts @@ -11,7 +11,6 @@ export const FX_fill = defineImageEffectorFx({ id: 'fill', name: i18n.ts._imageEffector._fxs.fill, shader, - uniforms: ['offset', 'scale', 'ellipse', 'angle', 'color', 'opacity'] as const, params: { offsetX: { label: i18n.ts._imageEffector._fxProps.offset + ' X', diff --git a/packages/frontend/src/utility/image-effector/fxs/frame.ts b/packages/frontend/src/utility/image-effector/fxs/frame.ts deleted file mode 100644 index 44f52518ec..0000000000 --- a/packages/frontend/src/utility/image-effector/fxs/frame.ts +++ /dev/null @@ -1,98 +0,0 @@ -/* - * SPDX-FileCopyrightText: syuilo and misskey-project - * SPDX-License-Identifier: AGPL-3.0-only - */ - -import { defineImageEffectorFx } from '../ImageEffector.js'; -import shader from './frame.glsl'; - -export const FX_frame = defineImageEffectorFx({ - id: 'frame', - name: '(internal)', - shader, - uniforms: ['image', 'topLabel', 'bottomLabel', 'topLabelEnabled', 'bottomLabelEnabled', 'paddingTop', 'paddingBottom', 'paddingLeft', 'paddingRight', 'bg'] as const, - params: { - image: { - type: 'textureRef', - default: null, - }, - topLabel: { - type: 'textureRef', - default: null, - }, - bottomLabel: { - type: 'textureRef', - default: null, - }, - topLabelEnabled: { - type: 'boolean', - default: false, - }, - bottomLabelEnabled: { - type: 'boolean', - default: false, - }, - paddingTop: { - type: 'number', - default: 0, - max: 1, - min: 0, - }, - paddingBottom: { - type: 'number', - default: 0, - max: 1, - min: 0, - }, - paddingLeft: { - type: 'number', - default: 0, - max: 1, - min: 0, - }, - paddingRight: { - type: 'number', - default: 0, - max: 1, - min: 0, - }, - bg: { - type: 'color', - default: [1, 1, 1], - }, - }, - main: ({ gl, u, params, textures }) => { - const image = textures.image; - if (image == null) return; - - gl.activeTexture(gl.TEXTURE1); - gl.bindTexture(gl.TEXTURE_2D, image.texture); - gl.uniform1i(u.image, 1); - - gl.uniform1i(u.topLabelEnabled, params.topLabelEnabled ? 1 : 0); - gl.uniform1i(u.bottomLabelEnabled, params.bottomLabelEnabled ? 1 : 0); - gl.uniform1f(u.paddingTop, params.paddingTop); - gl.uniform1f(u.paddingBottom, params.paddingBottom); - gl.uniform1f(u.paddingLeft, params.paddingLeft); - gl.uniform1f(u.paddingRight, params.paddingRight); - gl.uniform3f(u.bg, params.bg[0], params.bg[1], params.bg[2]); - - if (params.topLabelEnabled) { - const topLabel = textures.topLabel; - if (topLabel) { - gl.activeTexture(gl.TEXTURE2); - gl.bindTexture(gl.TEXTURE_2D, topLabel.texture); - gl.uniform1i(u.topLabel, 2); - } - } - - if (params.bottomLabelEnabled) { - const bottomLabel = textures.bottomLabel; - if (bottomLabel) { - gl.activeTexture(gl.TEXTURE3); - gl.bindTexture(gl.TEXTURE_2D, bottomLabel.texture); - gl.uniform1i(u.bottomLabel, 3); - } - } - }, -}); diff --git a/packages/frontend/src/utility/image-effector/fxs/grayscale.ts b/packages/frontend/src/utility/image-effector/fxs/grayscale.ts index 055e8b4618..855ccb631e 100644 --- a/packages/frontend/src/utility/image-effector/fxs/grayscale.ts +++ b/packages/frontend/src/utility/image-effector/fxs/grayscale.ts @@ -11,7 +11,6 @@ export const FX_grayscale = defineImageEffectorFx({ id: 'grayscale', name: i18n.ts._imageEffector._fxs.grayscale, shader, - uniforms: [] as const, params: { }, main: ({ gl, params }) => { diff --git a/packages/frontend/src/utility/image-effector/fxs/invert.ts b/packages/frontend/src/utility/image-effector/fxs/invert.ts index 9417047931..68c9f7de18 100644 --- a/packages/frontend/src/utility/image-effector/fxs/invert.ts +++ b/packages/frontend/src/utility/image-effector/fxs/invert.ts @@ -11,7 +11,6 @@ export const FX_invert = defineImageEffectorFx({ id: 'invert', name: i18n.ts._imageEffector._fxs.invert, shader, - uniforms: ['r', 'g', 'b'] as const, params: { r: { label: i18n.ts._imageEffector._fxProps.redComponent, diff --git a/packages/frontend/src/utility/image-effector/fxs/mirror.ts b/packages/frontend/src/utility/image-effector/fxs/mirror.ts index 6515454ead..ec896c8f65 100644 --- a/packages/frontend/src/utility/image-effector/fxs/mirror.ts +++ b/packages/frontend/src/utility/image-effector/fxs/mirror.ts @@ -11,7 +11,6 @@ export const FX_mirror = defineImageEffectorFx({ id: 'mirror', name: i18n.ts._imageEffector._fxs.mirror, shader, - uniforms: ['h', 'v'] as const, params: { h: { label: i18n.ts.horizontal, @@ -19,7 +18,7 @@ export const FX_mirror = defineImageEffectorFx({ enum: [ { value: -1 as const, icon: 'ti ti-arrow-bar-right' }, { value: 0 as const, icon: 'ti ti-minus-vertical' }, - { value: 1 as const, icon: 'ti ti-arrow-bar-left' } + { value: 1 as const, icon: 'ti ti-arrow-bar-left' }, ], default: -1, }, @@ -29,7 +28,7 @@ export const FX_mirror = defineImageEffectorFx({ enum: [ { value: -1 as const, icon: 'ti ti-arrow-bar-down' }, { value: 0 as const, icon: 'ti ti-minus' }, - { value: 1 as const, icon: 'ti ti-arrow-bar-up' } + { value: 1 as const, icon: 'ti ti-arrow-bar-up' }, ], default: 0, }, diff --git a/packages/frontend/src/utility/image-effector/fxs/pixelate.ts b/packages/frontend/src/utility/image-effector/fxs/pixelate.ts index e3eef49b23..048e09bfc4 100644 --- a/packages/frontend/src/utility/image-effector/fxs/pixelate.ts +++ b/packages/frontend/src/utility/image-effector/fxs/pixelate.ts @@ -11,7 +11,6 @@ export const FX_pixelate = defineImageEffectorFx({ id: 'pixelate', name: i18n.ts._imageEffector._fxs.pixelate, shader, - uniforms: ['offset', 'scale', 'ellipse', 'angle', 'strength', 'samples'] as const, params: { offsetX: { label: i18n.ts._imageEffector._fxProps.offset + ' X', diff --git a/packages/frontend/src/utility/image-effector/fxs/polkadot.ts b/packages/frontend/src/utility/image-effector/fxs/polkadot.ts index 521e08cc7b..87fa451988 100644 --- a/packages/frontend/src/utility/image-effector/fxs/polkadot.ts +++ b/packages/frontend/src/utility/image-effector/fxs/polkadot.ts @@ -12,7 +12,6 @@ export const FX_polkadot = defineImageEffectorFx({ id: 'polkadot', name: i18n.ts._imageEffector._fxs.polkadot, shader, - uniforms: ['angle', 'scale', 'major_radius', 'major_opacity', 'minor_divisions', 'minor_radius', 'minor_opacity', 'color'] as const, params: { angle: { label: i18n.ts._imageEffector._fxProps.angle, diff --git a/packages/frontend/src/utility/image-effector/fxs/stripe.ts b/packages/frontend/src/utility/image-effector/fxs/stripe.ts index 3a6ecf970c..cd7e953bb3 100644 --- a/packages/frontend/src/utility/image-effector/fxs/stripe.ts +++ b/packages/frontend/src/utility/image-effector/fxs/stripe.ts @@ -12,7 +12,6 @@ export const FX_stripe = defineImageEffectorFx({ id: 'stripe', name: i18n.ts._imageEffector._fxs.stripe, shader, - uniforms: ['angle', 'frequency', 'phase', 'threshold', 'color', 'opacity'] as const, params: { angle: { label: i18n.ts._imageEffector._fxProps.angle, diff --git a/packages/frontend/src/utility/image-effector/fxs/tearing.ts b/packages/frontend/src/utility/image-effector/fxs/tearing.ts index 453b16bb19..0ee9dd1322 100644 --- a/packages/frontend/src/utility/image-effector/fxs/tearing.ts +++ b/packages/frontend/src/utility/image-effector/fxs/tearing.ts @@ -4,15 +4,14 @@ */ import seedrandom from 'seedrandom'; -import shader from './tearing.glsl'; import { defineImageEffectorFx } from '../ImageEffector.js'; +import shader from './tearing.glsl'; import { i18n } from '@/i18n.js'; export const FX_tearing = defineImageEffectorFx({ id: 'tearing', name: i18n.ts._imageEffector._fxs.glitch + ': ' + i18n.ts._imageEffector._fxs.tearing, shader, - uniforms: ['amount', 'channelShift'] as const, params: { amount: { label: i18n.ts._imageEffector._fxProps.amount, diff --git a/packages/frontend/src/utility/image-effector/fxs/threshold.ts b/packages/frontend/src/utility/image-effector/fxs/threshold.ts index d0bb8305ae..8b418a30a8 100644 --- a/packages/frontend/src/utility/image-effector/fxs/threshold.ts +++ b/packages/frontend/src/utility/image-effector/fxs/threshold.ts @@ -11,7 +11,6 @@ export const FX_threshold = defineImageEffectorFx({ id: 'threshold', name: i18n.ts._imageEffector._fxs.threshold, shader, - uniforms: ['r', 'g', 'b'] as const, params: { r: { label: i18n.ts._imageEffector._fxProps.redComponent, diff --git a/packages/frontend/src/utility/image-effector/fxs/watermarkPlacement.ts b/packages/frontend/src/utility/image-effector/fxs/watermarkPlacement.ts index bb51ed796b..27ffc97186 100644 --- a/packages/frontend/src/utility/image-effector/fxs/watermarkPlacement.ts +++ b/packages/frontend/src/utility/image-effector/fxs/watermarkPlacement.ts @@ -10,7 +10,6 @@ export const FX_watermarkPlacement = defineImageEffectorFx({ id: 'watermarkPlacement', name: '(internal)', shader, - uniforms: ['opacity', 'scale', 'angle', 'cover', 'repeat', 'alignX', 'alignY', 'margin', 'repeatMargin', 'noBBoxExpansion', 'wmResolution', 'wmEnabled', 'watermark'] as const, params: { cover: { type: 'boolean', diff --git a/packages/frontend/src/utility/image-effector/fxs/zoomLines.ts b/packages/frontend/src/utility/image-effector/fxs/zoomLines.ts index 8c0956d24e..a7e541c7d7 100644 --- a/packages/frontend/src/utility/image-effector/fxs/zoomLines.ts +++ b/packages/frontend/src/utility/image-effector/fxs/zoomLines.ts @@ -11,7 +11,6 @@ export const FX_zoomLines = defineImageEffectorFx({ id: 'zoomLines', name: i18n.ts._imageEffector._fxs.zoomLines, shader, - uniforms: ['pos', 'frequency', 'thresholdEnabled', 'threshold', 'maskSize', 'black'] as const, params: { x: { label: i18n.ts._imageEffector._fxProps.centerX, diff --git a/packages/frontend/src/utility/image-effector/fxs/frame.glsl b/packages/frontend/src/utility/image-frame-renderer/frame.glsl similarity index 100% rename from packages/frontend/src/utility/image-effector/fxs/frame.glsl rename to packages/frontend/src/utility/image-frame-renderer/frame.glsl diff --git a/packages/frontend/src/utility/image-frame-renderer/frame.ts b/packages/frontend/src/utility/image-frame-renderer/frame.ts new file mode 100644 index 0000000000..3b4fd73a8a --- /dev/null +++ b/packages/frontend/src/utility/image-frame-renderer/frame.ts @@ -0,0 +1,57 @@ +/* + * SPDX-FileCopyrightText: syuilo and misskey-project + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { defineImageCompositorFunction } from '../image-effector/ImageCompositor.js'; +import shader from './frame.glsl'; + +export const FN_frame = defineImageCompositorFunction<{ + image: string | null; + topLabel: string | null; + bottomLabel: string | null; + topLabelEnabled: boolean; + bottomLabelEnabled: boolean; + paddingTop: number; + paddingBottom: number; + paddingLeft: number; + paddingRight: number; + bg: [number, number, number]; +}>({ + shader, + main: ({ gl, u, params, textures }) => { + if (params.image == null) return; + const image = textures.get(params.image); + if (image == null) return; + + gl.activeTexture(gl.TEXTURE1); + gl.bindTexture(gl.TEXTURE_2D, image.texture); + gl.uniform1i(u.image, 1); + + gl.uniform1i(u.topLabelEnabled, params.topLabelEnabled ? 1 : 0); + gl.uniform1i(u.bottomLabelEnabled, params.bottomLabelEnabled ? 1 : 0); + gl.uniform1f(u.paddingTop, params.paddingTop); + gl.uniform1f(u.paddingBottom, params.paddingBottom); + gl.uniform1f(u.paddingLeft, params.paddingLeft); + gl.uniform1f(u.paddingRight, params.paddingRight); + gl.uniform3f(u.bg, params.bg[0], params.bg[1], params.bg[2]); + + if (params.topLabelEnabled && params.topLabel != null) { + const topLabel = textures.get(params.topLabel); + if (topLabel) { + gl.activeTexture(gl.TEXTURE2); + gl.bindTexture(gl.TEXTURE_2D, topLabel.texture); + gl.uniform1i(u.topLabel, 2); + } + } + + if (params.bottomLabelEnabled && params.bottomLabel != null) { + const bottomLabel = textures.get(params.bottomLabel); + if (bottomLabel) { + gl.activeTexture(gl.TEXTURE3); + gl.bindTexture(gl.TEXTURE_2D, bottomLabel.texture); + gl.uniform1i(u.bottomLabel, 3); + } + } + }, +}); diff --git a/packages/frontend/src/utility/image-frame-renderer.ts b/packages/frontend/src/utility/image-frame-renderer/image-frame-renderer.ts similarity index 91% rename from packages/frontend/src/utility/image-frame-renderer.ts rename to packages/frontend/src/utility/image-frame-renderer/image-frame-renderer.ts index f90e801b33..fcf5c070b5 100644 --- a/packages/frontend/src/utility/image-frame-renderer.ts +++ b/packages/frontend/src/utility/image-frame-renderer/image-frame-renderer.ts @@ -6,19 +6,13 @@ import QRCodeStyling from 'qr-code-styling'; import { url } from '@@/js/config.js'; import ExifReader from 'exifreader'; -import { FX_frame } from './image-effector/fxs/frame.js'; -import type { ImageEffectorFx, ImageEffectorLayer } from '@/utility/image-effector/ImageEffector.js'; +import { ImageCompositor } from '../image-effector/ImageCompositor.js'; +import { FN_frame } from './frame.js'; import { ImageEffector } from '@/utility/image-effector/ImageEffector.js'; import { ensureSignin } from '@/i.js'; const $i = ensureSignin(); -const FXS = [ - FX_frame, -] as const satisfies ImageEffectorFx[]; - -// TODO: 上部にもラベルを配置できるようにする - type LabelParams = { enabled: boolean; scale: number; @@ -45,7 +39,7 @@ export type ImageFramePreset = { }; export class ImageFrameRenderer { - private effector: ImageEffector; + private compositor: ImageCompositor; private image: HTMLImageElement | ImageBitmap; private exif: ExifReader.Tags; private renderAsPreview = false; @@ -61,15 +55,16 @@ export class ImageFrameRenderer { this.renderAsPreview = options.renderAsPreview ?? false; console.log(this.exif); - this.effector = new ImageEffector({ + this.compositor = new ImageCompositor({ canvas: options.canvas, renderWidth: 1, renderHeight: 1, image: null, - fxs: FXS, }); - this.effector.registerTexture('image', this.image); + this.compositor.registerFunction('frame', FN_frame); + + this.compositor.registerTexture('image', this.image); } private interpolateTemplateText(text: string) { @@ -195,7 +190,7 @@ export class ImageFrameRenderer { return labelCanvasCtx.getImageData(0, 0, labelCanvasCtx.canvas.width, labelCanvasCtx.canvas.height); ; } - public async updateAndRender(params: ImageFrameParams): Promise { + public async render(params: ImageFrameParams): Promise { let imageAreaW = this.image.width; let imageAreaH = this.image.height; @@ -219,18 +214,18 @@ export class ImageFrameRenderer { if (params.labelTop.enabled) { const topLabelImage = await this.renderLabel(renderWidth, paddingTop, paddingLeft, paddingRight, imageAreaH, params.fgColor, params.labelTop); - this.effector.registerTexture('topLabel', topLabelImage); + this.compositor.registerTexture('topLabel', topLabelImage); } if (params.labelBottom.enabled) { const bottomLabelImage = await this.renderLabel(renderWidth, paddingBottom, paddingLeft, paddingRight, imageAreaH, params.fgColor, params.labelBottom); - this.effector.registerTexture('bottomLabel', bottomLabelImage); + this.compositor.registerTexture('bottomLabel', bottomLabelImage); } - this.effector.changeResolution(renderWidth, renderHeight); + this.compositor.changeResolution(renderWidth, renderHeight); - await this.effector.setLayersAndRender([{ - fxId: 'frame', + this.compositor.render([{ + functionId: 'frame', id: 'a', params: { image: 'image', @@ -247,14 +242,10 @@ export class ImageFrameRenderer { }]); } - public render(): void { - this.effector.render(); - } - /* * disposeCanvas = true だとloseContextを呼ぶため、コンストラクタで渡されたcanvasも再利用不可になるので注意 */ public destroy(disposeCanvas = true): void { - this.effector.destroy(disposeCanvas); + this.compositor.destroy(disposeCanvas); } }