diff --git a/packages/frontend/src/utility/image-effector/ImageCompositor.ts b/packages/frontend/src/utility/image-effector/ImageCompositor.ts new file mode 100644 index 0000000000..e2178c8b86 --- /dev/null +++ b/packages/frontend/src/utility/image-effector/ImageCompositor.ts @@ -0,0 +1,282 @@ +/* + * SPDX-FileCopyrightText: syuilo and misskey-project + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { createTexture, initShaderProgram } from '../webgl.js'; + +export type ImageCompositorFxParamDefs = Record; + +export function defineImageCompositorFx(fx: ImageCompositorFx) { + return fx; +} + +export type ImageCompositorFx = { + id: ID; + name: string; + shader: string; + uniforms: US; + params: PS, + main: (ctx: { + gl: WebGL2RenderingContext; + program: WebGLProgram; + params: PS; + u: Record; + width: number; + height: number; + textures: Map; + }) => void; +}; + +export type ImageCompositorNode = { + id: string; + fxId: string; + params: Record; +}; + +export class ImageCompositor { + private gl: WebGL2RenderingContext; + private canvas: HTMLCanvasElement | null = null; + private renderWidth: number; + private renderHeight: number; + private baseTexture: WebGLTexture; + private shaderCache: Map = new Map(); + private perLayerResultTextures: Map = new Map(); + private perLayerResultFrameBuffers: Map = new Map(); + private nopProgram: WebGLProgram; + private registeredTextures: Map = new Map(); + + constructor(options: { + canvas: HTMLCanvasElement; + renderWidth: number; + renderHeight: number; + image: ImageData | ImageBitmap | HTMLImageElement | HTMLCanvasElement | null; + }) { + this.canvas = options.canvas; + this.renderWidth = options.renderWidth; + this.renderHeight = options.renderHeight; + + this.canvas.width = this.renderWidth; + this.canvas.height = this.renderHeight; + + const gl = this.canvas.getContext('webgl2', { + preserveDrawingBuffer: false, + alpha: true, + premultipliedAlpha: false, + }); + + if (gl == null) throw new Error('Failed to initialize WebGL2 context'); + + this.gl = gl; + + gl.viewport(0, 0, gl.drawingBufferWidth, gl.drawingBufferHeight); + + const VERTICES = new Float32Array([-1, -1, -1, 1, 1, 1, -1, -1, 1, 1, 1, -1]); + const vertexBuffer = gl.createBuffer(); + gl.bindBuffer(gl.ARRAY_BUFFER, vertexBuffer); + gl.bufferData(gl.ARRAY_BUFFER, VERTICES, gl.STATIC_DRAW); + + if (options.image != null) { + this.baseTexture = createTexture(gl); + gl.activeTexture(gl.TEXTURE0); + gl.bindTexture(gl.TEXTURE_2D, this.baseTexture); + gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, options.image.width, options.image.height, 0, gl.RGBA, gl.UNSIGNED_BYTE, options.image); + gl.bindTexture(gl.TEXTURE_2D, null); + } else { + this.baseTexture = createTexture(gl); + gl.activeTexture(gl.TEXTURE0); + } + + this.nopProgram = initShaderProgram(this.gl, `#version 300 es + in vec2 position; + out vec2 in_uv; + + void main() { + in_uv = (position + 1.0) / 2.0; + gl_Position = vec4(position * vec2(1.0, -1.0), 0.0, 1.0); + } + `, `#version 300 es + precision mediump float; + + in vec2 in_uv; + uniform sampler2D u_texture; + out vec4 out_color; + + void main() { + out_color = texture(u_texture, in_uv); + } + `); + + // レジスタ番号はシェーダープログラムに属しているわけではなく、独立の存在なので、とりあえず nopProgram を使って設定する(その後は効果が持続する) + // ref. https://qiita.com/emadurandal/items/5966c8374f03d4de3266 + const positionLocation = gl.getAttribLocation(this.nopProgram, 'position'); + gl.vertexAttribPointer(positionLocation, 2, gl.FLOAT, false, 0, 0); + gl.enableVertexAttribArray(positionLocation); + } + + private renderNode(node: ImageCompositorNode, preTexture: WebGLTexture, invert = false) { + const gl = this.gl; + + const fx = this.fxs.find(fx => fx.id === node.fxId); + if (fx == null) return; + + const cachedShader = this.shaderCache.get(fx.id); + const shaderProgram = cachedShader ?? initShaderProgram(this.gl, `#version 300 es + in vec2 position; + uniform bool u_invert; + out vec2 in_uv; + + void main() { + in_uv = (position + 1.0) / 2.0; + gl_Position = u_invert ? vec4(position * vec2(1.0, -1.0), 0.0, 1.0) : vec4(position, 0.0, 1.0); + } + `, fx.shader); + if (cachedShader == null) { + this.shaderCache.set(fx.id, shaderProgram); + } + + gl.useProgram(shaderProgram); + + const in_resolution = gl.getUniformLocation(shaderProgram, 'in_resolution'); + gl.uniform2fv(in_resolution, [this.renderWidth, this.renderHeight]); + + const u_invert = gl.getUniformLocation(shaderProgram, 'u_invert'); + gl.uniform1i(u_invert, invert ? 1 : 0); + + gl.activeTexture(gl.TEXTURE0); + gl.bindTexture(gl.TEXTURE_2D, preTexture); + const in_texture = gl.getUniformLocation(shaderProgram, 'in_texture'); + gl.uniform1i(in_texture, 0); + + fx.main({ + gl: gl, + program: shaderProgram, + params: node.params, + u: Object.fromEntries(fx.uniforms.map(u => [u, gl.getUniformLocation(shaderProgram, 'u_' + u)!])), + width: this.renderWidth, + height: this.renderHeight, + textures: this.registeredTextures, + }); + + gl.drawArrays(gl.TRIANGLES, 0, 6); + } + + public render(nodes: ImageCompositorNode[]) { + const gl = this.gl; + + // 入力をそのまま出力 + if (nodes.length === 0) { + gl.activeTexture(gl.TEXTURE0); + gl.bindTexture(gl.TEXTURE_2D, this.baseTexture); + + gl.useProgram(this.nopProgram); + gl.uniform1i(gl.getUniformLocation(this.nopProgram, 'u_texture')!, 0); + + gl.drawArrays(gl.TRIANGLES, 0, 6); + return; + } + + let preTexture = this.baseTexture; + + for (const layer of nodes) { + const isLast = layer === nodes.at(-1); + + const cachedResultTexture = this.perLayerResultTextures.get(layer.id); + const resultTexture = cachedResultTexture ?? createTexture(gl); + if (cachedResultTexture == null) { + this.perLayerResultTextures.set(layer.id, resultTexture); + } + gl.bindTexture(gl.TEXTURE_2D, resultTexture); + gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, this.renderWidth, this.renderHeight, 0, gl.RGBA, gl.UNSIGNED_BYTE, null); + gl.bindTexture(gl.TEXTURE_2D, null); + + if (isLast) { + gl.bindFramebuffer(gl.FRAMEBUFFER, null); + } else { + const cachedResultFrameBuffer = this.perLayerResultFrameBuffers.get(layer.id); + const resultFrameBuffer = cachedResultFrameBuffer ?? gl.createFramebuffer()!; + if (cachedResultFrameBuffer == null) { + this.perLayerResultFrameBuffers.set(layer.id, resultFrameBuffer); + } + gl.bindFramebuffer(gl.FRAMEBUFFER, resultFrameBuffer); + gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, resultTexture, 0); + } + + this.renderNode(layer, preTexture, isLast); + + preTexture = resultTexture; + } + } + + public registerFx(fx: ImageCompositorFx) { + this.fxs.push(fx); + } + + public registerTexture(key: string, image: ImageData | ImageBitmap | HTMLImageElement | HTMLCanvasElement) { + const gl = this.gl; + + if (this.registeredTextures.has(key)) { + const existing = this.registeredTextures.get(key)!; + gl.deleteTexture(existing.texture); + this.registeredTextures.delete(key); + } + + const texture = createTexture(gl); + gl.activeTexture(gl.TEXTURE0); + gl.bindTexture(gl.TEXTURE_2D, texture); + gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, image.width, image.height, 0, gl.RGBA, gl.UNSIGNED_BYTE, image); + gl.bindTexture(gl.TEXTURE_2D, null); + + this.registeredTextures.set(key, { + texture: texture, + width: image.width, + height: image.height, + }); + } + + public changeResolution(width: number, height: number) { + if (this.renderWidth === width && this.renderHeight === height) return; + + this.renderWidth = width; + this.renderHeight = height; + if (this.canvas) { + this.canvas.width = this.renderWidth; + this.canvas.height = this.renderHeight; + } + this.gl.viewport(0, 0, this.renderWidth, this.renderHeight); + } + + /* + * disposeCanvas = true だとloseContextを呼ぶため、コンストラクタで渡されたcanvasも再利用不可になるので注意 + */ + public destroy(disposeCanvas = true) { + this.gl.deleteProgram(this.nopProgram); + + for (const shader of this.shaderCache.values()) { + this.gl.deleteProgram(shader); + } + this.shaderCache.clear(); + + for (const texture of this.perLayerResultTextures.values()) { + this.gl.deleteTexture(texture); + } + this.perLayerResultTextures.clear(); + + for (const framebuffer of this.perLayerResultFrameBuffers.values()) { + this.gl.deleteFramebuffer(framebuffer); + } + this.perLayerResultFrameBuffers.clear(); + + for (const texture of this.registeredTextures.values()) { + this.gl.deleteTexture(texture.texture); + } + this.registeredTextures.clear(); + + this.gl.deleteTexture(this.baseTexture); + + if (disposeCanvas) { + const loseContextExt = this.gl.getExtension('WEBGL_lose_context'); + if (loseContextExt) loseContextExt.loseContext(); + } + } +}