usbharu
/
misskey
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

wip

This commit is contained in:
syuilo 2025-11-03 18:15:57 +09:00
parent 30f5387b3b
commit 5602ee8615
28 changed files with 195 additions and 499 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
packages/frontend/src/components/MkImageFrameEditorDialog.vue
View File

@ -132,8 +132,8 @@ SPDX-License-Identifier: AGPL-3.0-only
import { ref, useTemplateRef, watch, onMounted, onUnmounted, reactive, nextTick } from 'vue';
import ExifReader from 'exifreader';
import { throttle } from 'throttle-debounce';
import type { ImageFrameParams } from '@/utility/image-frame-renderer.js';
import { ImageFrameRenderer } from '@/utility/image-frame-renderer.js';
import type { ImageFrameParams } from '@/utility/image-frame-renderer/image-frame-renderer.js';
import { ImageFrameRenderer } from '@/utility/image-frame-renderer/image-frame-renderer.js';
import { i18n } from '@/i18n.js';
import MkModalWindow from '@/components/MkModalWindow.vue';
import MkSelect from '@/components/MkSelect.vue';

4
packages/frontend/src/composables/use-uploader.ts
View File

@ -11,7 +11,7 @@ import { computed, markRaw, onMounted, onUnmounted, ref, triggerRef } from 'vue'
import ExifReader from 'exifreader';
import type { MenuItem } from '@/types/menu.js';
import type { WatermarkPreset } from '@/utility/watermark.js';
import type { ImageFrameParams, ImageFramePreset } from '@/utility/image-frame-renderer.js';
import type { ImageFrameParams, ImageFramePreset } from '@/utility/image-frame-renderer/image-frame-renderer.js';
import { genId } from '@/utility/id.js';
import { i18n } from '@/i18n.js';
import { prefer } from '@/preferences.js';
@ -20,7 +20,7 @@ import { uploadFile, UploadAbortedError } from '@/utility/drive.js';
import * as os from '@/os.js';
import { ensureSignin } from '@/i.js';
import { WatermarkRenderer } from '@/utility/watermark.js';
import { ImageFrameRenderer } from '@/utility/image-frame-renderer.js';
import { ImageFrameRenderer } from '@/utility/image-frame-renderer/image-frame-renderer.js';
export type UploaderFeatures = {
imageEditing?: boolean;

2
packages/frontend/src/preferences/def.ts
View File

@ -13,7 +13,7 @@ import type { Plugin } from '@/plugin.js';
import type { DeviceKind } from '@/utility/device-kind.js';
import type { DeckProfile } from '@/deck.js';
import type { WatermarkPreset } from '@/utility/watermark.js';
import type { ImageFramePreset } from '@/utility/image-frame-renderer.js';
import type { ImageFramePreset } from '@/utility/image-frame-renderer/image-frame-renderer.js';
import { genId } from '@/utility/id.js';
import { DEFAULT_DEVICE_KIND } from '@/utility/device-kind.js';
import { deepEqual } from '@/utility/deep-equal.js';

86
packages/frontend/src/utility/image-effector/ImageCompositor.ts
View File

@ -5,35 +5,32 @@
import { createTexture, initShaderProgram } from '../webgl.js';
export type ImageCompositorProgramParamDefs = Record<string, any>;
export type ImageCompositorFunctionParams = Record<string, any>;
export function defineImageCompositorFx<PS extends ImageCompositorProgramParamDefs, US extends string[]>(program: ImageCompositorProgram<PS, US>) {
return program;
}
export type ImageCompositorProgram<PS extends ImageCompositorProgramParamDefs = ImageCompositorProgramParamDefs, US extends string[] = string[]> = {
id: string;
export type ImageCompositorFunction<PS extends ImageCompositorFunctionParams = ImageCompositorFunctionParams> = {
shader: string;
uniforms: US;
params: PS,
main: (ctx: {
gl: WebGL2RenderingContext;
program: WebGLProgram;
params: PS;
u: Record<US[number], WebGLUniformLocation>;
u: Record<string, WebGLUniformLocation>;
width: number;
height: number;
textures: Map<string, { texture: WebGLTexture; width: number; height: number; }>;
}) => void;
};
export type ImageCompositorNode = {
export type ImageCompositorLayer = {
id: string;
programId: string;
functionId: string;
params: Record<string, any>;
};
// TODO: per node cache
export function defineImageCompositorFunction<PS extends ImageCompositorFunctionParams>(fn: ImageCompositorFunction<PS>) {
return fn;
}
// TODO: per layer cache
export class ImageCompositor {
private gl: WebGL2RenderingContext;
@ -46,7 +43,7 @@ export class ImageCompositor {
private perLayerResultFrameBuffers: Map<string, WebGLFramebuffer> = new Map();
private nopProgram: WebGLProgram;
private registeredTextures: Map<string, { texture: WebGLTexture; width: number; height: number; }> = new Map();
private programs: ImageCompositorProgram[] = [];
private registeredFunctions: Map<string, ImageCompositorFunction & { id: string; uniforms: string[] }> = new Map();
constructor(options: {
canvas: HTMLCanvasElement;
@ -116,13 +113,23 @@ export class ImageCompositor {
gl.enableVertexAttribArray(positionLocation);
}
private renderNode(node: ImageCompositorNode, preTexture: WebGLTexture, invert = false) {
private extractUniformNamesFromShader(shader: string): string[] {
const uniformRegex = /uniform\s+\w+\s+(\w+)\s*;/g;
const uniforms: string[] = [];
let match;
while ((match = uniformRegex.exec(shader)) !== null) {
uniforms.push(match[1].replace(/^u_/, ''));
}
return uniforms;
}
private renderLayer(layer: ImageCompositorLayer, preTexture: WebGLTexture, invert = false) {
const gl = this.gl;
const program = this.programs.find(p => p.id === node.programId);
if (program == null) return;
const fn = this.registeredFunctions.get(layer.functionId);
if (fn == null) return;
const cachedShader = this.shaderCache.get(program.id);
const cachedShader = this.shaderCache.get(fn.id);
const shaderProgram = cachedShader ?? initShaderProgram(this.gl, `#version 300 es
in vec2 position;
uniform bool u_invert;
@ -132,9 +139,9 @@ export class ImageCompositor {
in_uv = (position + 1.0) / 2.0;
gl_Position = u_invert ? vec4(position * vec2(1.0, -1.0), 0.0, 1.0) : vec4(position, 0.0, 1.0);
}
`, program.shader);
`, fn.shader);
if (cachedShader == null) {
this.shaderCache.set(program.id, shaderProgram);
this.shaderCache.set(fn.id, shaderProgram);
}
gl.useProgram(shaderProgram);
@ -150,11 +157,11 @@ export class ImageCompositor {
const in_texture = gl.getUniformLocation(shaderProgram, 'in_texture');
gl.uniform1i(in_texture, 0);
program.main({
fn.main({
gl: gl,
program: shaderProgram,
params: node.params,
u: Object.fromEntries(program.uniforms.map(u => [u, gl.getUniformLocation(shaderProgram, 'u_' + u)!])),
params: layer.params,
u: Object.fromEntries(fn.uniforms.map(u => [u, gl.getUniformLocation(shaderProgram, 'u_' + u)!])),
width: this.renderWidth,
height: this.renderHeight,
textures: this.registeredTextures,
@ -163,11 +170,11 @@ export class ImageCompositor {
gl.drawArrays(gl.TRIANGLES, 0, 6);
}
public render(nodes: ImageCompositorNode[]) {
public render(layers: ImageCompositorLayer[]) {
const gl = this.gl;
// 入力をそのまま出力
if (nodes.length === 0) {
if (layers.length === 0) {
gl.activeTexture(gl.TEXTURE0);
gl.bindTexture(gl.TEXTURE_2D, this.baseTexture);
@ -180,8 +187,8 @@ export class ImageCompositor {
let preTexture = this.baseTexture;
for (const layer of nodes) {
const isLast = layer === nodes.at(-1);
for (const layer of layers) {
const isLast = layer === layers.at(-1);
const cachedResultTexture = this.perLayerResultTextures.get(layer.id);
const resultTexture = cachedResultTexture ?? createTexture(gl);
@ -204,14 +211,15 @@ export class ImageCompositor {
gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, resultTexture, 0);
}
this.renderNode(layer, preTexture, isLast);
this.renderLayer(layer, preTexture, isLast);
preTexture = resultTexture;
}
}
public registerProgram(program: ImageCompositorProgram) {
this.programs.push(program);
public registerFunction(id: string, fn: ImageCompositorFunction) {
const uniforms = this.extractUniformNamesFromShader(fn.shader);
this.registeredFunctions.set(id, { ...fn, id, uniforms });
}
public registerTexture(key: string, image: ImageData | ImageBitmap | HTMLImageElement | HTMLCanvasElement) {
@ -236,6 +244,24 @@ export class ImageCompositor {
});
}
public unregisterTexture(key: string) {
const gl = this.gl;
const existing = this.registeredTextures.get(key);
if (existing != null) {
gl.deleteTexture(existing.texture);
this.registeredTextures.delete(key);
}
}
public hasTexture(key: string) {
return this.registeredTextures.has(key);
}
public getKeysOfRegisteredTextures() {
return this.registeredTextures.keys();
}
public changeResolution(width: number, height: number) {
if (this.renderWidth === width && this.renderHeight === height) return;

379
packages/frontend/src/utility/image-effector/ImageEffector.ts
View File

@ -6,7 +6,8 @@
import QRCodeStyling from 'qr-code-styling';
import { url, host } from '@@/js/config.js';
import { getProxiedImageUrl } from '../media-proxy.js';
import { createTexture, initShaderProgram } from '../webgl.js';
import { ImageCompositor } from './ImageCompositor.js';
import type { ImageCompositorFunction, ImageCompositorLayer } from './ImageCompositor.js';
import { ensureSignin } from '@/i.js';
export type ImageEffectorRGB = [r: number, g: number, b: number];
@ -94,243 +95,72 @@ export type ParamsRecordTypeToDefRecord<PS extends ImageEffectorFxParamDefs> = {
[K in keyof PS]: GetParamType<PS[K]>;
};
export function defineImageEffectorFx<ID extends string, PS extends ImageEffectorFxParamDefs, US extends string[]>(fx: ImageEffectorFx<ID, PS, US>) {
return fx;
}
export type ImageEffectorFx<ID extends string = string, PS extends ImageEffectorFxParamDefs = ImageEffectorFxParamDefs, US extends string[] = string[]> = {
id: ID;
export type ImageEffectorFxDefinition<PS extends ImageEffectorFxParamDefs = ImageEffectorFxParamDefs> = {
id: string;
name: string;
shader: string;
uniforms: US;
params: PS,
main: (ctx: {
gl: WebGL2RenderingContext;
program: WebGLProgram;
params: ParamsRecordTypeToDefRecord<PS>;
u: Record<US[number], WebGLUniformLocation>;
width: number;
height: number;
textures: Record<string, {
texture: WebGLTexture;
width: number;
height: number;
} | null>;
}) => void;
shader: string;
main: ImageCompositorFunction['main'];
};
export type ImageEffectorFx<PS extends ImageEffectorFxParamDefs = ImageEffectorFxParamDefs> = {
id: string;
name: string;
fn: ImageCompositorFunction;
params: PS,
};
export type ImageEffectorLayer = {
id: string;
fxId: string;
params: Record<string, any>;
params: ImageCompositorLayer['params'];
};
export function defineImageEffectorFx<PS extends ImageEffectorFxParamDefs>(fx: ImageEffectorFxDefinition<PS>): ImageEffectorFx<PS> {
return {
id: fx.id,
name: fx.name,
fn: {
shader: fx.shader,
main: fx.main,
},
params: fx.params,
};
}
function getValue<T extends keyof ParamTypeToPrimitive>(params: Record<string, any>, k: string): ParamTypeToPrimitive[T] {
return params[k];
}
export class ImageEffector<IEX extends ReadonlyArray<ImageEffectorFx<any, any, any>>> {
private gl: WebGL2RenderingContext;
export class ImageEffector {
private canvas: HTMLCanvasElement | null = null;
private renderWidth: number;
private renderHeight: number;
private layers: ImageEffectorLayer[] = [];
private baseTexture: WebGLTexture;
private shaderCache: Map<string, WebGLProgram> = new Map();
private perLayerResultTextures: Map<string, WebGLTexture> = new Map();
private perLayerResultFrameBuffers: Map<string, WebGLFramebuffer> = new Map();
private nopProgram: WebGLProgram;
private fxs: [...IEX];
private paramTextures: Map<string, { texture: WebGLTexture; width: number; height: number; }> = new Map();
private registeredTextures: Map<string, { texture: WebGLTexture; width: number; height: number; }> = new Map();
private fxs: ImageEffectorFx[];
private compositor: ImageCompositor;
constructor(options: {
canvas: HTMLCanvasElement;
renderWidth: number;
renderHeight: number;
image: ImageData | ImageBitmap | HTMLImageElement | HTMLCanvasElement | null;
fxs: [...IEX];
fxs: ImageEffectorFx[];
}) {
this.canvas = options.canvas;
this.renderWidth = options.renderWidth;
this.renderHeight = options.renderHeight;
this.fxs = options.fxs;
this.canvas.width = this.renderWidth;
this.canvas.height = this.renderHeight;
const gl = this.canvas.getContext('webgl2', {
preserveDrawingBuffer: false,
alpha: true,
premultipliedAlpha: false,
this.compositor = new ImageCompositor({
canvas: this.canvas,
renderWidth: options.renderWidth,
renderHeight: options.renderHeight,
image: options.image,
});
if (gl == null) throw new Error('Failed to initialize WebGL2 context');
this.gl = gl;
gl.viewport(0, 0, gl.drawingBufferWidth, gl.drawingBufferHeight);
const VERTICES = new Float32Array([-1, -1, -1, 1, 1, 1, -1, -1, 1, 1, 1, -1]);
const vertexBuffer = gl.createBuffer();
gl.bindBuffer(gl.ARRAY_BUFFER, vertexBuffer);
gl.bufferData(gl.ARRAY_BUFFER, VERTICES, gl.STATIC_DRAW);
if (options.image != null) {
this.baseTexture = createTexture(gl);
gl.activeTexture(gl.TEXTURE0);
gl.bindTexture(gl.TEXTURE_2D, this.baseTexture);
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, options.image.width, options.image.height, 0, gl.RGBA, gl.UNSIGNED_BYTE, options.image);
gl.bindTexture(gl.TEXTURE_2D, null);
} else {
this.baseTexture = createTexture(gl);
gl.activeTexture(gl.TEXTURE0);
}
this.nopProgram = initShaderProgram(this.gl, `#version 300 es
in vec2 position;
out vec2 in_uv;
void main() {
in_uv = (position + 1.0) / 2.0;
gl_Position = vec4(position * vec2(1.0, -1.0), 0.0, 1.0);
}
`, `#version 300 es
precision mediump float;
in vec2 in_uv;
uniform sampler2D u_texture;
out vec4 out_color;
void main() {
out_color = texture(u_texture, in_uv);
}
`);
// レジスタ番号はシェーダープログラムに属しているわけではなく、独立の存在なので、とりあえず nopProgram を使って設定する(その後は効果が持続する)
// ref. https://qiita.com/emadurandal/items/5966c8374f03d4de3266
const positionLocation = gl.getAttribLocation(this.nopProgram, 'position');
gl.vertexAttribPointer(positionLocation, 2, gl.FLOAT, false, 0, 0);
gl.enableVertexAttribArray(positionLocation);
}
private renderLayer(layer: ImageEffectorLayer, preTexture: WebGLTexture, invert = false) {
const gl = this.gl;
const fx = this.fxs.find(fx => fx.id === layer.fxId);
if (fx == null) return;
const cachedShader = this.shaderCache.get(fx.id);
const shaderProgram = cachedShader ?? initShaderProgram(this.gl, `#version 300 es
in vec2 position;
uniform bool u_invert;
out vec2 in_uv;
void main() {
in_uv = (position + 1.0) / 2.0;
gl_Position = u_invert ? vec4(position * vec2(1.0, -1.0), 0.0, 1.0) : vec4(position, 0.0, 1.0);
}
`, fx.shader);
if (cachedShader == null) {
this.shaderCache.set(fx.id, shaderProgram);
}
gl.useProgram(shaderProgram);
const in_resolution = gl.getUniformLocation(shaderProgram, 'in_resolution');
gl.uniform2fv(in_resolution, [this.renderWidth, this.renderHeight]);
const u_invert = gl.getUniformLocation(shaderProgram, 'u_invert');
gl.uniform1i(u_invert, invert ? 1 : 0);
gl.activeTexture(gl.TEXTURE0);
gl.bindTexture(gl.TEXTURE_2D, preTexture);
const in_texture = gl.getUniformLocation(shaderProgram, 'in_texture');
gl.uniform1i(in_texture, 0);
fx.main({
gl: gl,
program: shaderProgram,
params: Object.fromEntries(
Object.entries(fx.params as ImageEffectorFxParamDefs).map(([key, param]) => {
return [key, layer.params[key] ?? param.default];
}),
),
u: Object.fromEntries(fx.uniforms.map(u => [u, gl.getUniformLocation(shaderProgram, 'u_' + u)!])),
width: this.renderWidth,
height: this.renderHeight,
textures: Object.fromEntries(
Object.entries(fx.params as ImageEffectorFxParamDefs).map(([k, v]) => {
if (v.type === 'textureRef') {
const param = getValue<typeof v.type>(layer.params, k);
if (param == null) return [k, null];
const texture = this.registeredTextures.get(param) ?? null;
return [k, texture];
} else if (v.type === 'texture') {
const param = getValue<typeof v.type>(layer.params, k);
if (param == null) return [k, null];
const texture = this.paramTextures.get(this.getTextureKeyForParam(param)) ?? null;
return [k, texture];
} else {
return [k, null];
}
})),
});
gl.drawArrays(gl.TRIANGLES, 0, 6);
}
public render() {
const gl = this.gl;
// 入力をそのまま出力
if (this.layers.length === 0) {
gl.activeTexture(gl.TEXTURE0);
gl.bindTexture(gl.TEXTURE_2D, this.baseTexture);
gl.useProgram(this.nopProgram);
gl.uniform1i(gl.getUniformLocation(this.nopProgram, 'u_texture')!, 0);
gl.drawArrays(gl.TRIANGLES, 0, 6);
return;
}
let preTexture = this.baseTexture;
for (const layer of this.layers) {
const isLast = layer === this.layers.at(-1);
const cachedResultTexture = this.perLayerResultTextures.get(layer.id);
const resultTexture = cachedResultTexture ?? createTexture(gl);
if (cachedResultTexture == null) {
this.perLayerResultTextures.set(layer.id, resultTexture);
}
gl.bindTexture(gl.TEXTURE_2D, resultTexture);
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, this.renderWidth, this.renderHeight, 0, gl.RGBA, gl.UNSIGNED_BYTE, null);
gl.bindTexture(gl.TEXTURE_2D, null);
if (isLast) {
gl.bindFramebuffer(gl.FRAMEBUFFER, null);
} else {
const cachedResultFrameBuffer = this.perLayerResultFrameBuffers.get(layer.id);
const resultFrameBuffer = cachedResultFrameBuffer ?? gl.createFramebuffer()!;
if (cachedResultFrameBuffer == null) {
this.perLayerResultFrameBuffers.set(layer.id, resultFrameBuffer);
}
gl.bindFramebuffer(gl.FRAMEBUFFER, resultFrameBuffer);
gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, resultTexture, 0);
}
this.renderLayer(layer, preTexture, isLast);
preTexture = resultTexture;
for (const fx of this.fxs) {
this.compositor.registerFunction(fx.id, fx.fn);
}
}
public async setLayersAndRender(layers: ImageEffectorLayer[]) {
this.layers = layers;
const unused = new Set(this.paramTextures.keys());
const unused = new Set(this.compositor.getKeysOfRegisteredTextures());
for (const layer of layers) {
const fx = this.fxs.find(fx => fx.id === layer.fxId);
@ -345,62 +175,35 @@ export class ImageEffector<IEX extends ReadonlyArray<ImageEffectorFx<any, any, a
const textureKey = this.getTextureKeyForParam(v);
unused.delete(textureKey);
if (this.paramTextures.has(textureKey)) continue;
if (this.compositor.hasTexture(textureKey)) continue;
if (_DEV_) console.log(`Baking texture of <${textureKey}>...`);
const texture =
v.type === 'text' ? await createTextureFromText(this.gl, v.text) :
v.type === 'url' ? await createTextureFromUrl(this.gl, v.url) :
v.type === 'qr' ? await createTextureFromQr(this.gl, { data: v.data }) :
const image =
v.type === 'text' ? await createTextureFromText(v.text) :
v.type === 'url' ? await createTextureFromUrl(v.url) :
v.type === 'qr' ? await createTextureFromQr({ data: v.data }) :
null;
if (texture == null) continue;
if (image == null) continue;
this.paramTextures.set(textureKey, texture);
this.compositor.registerTexture(textureKey, image);
}
}
for (const k of unused) {
if (_DEV_) console.log(`Dispose unused texture <${k}>...`);
this.gl.deleteTexture(this.paramTextures.get(k)!.texture);
this.paramTextures.delete(k);
this.compositor.unregisterTexture(k);
}
this.render();
}
public registerTexture(key: string, image: ImageData | ImageBitmap | HTMLImageElement | HTMLCanvasElement) {
const gl = this.gl;
if (this.registeredTextures.has(key)) {
const existing = this.registeredTextures.get(key)!;
gl.deleteTexture(existing.texture);
this.registeredTextures.delete(key);
}
const texture = createTexture(gl);
gl.activeTexture(gl.TEXTURE0);
gl.bindTexture(gl.TEXTURE_2D, texture);
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, image.width, image.height, 0, gl.RGBA, gl.UNSIGNED_BYTE, image);
gl.bindTexture(gl.TEXTURE_2D, null);
this.registeredTextures.set(key, {
texture: texture,
width: image.width,
height: image.height,
});
this.compositor.render(layers.map(layer => ({
id: layer.id,
functionId: layer.fxId,
params: layer.params,
})));
}
public changeResolution(width: number, height: number) {
if (this.renderWidth === width && this.renderHeight === height) return;
this.renderWidth = width;
this.renderHeight = height;
if (this.canvas) {
this.canvas.width = this.renderWidth;
this.canvas.height = this.renderHeight;
}
this.gl.viewport(0, 0, this.renderWidth, this.renderHeight);
this.compositor.changeResolution(width, height);
}
private getTextureKeyForParam(v: ParamTypeToPrimitive['texture']) {
@ -417,43 +220,11 @@ export class ImageEffector<IEX extends ReadonlyArray<ImageEffectorFx<any, any, a
* disposeCanvas = true だとloseContextを呼ぶため、コンストラクタで渡されたcanvasも再利用不可になるので注意
*/
public destroy(disposeCanvas = true) {
this.gl.deleteProgram(this.nopProgram);
for (const shader of this.shaderCache.values()) {
this.gl.deleteProgram(shader);
}
this.shaderCache.clear();
for (const texture of this.perLayerResultTextures.values()) {
this.gl.deleteTexture(texture);
}
this.perLayerResultTextures.clear();
for (const framebuffer of this.perLayerResultFrameBuffers.values()) {
this.gl.deleteFramebuffer(framebuffer);
}
this.perLayerResultFrameBuffers.clear();
for (const texture of this.paramTextures.values()) {
this.gl.deleteTexture(texture.texture);
}
this.paramTextures.clear();
for (const texture of this.registeredTextures.values()) {
this.gl.deleteTexture(texture.texture);
}
this.registeredTextures.clear();
this.gl.deleteTexture(this.baseTexture);
if (disposeCanvas) {
const loseContextExt = this.gl.getExtension('WEBGL_lose_context');
if (loseContextExt) loseContextExt.loseContext();
}
this.compositor.destroy(disposeCanvas);
}
}
async function createTextureFromUrl(gl: WebGL2RenderingContext, imageUrl: string | null): Promise<{ texture: WebGLTexture, width: number, height: number } | null> {
async function createTextureFromUrl(imageUrl: string | null) {
if (imageUrl == null || imageUrl.trim() === '') return null;
const image = await new Promise<HTMLImageElement>((resolve, reject) => {
@ -465,20 +236,10 @@ async function createTextureFromUrl(gl: WebGL2RenderingContext, imageUrl: string
if (image == null) return null;
const texture = createTexture(gl);
gl.activeTexture(gl.TEXTURE0);
gl.bindTexture(gl.TEXTURE_2D, texture);
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, image.width, image.height, 0, gl.RGBA, gl.UNSIGNED_BYTE, image);
gl.bindTexture(gl.TEXTURE_2D, null);
return {
texture,
width: image.width,
height: image.height,
};
return image;
}
async function createTextureFromText(gl: WebGL2RenderingContext, text: string | null, resolution = 2048): Promise<{ texture: WebGLTexture, width: number, height: number } | null> {
async function createTextureFromText(text: string | null, resolution = 2048) {
if (text == null || text.trim() === '') return null;
const ctx = window.document.createElement('canvas').getContext('2d')!;
@ -503,24 +264,12 @@ async function createTextureFromText(gl: WebGL2RenderingContext, text: string |
const cropHeight = (Math.ceil(textMetrics.actualBoundingBoxAscent + textMetrics.actualBoundingBoxDescent) + margin + margin);
const data = ctx.getImageData(0, (ctx.canvas.height / 2) - (cropHeight / 2), ctx.canvas.width, ctx.canvas.height);
const texture = createTexture(gl);
gl.activeTexture(gl.TEXTURE0);
gl.bindTexture(gl.TEXTURE_2D, texture);
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, cropWidth, cropHeight, 0, gl.RGBA, gl.UNSIGNED_BYTE, data);
gl.bindTexture(gl.TEXTURE_2D, null);
const info = {
texture: texture,
width: cropWidth,
height: cropHeight,
};
ctx.canvas.remove();
return info;
return data;
}
async function createTextureFromQr(gl: WebGL2RenderingContext, options: { data: string | null }, resolution = 512): Promise<{ texture: WebGLTexture, width: number, height: number } | null> {
async function createTextureFromQr(options: { data: string | null }, resolution = 512) {
const $i = ensureSignin();
const qrCodeInstance = new QRCodeStyling({
@ -557,15 +306,5 @@ async function createTextureFromQr(gl: WebGL2RenderingContext, options: { data:
const image = await window.createImageBitmap(blob);
const texture = createTexture(gl);
gl.activeTexture(gl.TEXTURE0);
gl.bindTexture(gl.TEXTURE_2D, texture);
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, resolution, resolution, 0, gl.RGBA, gl.UNSIGNED_BYTE, image);
gl.bindTexture(gl.TEXTURE_2D, null);
return {
texture,
width: resolution,
height: resolution,
};
return image;
}

3
packages/frontend/src/utility/image-effector/fxs/blockNoise.ts
View File

@ -4,15 +4,14 @@
*/
import seedrandom from 'seedrandom';
import shader from './blockNoise.glsl';
import { defineImageEffectorFx } from '../ImageEffector.js';
import shader from './blockNoise.glsl';
import { i18n } from '@/i18n.js';
export const FX_blockNoise = defineImageEffectorFx({
id: 'blockNoise',
name: i18n.ts._imageEffector._fxs.glitch + ': ' + i18n.ts._imageEffector._fxs.blockNoise,
shader,
uniforms: ['amount', 'channelShift'] as const,
params: {
amount: {
label: i18n.ts._imageEffector._fxProps.amount,

1
packages/frontend/src/utility/image-effector/fxs/blur.ts
View File

@ -11,7 +11,6 @@ export const FX_blur = defineImageEffectorFx({
id: 'blur',
name: i18n.ts._imageEffector._fxs.blur,
shader,
uniforms: ['offset', 'scale', 'ellipse', 'angle', 'radius', 'samples'] as const,
params: {
offsetX: {
label: i18n.ts._imageEffector._fxProps.offset + ' X',

1
packages/frontend/src/utility/image-effector/fxs/checker.ts
View File

@ -11,7 +11,6 @@ export const FX_checker = defineImageEffectorFx({
id: 'checker',
name: i18n.ts._imageEffector._fxs.checker,
shader,
uniforms: ['angle', 'scale', 'color', 'opacity'] as const,
params: {
angle: {
label: i18n.ts._imageEffector._fxProps.angle,

1
packages/frontend/src/utility/image-effector/fxs/chromaticAberration.ts
View File

@ -11,7 +11,6 @@ export const FX_chromaticAberration = defineImageEffectorFx({
id: 'chromaticAberration',
name: i18n.ts._imageEffector._fxs.chromaticAberration,
shader,
uniforms: ['amount', 'start', 'normalize'] as const,
params: {
normalize: {
label: i18n.ts._imageEffector._fxProps.normalize,

1
packages/frontend/src/utility/image-effector/fxs/colorAdjust.ts
View File

@ -11,7 +11,6 @@ export const FX_colorAdjust = defineImageEffectorFx({
id: 'colorAdjust',
name: i18n.ts._imageEffector._fxs.colorAdjust,
shader,
uniforms: ['lightness', 'contrast', 'hue', 'brightness', 'saturation'] as const,
params: {
lightness: {
label: i18n.ts._imageEffector._fxProps.lightness,

1
packages/frontend/src/utility/image-effector/fxs/colorClamp.ts
View File

@ -11,7 +11,6 @@ export const FX_colorClamp = defineImageEffectorFx({
id: 'colorClamp',
name: i18n.ts._imageEffector._fxs.colorClamp,
shader,
uniforms: ['rMax', 'rMin', 'gMax', 'gMin', 'bMax', 'bMin'] as const,
params: {
max: {
label: i18n.ts._imageEffector._fxProps.max,

1
packages/frontend/src/utility/image-effector/fxs/colorClampAdvanced.ts
View File

@ -11,7 +11,6 @@ export const FX_colorClampAdvanced = defineImageEffectorFx({
id: 'colorClampAdvanced',
name: i18n.ts._imageEffector._fxs.colorClampAdvanced,
shader,
uniforms: ['rMax', 'rMin', 'gMax', 'gMin', 'bMax', 'bMin'] as const,
params: {
rMax: {
label: `${i18n.ts._imageEffector._fxProps.max} (${i18n.ts._imageEffector._fxProps.redComponent})`,

1
packages/frontend/src/utility/image-effector/fxs/distort.ts
View File

@ -11,7 +11,6 @@ export const FX_distort = defineImageEffectorFx({
id: 'distort',
name: i18n.ts._imageEffector._fxs.distort,
shader,
uniforms: ['phase', 'frequency', 'strength', 'direction'] as const,
params: {
direction: {
label: i18n.ts._imageEffector._fxProps.direction,

1
packages/frontend/src/utility/image-effector/fxs/fill.ts
View File

@ -11,7 +11,6 @@ export const FX_fill = defineImageEffectorFx({
id: 'fill',
name: i18n.ts._imageEffector._fxs.fill,
shader,
uniforms: ['offset', 'scale', 'ellipse', 'angle', 'color', 'opacity'] as const,
params: {
offsetX: {
label: i18n.ts._imageEffector._fxProps.offset + ' X',

98
packages/frontend/src/utility/image-effector/fxs/frame.ts
View File

@ -1,98 +0,0 @@
/*
* SPDX-FileCopyrightText: syuilo and misskey-project
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { defineImageEffectorFx } from '../ImageEffector.js';
import shader from './frame.glsl';
export const FX_frame = defineImageEffectorFx({
id: 'frame',
name: '(internal)',
shader,
uniforms: ['image', 'topLabel', 'bottomLabel', 'topLabelEnabled', 'bottomLabelEnabled', 'paddingTop', 'paddingBottom', 'paddingLeft', 'paddingRight', 'bg'] as const,
params: {
image: {
type: 'textureRef',
default: null,
},
topLabel: {
type: 'textureRef',
default: null,
},
bottomLabel: {
type: 'textureRef',
default: null,
},
topLabelEnabled: {
type: 'boolean',
default: false,
},
bottomLabelEnabled: {
type: 'boolean',
default: false,
},
paddingTop: {
type: 'number',
default: 0,
max: 1,
min: 0,
},
paddingBottom: {
type: 'number',
default: 0,
max: 1,
min: 0,
},
paddingLeft: {
type: 'number',
default: 0,
max: 1,
min: 0,
},
paddingRight: {
type: 'number',
default: 0,
max: 1,
min: 0,
},
bg: {
type: 'color',
default: [1, 1, 1],
},
},
main: ({ gl, u, params, textures }) => {
const image = textures.image;
if (image == null) return;
gl.activeTexture(gl.TEXTURE1);
gl.bindTexture(gl.TEXTURE_2D, image.texture);
gl.uniform1i(u.image, 1);
gl.uniform1i(u.topLabelEnabled, params.topLabelEnabled ? 1 : 0);
gl.uniform1i(u.bottomLabelEnabled, params.bottomLabelEnabled ? 1 : 0);
gl.uniform1f(u.paddingTop, params.paddingTop);
gl.uniform1f(u.paddingBottom, params.paddingBottom);
gl.uniform1f(u.paddingLeft, params.paddingLeft);
gl.uniform1f(u.paddingRight, params.paddingRight);
gl.uniform3f(u.bg, params.bg[0], params.bg[1], params.bg[2]);
if (params.topLabelEnabled) {
const topLabel = textures.topLabel;
if (topLabel) {
gl.activeTexture(gl.TEXTURE2);
gl.bindTexture(gl.TEXTURE_2D, topLabel.texture);
gl.uniform1i(u.topLabel, 2);
}
}
if (params.bottomLabelEnabled) {
const bottomLabel = textures.bottomLabel;
if (bottomLabel) {
gl.activeTexture(gl.TEXTURE3);
gl.bindTexture(gl.TEXTURE_2D, bottomLabel.texture);
gl.uniform1i(u.bottomLabel, 3);
}
}
},
});

1
packages/frontend/src/utility/image-effector/fxs/grayscale.ts
View File

@ -11,7 +11,6 @@ export const FX_grayscale = defineImageEffectorFx({
id: 'grayscale',
name: i18n.ts._imageEffector._fxs.grayscale,
shader,
uniforms: [] as const,
params: {
},
main: ({ gl, params }) => {

1
packages/frontend/src/utility/image-effector/fxs/invert.ts
View File

@ -11,7 +11,6 @@ export const FX_invert = defineImageEffectorFx({
id: 'invert',
name: i18n.ts._imageEffector._fxs.invert,
shader,
uniforms: ['r', 'g', 'b'] as const,
params: {
r: {
label: i18n.ts._imageEffector._fxProps.redComponent,

5
packages/frontend/src/utility/image-effector/fxs/mirror.ts
View File

@ -11,7 +11,6 @@ export const FX_mirror = defineImageEffectorFx({
id: 'mirror',
name: i18n.ts._imageEffector._fxs.mirror,
shader,
uniforms: ['h', 'v'] as const,
params: {
h: {
label: i18n.ts.horizontal,
@ -19,7 +18,7 @@ export const FX_mirror = defineImageEffectorFx({
enum: [
{ value: -1 as const, icon: 'ti ti-arrow-bar-right' },
{ value: 0 as const, icon: 'ti ti-minus-vertical' },
{ value: 1 as const, icon: 'ti ti-arrow-bar-left' }
{ value: 1 as const, icon: 'ti ti-arrow-bar-left' },
],
default: -1,
},
@ -29,7 +28,7 @@ export const FX_mirror = defineImageEffectorFx({
enum: [
{ value: -1 as const, icon: 'ti ti-arrow-bar-down' },
{ value: 0 as const, icon: 'ti ti-minus' },
{ value: 1 as const, icon: 'ti ti-arrow-bar-up' }
{ value: 1 as const, icon: 'ti ti-arrow-bar-up' },
],
default: 0,
},

1
packages/frontend/src/utility/image-effector/fxs/pixelate.ts
View File

@ -11,7 +11,6 @@ export const FX_pixelate = defineImageEffectorFx({
id: 'pixelate',
name: i18n.ts._imageEffector._fxs.pixelate,
shader,
uniforms: ['offset', 'scale', 'ellipse', 'angle', 'strength', 'samples'] as const,
params: {
offsetX: {
label: i18n.ts._imageEffector._fxProps.offset + ' X',

1
packages/frontend/src/utility/image-effector/fxs/polkadot.ts
View File

@ -12,7 +12,6 @@ export const FX_polkadot = defineImageEffectorFx({
id: 'polkadot',
name: i18n.ts._imageEffector._fxs.polkadot,
shader,
uniforms: ['angle', 'scale', 'major_radius', 'major_opacity', 'minor_divisions', 'minor_radius', 'minor_opacity', 'color'] as const,
params: {
angle: {
label: i18n.ts._imageEffector._fxProps.angle,

1
packages/frontend/src/utility/image-effector/fxs/stripe.ts
View File

@ -12,7 +12,6 @@ export const FX_stripe = defineImageEffectorFx({
id: 'stripe',
name: i18n.ts._imageEffector._fxs.stripe,
shader,
uniforms: ['angle', 'frequency', 'phase', 'threshold', 'color', 'opacity'] as const,
params: {
angle: {
label: i18n.ts._imageEffector._fxProps.angle,

3
packages/frontend/src/utility/image-effector/fxs/tearing.ts
View File

@ -4,15 +4,14 @@
*/
import seedrandom from 'seedrandom';
import shader from './tearing.glsl';
import { defineImageEffectorFx } from '../ImageEffector.js';
import shader from './tearing.glsl';
import { i18n } from '@/i18n.js';
export const FX_tearing = defineImageEffectorFx({
id: 'tearing',
name: i18n.ts._imageEffector._fxs.glitch + ': ' + i18n.ts._imageEffector._fxs.tearing,
shader,
uniforms: ['amount', 'channelShift'] as const,
params: {
amount: {
label: i18n.ts._imageEffector._fxProps.amount,

1
packages/frontend/src/utility/image-effector/fxs/threshold.ts
View File

@ -11,7 +11,6 @@ export const FX_threshold = defineImageEffectorFx({
id: 'threshold',
name: i18n.ts._imageEffector._fxs.threshold,
shader,
uniforms: ['r', 'g', 'b'] as const,
params: {
r: {
label: i18n.ts._imageEffector._fxProps.redComponent,

1
packages/frontend/src/utility/image-effector/fxs/watermarkPlacement.ts
View File

@ -10,7 +10,6 @@ export const FX_watermarkPlacement = defineImageEffectorFx({
id: 'watermarkPlacement',
name: '(internal)',
shader,
uniforms: ['opacity', 'scale', 'angle', 'cover', 'repeat', 'alignX', 'alignY', 'margin', 'repeatMargin', 'noBBoxExpansion', 'wmResolution', 'wmEnabled', 'watermark'] as const,
params: {
cover: {
type: 'boolean',

1
packages/frontend/src/utility/image-effector/fxs/zoomLines.ts
View File

@ -11,7 +11,6 @@ export const FX_zoomLines = defineImageEffectorFx({
id: 'zoomLines',
name: i18n.ts._imageEffector._fxs.zoomLines,
shader,
uniforms: ['pos', 'frequency', 'thresholdEnabled', 'threshold', 'maskSize', 'black'] as const,
params: {
x: {
label: i18n.ts._imageEffector._fxProps.centerX,

0
packages/frontend/src/utility/image-effector/fxs/frame.glsl → packages/frontend/src/utility/image-frame-renderer/frame.glsl
View File

57
packages/frontend/src/utility/image-frame-renderer/frame.ts Normal file
View File

@ -0,0 +1,57 @@
/*
* SPDX-FileCopyrightText: syuilo and misskey-project
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { defineImageCompositorFunction } from '../image-effector/ImageCompositor.js';
import shader from './frame.glsl';
export const FN_frame = defineImageCompositorFunction<{
image: string | null;
topLabel: string | null;
bottomLabel: string | null;
topLabelEnabled: boolean;
bottomLabelEnabled: boolean;
paddingTop: number;
paddingBottom: number;
paddingLeft: number;
paddingRight: number;
bg: [number, number, number];
}>({
shader,
main: ({ gl, u, params, textures }) => {
if (params.image == null) return;
const image = textures.get(params.image);
if (image == null) return;
gl.activeTexture(gl.TEXTURE1);
gl.bindTexture(gl.TEXTURE_2D, image.texture);
gl.uniform1i(u.image, 1);
gl.uniform1i(u.topLabelEnabled, params.topLabelEnabled ? 1 : 0);
gl.uniform1i(u.bottomLabelEnabled, params.bottomLabelEnabled ? 1 : 0);
gl.uniform1f(u.paddingTop, params.paddingTop);
gl.uniform1f(u.paddingBottom, params.paddingBottom);
gl.uniform1f(u.paddingLeft, params.paddingLeft);
gl.uniform1f(u.paddingRight, params.paddingRight);
gl.uniform3f(u.bg, params.bg[0], params.bg[1], params.bg[2]);
if (params.topLabelEnabled && params.topLabel != null) {
const topLabel = textures.get(params.topLabel);
if (topLabel) {
gl.activeTexture(gl.TEXTURE2);
gl.bindTexture(gl.TEXTURE_2D, topLabel.texture);
gl.uniform1i(u.topLabel, 2);
}
}
if (params.bottomLabelEnabled && params.bottomLabel != null) {
const bottomLabel = textures.get(params.bottomLabel);
if (bottomLabel) {
gl.activeTexture(gl.TEXTURE3);
gl.bindTexture(gl.TEXTURE_2D, bottomLabel.texture);
gl.uniform1i(u.bottomLabel, 3);
}
}
},
});

37
packages/frontend/src/utility/image-frame-renderer.ts → packages/frontend/src/utility/image-frame-renderer/image-frame-renderer.ts
View File

@ -6,19 +6,13 @@
import QRCodeStyling from 'qr-code-styling';
import { url } from '@@/js/config.js';
import ExifReader from 'exifreader';
import { FX_frame } from './image-effector/fxs/frame.js';
import type { ImageEffectorFx, ImageEffectorLayer } from '@/utility/image-effector/ImageEffector.js';
import { ImageCompositor } from '../image-effector/ImageCompositor.js';
import { FN_frame } from './frame.js';
import { ImageEffector } from '@/utility/image-effector/ImageEffector.js';
import { ensureSignin } from '@/i.js';
const $i = ensureSignin();
const FXS = [
FX_frame,
] as const satisfies ImageEffectorFx<string, any>[];
// TODO: 上部にもラベルを配置できるようにする
type LabelParams = {
enabled: boolean;
scale: number;
@ -45,7 +39,7 @@ export type ImageFramePreset = {
};
export class ImageFrameRenderer {
private effector: ImageEffector<typeof FXS>;
private compositor: ImageCompositor;
private image: HTMLImageElement | ImageBitmap;
private exif: ExifReader.Tags;
private renderAsPreview = false;
@ -61,15 +55,16 @@ export class ImageFrameRenderer {
this.renderAsPreview = options.renderAsPreview ?? false;
console.log(this.exif);
this.effector = new ImageEffector({
this.compositor = new ImageCompositor({
canvas: options.canvas,
renderWidth: 1,
renderHeight: 1,
image: null,
fxs: FXS,
});
this.effector.registerTexture('image', this.image);
this.compositor.registerFunction('frame', FN_frame);
this.compositor.registerTexture('image', this.image);
}
private interpolateTemplateText(text: string) {
@ -195,7 +190,7 @@ export class ImageFrameRenderer {
return labelCanvasCtx.getImageData(0, 0, labelCanvasCtx.canvas.width, labelCanvasCtx.canvas.height); ;
}
public async updateAndRender(params: ImageFrameParams): Promise<void> {
public async render(params: ImageFrameParams): Promise<void> {
let imageAreaW = this.image.width;
let imageAreaH = this.image.height;
@ -219,18 +214,18 @@ export class ImageFrameRenderer {
if (params.labelTop.enabled) {
const topLabelImage = await this.renderLabel(renderWidth, paddingTop, paddingLeft, paddingRight, imageAreaH, params.fgColor, params.labelTop);
this.effector.registerTexture('topLabel', topLabelImage);
this.compositor.registerTexture('topLabel', topLabelImage);
}
if (params.labelBottom.enabled) {
const bottomLabelImage = await this.renderLabel(renderWidth, paddingBottom, paddingLeft, paddingRight, imageAreaH, params.fgColor, params.labelBottom);
this.effector.registerTexture('bottomLabel', bottomLabelImage);
this.compositor.registerTexture('bottomLabel', bottomLabelImage);
}
this.effector.changeResolution(renderWidth, renderHeight);
this.compositor.changeResolution(renderWidth, renderHeight);
await this.effector.setLayersAndRender([{
fxId: 'frame',
this.compositor.render([{
functionId: 'frame',
id: 'a',
params: {
image: 'image',
@ -247,14 +242,10 @@ export class ImageFrameRenderer {
}]);
}
public render(): void {
this.effector.render();
}
/*
* disposeCanvas = true だとloseContextを呼ぶため、コンストラクタで渡されたcanvasも再利用不可になるので注意
*/
public destroy(disposeCanvas = true): void {
this.effector.destroy(disposeCanvas);
this.compositor.destroy(disposeCanvas);
}
}