Display images in the SVG viewport renderer via canvases instead of base64 PNGs (#2903)

* add: move images as rendered canvases to node_graph_executor

* add: added the frontend message

* fix: bytemuck stuff

* fix: canvas element breaking

* fix: width issues

* fix: remove the old message

* npm: run lint-fix

* fix

* works finally

* fix transforms

* Fix self closing tag

* fix: reuse id

* fix: have it working with repeat instance

* cargo: fmt

* fix

* Avoid "canvas" prefix to IDs

* fix

* fix: vello issue from 6111440

* fix: gpu stuff

* fix: vello bbox

* Code review

---------

Co-authored-by: hypercube <0hypercube@gmail.com>
Co-authored-by: Keavon Chambers <keavon@keavon.com>

Author: 3 people

Cargo.lock

@@ -46,6 +46,7 @@ num_enum = { workspace = true }
 usvg = { workspace = true }
 once_cell = { workspace = true }
 web-sys = { workspace = true }
+bytemuck = { workspace = true }
 
 # Required dependencies
 spin = "0.9.8"

editor/Cargo.toml

@@ -8,6 +8,7 @@ use crate::messages::portfolio::document::utility_types::wires::{WirePath, WireP
 use crate::messages::prelude::*;
 use crate::messages::tool::utility_types::HintData;
 use graph_craft::document::NodeId;
+use graphene_std::raster::Image;
 use graphene_std::raster::color::Color;
 use graphene_std::text::Font;
 
@@ -179,6 +180,9 @@ pub enum FrontendMessage {
 	UpdateDocumentArtwork {
 		svg: String,
 	},
+	UpdateImageData {
+		image_data: Vec<(u64, Image<Color>)>,
+	},
 	UpdateDocumentBarLayout {
 		#[serde(rename = "layoutTarget")]
 		layout_target: LayoutTarget,

editor/src/messages/frontend/frontend_message.rs

@@ -213,7 +213,7 @@ impl NodeGraphExecutor {
 
 	fn export(&self, node_graph_output: TaggedValue, export_config: ExportConfig, responses: &mut VecDeque<Message>) -> Result<(), String> {
 		let TaggedValue::RenderOutput(RenderOutput {
-			data: graphene_std::wasm_application_io::RenderOutputType::Svg(svg),
+			data: graphene_std::wasm_application_io::RenderOutputType::Svg { svg, .. },
 			..
 		}) = node_graph_output
 		else {
@@ -350,15 +350,16 @@ impl NodeGraphExecutor {
 		match node_graph_output {
 			TaggedValue::RenderOutput(render_output) => {
 				match render_output.data {
-					graphene_std::wasm_application_io::RenderOutputType::Svg(svg) => {
+					graphene_std::wasm_application_io::RenderOutputType::Svg { svg, image_data } => {
 						// Send to frontend
+						responses.add(FrontendMessage::UpdateImageData { image_data });
 						responses.add(FrontendMessage::UpdateDocumentArtwork { svg });
 					}
 					graphene_std::wasm_application_io::RenderOutputType::CanvasFrame(frame) => {
 						let matrix = format_transform_matrix(frame.transform);
-						let transform = if matrix.is_empty() { String::new() } else { format!(" transform=\"{}\"", matrix) };
+						let transform = if matrix.is_empty() { String::new() } else { format!(" transform=\"{matrix}\"") };
 						let svg = format!(
-							r#"<svg><foreignObject width="{}" height="{}"{transform}><div data-canvas-placeholder="canvas{}"></div></foreignObject></svg>"#,
+							r#"<svg><foreignObject width="{}" height="{}"{transform}><div data-canvas-placeholder="{}"></div></foreignObject></svg>"#,
 							frame.resolution.x, frame.resolution.y, frame.surface_id.0
 						);
 						responses.add(FrontendMessage::UpdateDocumentArtwork { svg });

editor/src/node_graph_executor.rs

@@ -192,12 +192,25 @@
 
 		const placeholders = window.document.querySelectorAll("[data-viewport] [data-canvas-placeholder]");
 		// Replace the placeholders with the actual canvas elements
-		placeholders.forEach((placeholder) => {
+		Array.from(placeholders).forEach((placeholder) => {
 			const canvasName = placeholder.getAttribute("data-canvas-placeholder");
 			if (!canvasName) return;
 			// Get the canvas element from the global storage
 			// eslint-disable-next-line @typescript-eslint/no-explicit-any
-			const canvas = (window as any).imageCanvases[canvasName];
+			let canvas = (window as any).imageCanvases[canvasName];
+
+			if (canvasName !== "0" && canvas.parentElement) {
+				var newCanvas = window.document.createElement("canvas");
+				var context = newCanvas.getContext("2d");
+
+				newCanvas.width = canvas.width;
+				newCanvas.height = canvas.height;
+
+				context?.drawImage(canvas, 0, 0);
+
+				canvas = newCanvas;
+			}
+
 			placeholder.replaceWith(canvas);
 		});
 	}

frontend/src/components/panels/Document.svelte

@@ -16,13 +16,27 @@ use editor::messages::portfolio::utility_types::Platform;
 use editor::messages::prelude::*;
 use editor::messages::tool::tool_messages::tool_prelude::WidgetId;
 use graph_craft::document::NodeId;
+use graphene_std::raster::Image;
 use graphene_std::raster::color::Color;
+use js_sys::{Object, Reflect};
 use serde::Serialize;
 use serde_wasm_bindgen::{self, from_value};
 use std::cell::RefCell;
-use std::sync::atomic::Ordering;
+use std::sync::atomic::{AtomicU64, Ordering};
 use std::time::Duration;
+use wasm_bindgen::JsCast;
 use wasm_bindgen::prelude::*;
+use web_sys::{CanvasRenderingContext2d, HtmlCanvasElement, ImageData, window};
+
+static IMAGE_DATA_HASH: AtomicU64 = AtomicU64::new(0);
+
+fn calculate_hash<T: std::hash::Hash>(t: &T) -> u64 {
+	use std::collections::hash_map::DefaultHasher;
+	use std::hash::Hasher;
+	let mut hasher = DefaultHasher::new();
+	t.hash(&mut hasher);
+	hasher.finish()
+}
 
 /// Set the random seed used by the editor by calling this from JS upon initialization.
 /// This is necessary because WASM doesn't have a random number generator.
@@ -37,6 +51,75 @@ pub fn wasm_memory() -> JsValue {
 	wasm_bindgen::memory()
 }
 
+fn render_image_data_to_canvases(image_data: &[(u64, Image<Color>)]) {
+	let window = match window() {
+		Some(window) => window,
+		None => {
+			error!("Cannot render canvas: window object not found");
+			return;
+		}
+	};
+	let document = window.document().expect("window should have a document");
+	let window_obj = Object::from(window);
+	let image_canvases_key = JsValue::from_str("imageCanvases");
+
+	let canvases_obj = match Reflect::get(&window_obj, &image_canvases_key) {
+		Ok(obj) if !obj.is_undefined() && !obj.is_null() => obj,
+		_ => {
+			let new_obj = Object::new();
+			if Reflect::set(&window_obj, &image_canvases_key, &new_obj).is_err() {
+				error!("Failed to create and set imageCanvases object on window");
+				return;
+			}
+			new_obj.into()
+		}
+	};
+	let canvases_obj = Object::from(canvases_obj);
+
+	for (placeholder_id, image) in image_data.iter() {
+		let canvas_name = placeholder_id.to_string();
+		let js_key = JsValue::from_str(&canvas_name);
+
+		if Reflect::has(&canvases_obj, &js_key).unwrap_or(false) || image.width == 0 || image.height == 0 {
+			continue;
+		}
+
+		let canvas: HtmlCanvasElement = document
+			.create_element("canvas")
+			.expect("Failed to create canvas element")
+			.dyn_into::<HtmlCanvasElement>()
+			.expect("Failed to cast element to HtmlCanvasElement");
+
+		canvas.set_width(image.width);
+		canvas.set_height(image.height);
+
+		let context: CanvasRenderingContext2d = canvas
+			.get_context("2d")
+			.expect("Failed to get 2d context")
+			.expect("2d context was not found")
+			.dyn_into::<CanvasRenderingContext2d>()
+			.expect("Failed to cast context to CanvasRenderingContext2d");
+		let u8_data: Vec<u8> = image.data.iter().flat_map(|color| color.to_rgba8_srgb()).collect();
+		let clamped_u8_data = wasm_bindgen::Clamped(&u8_data[..]);
+		match ImageData::new_with_u8_clamped_array_and_sh(clamped_u8_data, image.width, image.height) {
+			Ok(image_data_obj) => {
+				if context.put_image_data(&image_data_obj, 0., 0.).is_err() {
+					error!("Failed to put image data on canvas for id: {placeholder_id}");
+				}
+			}
+			Err(e) => {
+				error!("Failed to create ImageData for id: {placeholder_id}: {e:?}");
+			}
+		}
+
+		let js_value = JsValue::from(canvas);
+
+		if Reflect::set(&canvases_obj, &js_key, &js_value).is_err() {
+			error!("Failed to set canvas '{canvas_name}' on imageCanvases object");
+		}
+	}
+}
+
 // ============================================================================
 
 /// This struct is, via wasm-bindgen, used by JS to interact with the editor backend. It does this by calling functions, which are `impl`ed
@@ -88,6 +171,17 @@ impl EditorHandle {
 
 	// Sends a FrontendMessage to JavaScript
 	fn send_frontend_message_to_js(&self, mut message: FrontendMessage) {
+		if let FrontendMessage::UpdateImageData { ref image_data } = message {
+			let new_hash = calculate_hash(image_data);
+			let prev_hash = IMAGE_DATA_HASH.load(Ordering::Relaxed);
+
+			if new_hash != prev_hash {
+				render_image_data_to_canvases(image_data.as_slice());
+				IMAGE_DATA_HASH.store(new_hash, Ordering::Relaxed);
+			}
+			return;
+		}
+
 		if let FrontendMessage::UpdateDocumentLayerStructure { data_buffer } = message {
 			message = FrontendMessage::UpdateDocumentLayerStructureJs { data_buffer: data_buffer.into() };
 		}

frontend/wasm/src/editor_api.rs

@@ -56,6 +56,10 @@ impl AlphaBlending {
 			clip: if t < 0.5 { self.clip } else { other.clip },
 		}
 	}
+
+	pub fn opacity(&self, mask: bool) -> f32 {
+		self.opacity * if mask { 1. } else { self.fill }
+	}
 }
 
 #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]

node-graph/gcore/src/blending.rs

@@ -1,20 +1,18 @@
-use crate::GraphicGroupTable;
-pub use crate::color::*;
-use crate::raster_types::{CPU, RasterDataTable};
-use crate::vector::VectorDataTable;
-use std::fmt::Debug;
-
-#[cfg(target_arch = "spirv")]
-use spirv_std::num_traits::float::Float;
-
 /// as to not yet rename all references
 pub mod color {
 	pub use super::*;
 }
 
 pub mod image;
 
-pub use self::image::Image;
+pub use self::image::{Image, TransformImage};
+use crate::GraphicGroupTable;
+pub use crate::color::*;
+use crate::raster_types::{CPU, RasterDataTable};
+use crate::vector::VectorDataTable;
+#[cfg(target_arch = "spirv")]
+use spirv_std::num_traits::float::Float;
+use std::fmt::Debug;
 
 pub trait Bitmap {
 	type Pixel: Pixel;

node-graph/gcore/src/raster.rs

@@ -50,6 +50,13 @@ pub struct Image<P: Pixel> {
 	// TODO: Currently it is always anchored at the top left corner at (0, 0). The bottom right corner of the new origin field would correspond to (1, 1).
 }
 
+#[derive(Debug, Clone, dyn_any::DynAny, Default, PartialEq, serde::Serialize, serde::Deserialize, specta::Type)]
+pub struct TransformImage(pub DAffine2);
+
+impl Hash for TransformImage {
+	fn hash<H: std::hash::Hasher>(&self, _: &mut H) {}
+}
+
 impl<P: Pixel + Debug> Debug for Image<P> {
 	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
 		let length = self.data.len();

node-graph/gcore/src/raster/image.rs

@@ -417,7 +417,6 @@ where
 	// Create and add mirrored instance
 	for mut instance in instance.instance_iter() {
 		instance.transform = reflected_transform * instance.transform;
-		instance.source_node_id = None;
 		result_table.push(instance);
 	}
 
@@ -448,6 +447,7 @@ async fn round_corners(
 		.map(|source| {
 			let source_transform = *source.transform;
 			let source_transform_inverse = source_transform.inverse();
+			let source_node_id = source.source_node_id;
 			let source = source.instance;
 
 			let upstream_graphic_group = source.upstream_graphic_group.clone();
@@ -542,7 +542,7 @@ async fn round_corners(
 				instance: result,
 				transform: source_transform,
 				alpha_blending: Default::default(),
-				source_node_id: None,
+				source_node_id: *source_node_id,
 			}
 		})
 		.collect()
@@ -645,7 +645,6 @@ async fn box_warp(_: impl Ctx, vector_data: VectorDataTable, #[expose] rectangle
 			// Add this to the table and reset the transform since we've applied it directly to the points
 			vector_data_instance.instance = result;
 			vector_data_instance.transform = DAffine2::IDENTITY;
-			vector_data_instance.source_node_id = None;
 			vector_data_instance
 		})
 		.collect()
@@ -917,7 +916,6 @@ async fn bounding_box(_: impl Ctx, vector_data: VectorDataTable) -> VectorDataTa
 			result.style.set_stroke_transform(DAffine2::IDENTITY);
 
 			vector_data_instance.instance = result;
-			vector_data_instance.source_node_id = None;
 			vector_data_instance
 		})
 		.collect()
@@ -1013,7 +1011,6 @@ async fn offset_path(_: impl Ctx, vector_data: VectorDataTable, distance: f64, j
 			}
 
 			vector_data_instance.instance = result;
-			vector_data_instance.source_node_id = None;
 			vector_data_instance
 		})
 		.collect()
@@ -1068,7 +1065,6 @@ async fn solidify_stroke(_: impl Ctx, vector_data: VectorDataTable) -> VectorDat
 			}
 
 			vector_data_instance.instance = result;
-			vector_data_instance.source_node_id = None;
 			vector_data_instance
 		})
 		.collect()