From ae07461324a3a97fd428036d472528f364f4603a Mon Sep 17 00:00:00 2001 From: Micah Galizia Date: Tue, 18 Jun 2024 21:51:09 -0400 Subject: [PATCH 1/3] feat: generic thing rendering --- .../ContentEditor/ContentEditor.tsx | 4 +- .../RemoteDisplayComponent.tsx | 2 + packages/mui/src/utils/contentworker.ts | 61 +++++------ packages/mui/src/utils/drawing.ts | 101 +++++++++++------- 4 files changed, 97 insertions(+), 71 deletions(-) diff --git a/packages/mui/src/components/ContentEditor/ContentEditor.tsx b/packages/mui/src/components/ContentEditor/ContentEditor.tsx index a905631..de20cee 100644 --- a/packages/mui/src/components/ContentEditor/ContentEditor.tsx +++ b/packages/mui/src/components/ContentEditor/ContentEditor.tsx @@ -8,7 +8,7 @@ import React, { } from "react"; import { useDispatch, useSelector } from "react-redux"; import { AppReducerState } from "../../reducers/AppReducer"; -import { getRect } from "../../utils/drawing"; +import { getRect, newSelectedRegion } from "../../utils/drawing"; import { Rect } from "../../utils/geometry"; import { MouseStateMachine } from "../../utils/mousestatemachine"; import { setCallback } from "../../utils/statemachine"; @@ -660,6 +660,7 @@ const ContentEditor = ({ const background = drawBG ? `${apiUrl}/${bContent}` : undefined; const angle = scene.angle || 0; + const things = scene.viewport ? [newSelectedRegion(scene.viewport)] : []; worker.postMessage({ cmd: "update", @@ -668,6 +669,7 @@ const ContentEditor = ({ overlay, bearer, angle, + things, }, }); } diff --git a/packages/mui/src/components/RemoteDisplayComponent/RemoteDisplayComponent.tsx b/packages/mui/src/components/RemoteDisplayComponent/RemoteDisplayComponent.tsx index 4534059..7a7cba7 100644 --- a/packages/mui/src/components/RemoteDisplayComponent/RemoteDisplayComponent.tsx +++ b/packages/mui/src/components/RemoteDisplayComponent/RemoteDisplayComponent.tsx @@ -10,6 +10,7 @@ import { useNavigate } from "react-router-dom"; import { Box } from "@mui/material"; import { setupOffscreenCanvas } from "../../utils/offscreencanvas"; import { debounce } from "lodash"; +import { Thing } from "../../utils/drawing"; /** * Table state sent to display client by websocket. A partial Scene. @@ -25,6 +26,7 @@ export interface TableState { // TODO UNION MICAH DON"T SKIP NOW export type TableUpdate = TableState & { bearer: string; + things?: Thing[]; }; interface WSStateMessage { diff --git a/packages/mui/src/utils/contentworker.ts b/packages/mui/src/utils/contentworker.ts index 109776a..f96a07c 100644 --- a/packages/mui/src/utils/contentworker.ts +++ b/packages/mui/src/utils/contentworker.ts @@ -1,5 +1,6 @@ import { TableUpdate } from "../components/RemoteDisplayComponent/RemoteDisplayComponent"; import { LoadProgress, loadImage } from "./content"; +import { Drawable, newDrawableThing } from "./drawing"; import { Point, Rect, @@ -43,6 +44,8 @@ const _img: Rect = { x: 0, y: 0, width: 0, height: 0 }; // viewport const _vp: Rect = { x: 0, y: 0, width: 0, height: 0 }; +const _things: Drawable[] = []; + // rotated image width and height - cached to avoid recalculation after load let _fullRotW: number; let _fullRotH: number; @@ -216,7 +219,8 @@ function renderAllCanvasses(background: ImageBitmap | null) { if (background) { sizeVisibleCanvasses(_canvas.width, _canvas.height); renderImage(backgroundCtx, [background], _angle); - renderImage(overlayCtx, [fullCtx.canvas, thingCtx.canvas], _angle); + renderThings(thingCtx); + renderImage(overlayCtx, [thingCtx.canvas, fullCtx.canvas], _angle); } } @@ -335,27 +339,15 @@ function renderBox( renderImage(overlayCtx, [fullCtx.canvas, thingCtx.canvas], _angle); } -function renderDottedBox(rect: Rect, ctx: OffscreenCanvasRenderingContext2D) { - const [x, y] = [rect.x, rect.y]; - const [x1, y1] = [x + rect.width, y + rect.height]; - ctx.beginPath(); - ctx.lineWidth = 10; - ctx.strokeStyle = "black"; - ctx.setLineDash([]); - ctx.moveTo(x, y); - ctx.lineTo(x1, y); - ctx.lineTo(x1, y1); - ctx.lineTo(x, y1); - ctx.lineTo(x, y); - ctx.stroke(); - ctx.strokeStyle = "white"; - ctx.setLineDash([10, 10]); - ctx.moveTo(x, y); - ctx.lineTo(x1, y); - ctx.lineTo(x1, y1); - ctx.lineTo(x, y1); - ctx.lineTo(x, y); - ctx.stroke(); +function renderThings(ctx: OffscreenCanvasRenderingContext2D) { + ctx.clearRect(0, 0, ctx.canvas.width, ctx.canvas.height); + for (const thing of _things) { + try { + thing.draw(ctx); + } catch (err) { + console.error(err); + } + } } function clearBox(x1: number, y1: number, x2: number, y2: number) { @@ -452,12 +444,20 @@ function adjustZoom(zoom: number, x: number, y: number) { } async function update(values: TableUpdate) { - const { angle, bearer, background, overlay, viewport } = values; + const { angle, bearer, background, overlay, viewport, things } = values; if (!background) { console.error(`Ignoring update without background`); return; } _angle = angle; + _things.length = 0; + if (things) { + things + .map((thing) => newDrawableThing(thing)) + .filter((thing) => thing) + .forEach((thing) => (thing ? _things.push(thing) : null)); + } + try { const [bgImg, ovImg] = await loadAllImages(bearer, background, overlay); if (!bgImg) return; @@ -735,13 +735,14 @@ self.onmessage = async (evt) => { postMessage({ cmd: "viewport", viewport: fullVp }); break; } - case "set_highlighted_rect": { - const rect = evt.data.rect; - thingCtx.clearRect(0, 0, thingCtx.canvas.width, thingCtx.canvas.height); - if (rect) renderDottedBox(rect, thingCtx); - renderImage(overlayCtx, [fullCtx.canvas, thingCtx.canvas], _angle); - break; - } + // case "set_highlighted_rect": { + // _things.push(new SelectedRegion(evt.data.rect)); + // if (thingCtx) { + // renderThings(thingCtx); + // renderImage(overlayCtx, [fullCtx.canvas, thingCtx.canvas], _angle); + // } + // break; + // } case "zoom_in": { let zoom = _zoom; if (!_zoom) zoom = _max_zoom; diff --git a/packages/mui/src/utils/drawing.ts b/packages/mui/src/utils/drawing.ts index 166474f..3bd24c7 100644 --- a/packages/mui/src/utils/drawing.ts +++ b/packages/mui/src/utils/drawing.ts @@ -1,6 +1,65 @@ -import { Rect, calculateBounds, rotatedWidthAndHeight } from "./geometry"; +import { Rect } from "./geometry"; -export const CONTROLS_HEIGHT = 46; +export type DrawContext = CanvasDrawPath & + CanvasPathDrawingStyles & + CanvasFillStrokeStyles & + CanvasPath; + +export interface Drawable { + draw(ctx: DrawContext): void; +} + +export type Thing = SelectedRegion | Marker; + +export type SelectedRegion = { + base: "SelectedRegion"; + rect: Rect; +}; + +export type Marker = { + base: "Marker"; + value: "hi"; +}; + +export function newSelectedRegion(rect: Rect): SelectedRegion { + return { + base: "SelectedRegion", + rect: rect, + }; +} + +class DrawableSelectedRegion implements Drawable { + region: SelectedRegion; + constructor(region: SelectedRegion) { + this.region = region; + } + draw(ctx: DrawContext) { + const [x, y] = [this.region.rect.x, this.region.rect.y]; + const [x1, y1] = [x + this.region.rect.width, y + this.region.rect.height]; + ctx.beginPath(); + ctx.lineWidth = 10; + ctx.strokeStyle = "black"; + ctx.setLineDash([]); + ctx.moveTo(x, y); + ctx.lineTo(x1, y); + ctx.lineTo(x1, y1); + ctx.lineTo(x, y1); + ctx.lineTo(x, y); + ctx.stroke(); + ctx.strokeStyle = "white"; + ctx.setLineDash([10, 10]); + ctx.moveTo(x, y); + ctx.lineTo(x1, y); + ctx.lineTo(x1, y1); + ctx.lineTo(x, y1); + ctx.lineTo(x, y); + ctx.stroke(); + } +} + +export function newDrawableThing(thing: Thing): Drawable | undefined { + if (thing.base === "SelectedRegion") return new DrawableSelectedRegion(thing); +} export function getRect(x1: number, y1: number, x2: number, y2: number): Rect { let x: number; @@ -23,41 +82,3 @@ export function getRect(x1: number, y1: number, x2: number, y2: number): Rect { } return { x: x, y: y, width: w, height: h }; } - -export function renderViewPort( - ctx: CanvasRenderingContext2D, - image: ImageBitmap, - angle: number, - viewport: Rect, -) { - const [rv_w, rv_h] = rotatedWidthAndHeight( - angle, - viewport.width, - viewport.height, - ); - const bounds = calculateBounds( - ctx.canvas.width, - ctx.canvas.height, - rv_w, - rv_h, - ); - const [b_x, b_y] = [bounds.width, bounds.height]; - const [c_x, c_y] = rotatedWidthAndHeight(-angle, b_x, b_y); - ctx.clearRect(0, 0, ctx.canvas.width, ctx.canvas.height); - ctx.save(); - ctx.translate(ctx.canvas.width / 2, ctx.canvas.height / 2); - ctx.rotate((angle * Math.PI) / 180); - ctx.drawImage( - image, - viewport.x, - viewport.y, - viewport.width, - viewport.height, - -c_x / 2, - -c_y / 2, - c_x, - c_y, - ); - ctx.restore(); - return; -} From f4cc965e8290642ccbd0d8b14faaadf74dcbb57c Mon Sep 17 00:00:00 2001 From: Micah Galizia Date: Tue, 18 Jun 2024 23:08:22 -0400 Subject: [PATCH 2/3] feat: fix up selection rendering --- .../ContentEditor/ContentEditor.tsx | 22 +++++++++----- packages/mui/src/utils/contentworker.ts | 30 ++++++++++++++----- packages/mui/src/utils/geometry.ts | 9 ++++++ 3 files changed, 45 insertions(+), 16 deletions(-) diff --git a/packages/mui/src/components/ContentEditor/ContentEditor.tsx b/packages/mui/src/components/ContentEditor/ContentEditor.tsx index de20cee..450c391 100644 --- a/packages/mui/src/components/ContentEditor/ContentEditor.tsx +++ b/packages/mui/src/components/ContentEditor/ContentEditor.tsx @@ -9,7 +9,7 @@ import React, { import { useDispatch, useSelector } from "react-redux"; import { AppReducerState } from "../../reducers/AppReducer"; import { getRect, newSelectedRegion } from "../../utils/drawing"; -import { Rect } from "../../utils/geometry"; +import { Rect, equalRects } from "../../utils/geometry"; import { MouseStateMachine } from "../../utils/mousestatemachine"; import { setCallback } from "../../utils/statemachine"; import styles from "./ContentEditor.module.css"; @@ -101,6 +101,7 @@ const ContentEditor = ({ const [worker, setWorker] = useState(); const [downloads] = useState>({}); const [downloadProgress, setDownloadProgress] = useState(0); + // selection is sized relative to the visible canvas size -- not the full background size const [selection, setSelection] = useState(null); /** @@ -441,9 +442,6 @@ const ContentEditor = ({ v.y === bg.y && v.width === bg.width && v.height === bg.height; - if (!zoomedOut) { - worker.postMessage({ cmd: "set_highlighted_rect", rect: v }); - } if (zoomedOut !== internalState.zoom) return; internalState.zoom = !zoomedOut; redrawToolbar(); @@ -507,7 +505,6 @@ const ContentEditor = ({ }); setCallback(sm, "remoteZoomOut", () => { const imgRect = getRect(0, 0, imageSize[0], imageSize[1]); - worker.postMessage({ cmd: "set_highlighted_rect" }); dispatch({ type: "content/zoom", payload: { backgroundSize: imgRect, viewport: imgRect }, @@ -638,6 +635,7 @@ const ContentEditor = ({ // update the revisions and trigger rendering if a revision has changed let drawBG = bRev > bgRev; let drawOV = oRev > ovRev; + let drawTH = false; if (drawBG) setBgRev(bRev); // takes effect next render cycle if (drawOV) setOvRev(oRev); // takes effect next render cycle @@ -652,15 +650,23 @@ const ContentEditor = ({ drawOV = scene.overlayContent !== undefined; } + if (scene.viewport) drawTH = true; + // if we have nothing new to draw then cheese it - if (!drawBG && !drawOV) return; + if (!drawBG && !drawOV && !drawTH) return; - if (drawBG) { + if (drawBG || drawTH) { const overlay = drawOV ? `${apiUrl}/${oContent}` : undefined; const background = drawBG ? `${apiUrl}/${bContent}` : undefined; const angle = scene.angle || 0; - const things = scene.viewport ? [newSelectedRegion(scene.viewport)] : []; + // add the viewport as a selected region if it exists and isn't just the entire background + const things = + scene.viewport && + scene.backgroundSize && + !equalRects(scene.viewport, scene.backgroundSize) + ? [newSelectedRegion(scene.viewport)] + : []; worker.postMessage({ cmd: "update", diff --git a/packages/mui/src/utils/contentworker.ts b/packages/mui/src/utils/contentworker.ts index f96a07c..f7e5094 100644 --- a/packages/mui/src/utils/contentworker.ts +++ b/packages/mui/src/utils/contentworker.ts @@ -1,6 +1,6 @@ import { TableUpdate } from "../components/RemoteDisplayComponent/RemoteDisplayComponent"; import { LoadProgress, loadImage } from "./content"; -import { Drawable, newDrawableThing } from "./drawing"; +import { Drawable, Thing, newDrawableThing } from "./drawing"; import { Point, Rect, @@ -443,20 +443,34 @@ function adjustZoom(zoom: number, x: number, y: number) { renderAllCanvasses(backgroundImage); } +function updateThings(things?: Thing[], render = false) { + // clear the existing thing list + _things.length = 0; + + // cheese it if there are no things to render + if (!things) return; + + // map things to drawable things + things + .map((thing) => newDrawableThing(thing)) + .filter((thing) => thing) + .forEach((thing) => (thing ? _things.push(thing) : null)); + + // render if we're asked (avoided in cases of subsequent full renders) + if (!render) return; + renderThings(thingCtx); + renderImage(overlayCtx, [thingCtx.canvas, fullCtx.canvas], _angle); +} + async function update(values: TableUpdate) { const { angle, bearer, background, overlay, viewport, things } = values; if (!background) { + if (things) return updateThings(things, true); console.error(`Ignoring update without background`); return; } _angle = angle; - _things.length = 0; - if (things) { - things - .map((thing) => newDrawableThing(thing)) - .filter((thing) => thing) - .forEach((thing) => (thing ? _things.push(thing) : null)); - } + updateThings(things); try { const [bgImg, ovImg] = await loadAllImages(bearer, background, overlay); diff --git a/packages/mui/src/utils/geometry.ts b/packages/mui/src/utils/geometry.ts index a707e6d..8be2490 100644 --- a/packages/mui/src/utils/geometry.ts +++ b/packages/mui/src/utils/geometry.ts @@ -273,3 +273,12 @@ export function fillRotatedViewport( height: h * silkScale, }; } + +export function equalRects(r1: Rect, r2: Rect): boolean { + return ( + r1.x === r2.x && + r1.y === r2.y && + r1.width === r2.width && + r1.height === r2.height + ); +} From 56e44f84367d2235c875659d5e10ff3aad38d467 Mon Sep 17 00:00:00 2001 From: Micah Galizia Date: Tue, 18 Jun 2024 23:10:20 -0400 Subject: [PATCH 3/3] chore: remove unneeded code --- packages/mui/src/utils/contentworker.ts | 8 -------- 1 file changed, 8 deletions(-) diff --git a/packages/mui/src/utils/contentworker.ts b/packages/mui/src/utils/contentworker.ts index f7e5094..2e27fcd 100644 --- a/packages/mui/src/utils/contentworker.ts +++ b/packages/mui/src/utils/contentworker.ts @@ -749,14 +749,6 @@ self.onmessage = async (evt) => { postMessage({ cmd: "viewport", viewport: fullVp }); break; } - // case "set_highlighted_rect": { - // _things.push(new SelectedRegion(evt.data.rect)); - // if (thingCtx) { - // renderThings(thingCtx); - // renderImage(overlayCtx, [fullCtx.canvas, thingCtx.canvas], _angle); - // } - // break; - // } case "zoom_in": { let zoom = _zoom; if (!_zoom) zoom = _max_zoom;