diff --git a/package.json b/package.json index 30dba62b743..9cc82f94bdc 100644 --- a/package.json +++ b/package.json @@ -61,7 +61,7 @@ "dependencies": { "@babel/runtime": "^7.12.5", "@matrix-org/analytics-events": "^0.5.0", - "@matrix-org/matrix-wysiwyg": "^2.0.0", + "@matrix-org/matrix-wysiwyg": "^2.2.2", "@matrix-org/react-sdk-module-api": "^0.0.5", "@sentry/browser": "^7.0.0", "@sentry/tracing": "^7.0.0", diff --git a/src/components/views/rooms/wysiwyg_composer/SendWysiwygComposer.tsx b/src/components/views/rooms/wysiwyg_composer/SendWysiwygComposer.tsx index d12432481db..34307ce4abd 100644 --- a/src/components/views/rooms/wysiwyg_composer/SendWysiwygComposer.tsx +++ b/src/components/views/rooms/wysiwyg_composer/SendWysiwygComposer.tsx @@ -57,11 +57,10 @@ export default function SendWysiwygComposer({ isRichTextEnabled, e2eStatus, menuPosition, - eventRelation, ...props }: SendWysiwygComposerProps): JSX.Element { const Composer = isRichTextEnabled ? WysiwygComposer : PlainTextComposer; - const defaultContextValue = useRef(getDefaultContextValue({ eventRelation })); + const defaultContextValue = useRef(getDefaultContextValue({ eventRelation: props.eventRelation })); return ( diff --git a/src/components/views/rooms/wysiwyg_composer/components/PlainTextComposer.tsx b/src/components/views/rooms/wysiwyg_composer/components/PlainTextComposer.tsx index c6abc1230b4..efc4971657d 100644 --- a/src/components/views/rooms/wysiwyg_composer/components/PlainTextComposer.tsx +++ b/src/components/views/rooms/wysiwyg_composer/components/PlainTextComposer.tsx @@ -15,6 +15,7 @@ limitations under the License. */ import classNames from "classnames"; +import { IEventRelation } from "matrix-js-sdk/src/matrix"; import React, { MutableRefObject, ReactNode } from "react"; import { useComposerFunctions } from "../hooks/useComposerFunctions"; @@ -36,6 +37,7 @@ interface PlainTextComposerProps { leftComponent?: ReactNode; rightComponent?: ReactNode; children?: (ref: MutableRefObject, composerFunctions: ComposerFunctions) => ReactNode; + eventRelation?: IEventRelation; } export function PlainTextComposer({ diff --git a/src/components/views/rooms/wysiwyg_composer/components/WysiwygComposer.tsx b/src/components/views/rooms/wysiwyg_composer/components/WysiwygComposer.tsx index 56f79c94a87..c4c50b8392c 100644 --- a/src/components/views/rooms/wysiwyg_composer/components/WysiwygComposer.tsx +++ b/src/components/views/rooms/wysiwyg_composer/components/WysiwygComposer.tsx @@ -15,6 +15,7 @@ limitations under the License. */ import React, { memo, MutableRefObject, ReactNode, useEffect, useRef } from "react"; +import { IEventRelation } from "matrix-js-sdk/src/matrix"; import { useWysiwyg, FormattingFunctions } from "@matrix-org/matrix-wysiwyg"; import classNames from "classnames"; @@ -40,6 +41,7 @@ interface WysiwygComposerProps { leftComponent?: ReactNode; rightComponent?: ReactNode; children?: (ref: MutableRefObject, wysiwyg: FormattingFunctions) => ReactNode; + eventRelation?: IEventRelation; } export const WysiwygComposer = memo(function WysiwygComposer({ @@ -52,11 +54,12 @@ export const WysiwygComposer = memo(function WysiwygComposer({ leftComponent, rightComponent, children, + eventRelation, }: WysiwygComposerProps) { const { room } = useRoomContext(); const autocompleteRef = useRef(null); - const inputEventProcessor = useInputEventProcessor(onSend, autocompleteRef, initialContent); + const inputEventProcessor = useInputEventProcessor(onSend, autocompleteRef, initialContent, eventRelation); const { ref, isWysiwygReady, content, actionStates, wysiwyg, suggestion } = useWysiwyg({ initialContent, inputEventProcessor, diff --git a/src/components/views/rooms/wysiwyg_composer/hooks/useInputEventProcessor.ts b/src/components/views/rooms/wysiwyg_composer/hooks/useInputEventProcessor.ts index 9b41227ba3b..27f40880140 100644 --- a/src/components/views/rooms/wysiwyg_composer/hooks/useInputEventProcessor.ts +++ b/src/components/views/rooms/wysiwyg_composer/hooks/useInputEventProcessor.ts @@ -16,7 +16,7 @@ limitations under the License. import { Wysiwyg, WysiwygEvent } from "@matrix-org/matrix-wysiwyg"; import { useCallback } from "react"; -import { MatrixClient } from "matrix-js-sdk/src/matrix"; +import { IEventRelation, MatrixClient } from "matrix-js-sdk/src/matrix"; import { useSettingValue } from "../../../../../hooks/useSettings"; import { getKeyBindingsManager } from "../../../../../KeyBindingsManager"; @@ -34,11 +34,15 @@ import { getEventsFromEditorStateTransfer, getEventsFromRoom } from "../utils/ev import { endEditing } from "../utils/editing"; import Autocomplete from "../../Autocomplete"; import { handleEventWithAutocomplete } from "./utils"; +import ContentMessages from "../../../../../ContentMessages"; +import { getBlobSafeMimeType } from "../../../../../utils/blobs"; +import { isNotNull } from "../../../../../Typeguards"; export function useInputEventProcessor( onSend: () => void, autocompleteRef: React.RefObject, initialContent?: string, + eventRelation?: IEventRelation, ): (event: WysiwygEvent, composer: Wysiwyg, editor: HTMLElement) => WysiwygEvent | null { const roomContext = useRoomContext(); const composerContext = useComposerContext(); @@ -47,10 +51,6 @@ export function useInputEventProcessor( return useCallback( (event: WysiwygEvent, composer: Wysiwyg, editor: HTMLElement) => { - if (event instanceof ClipboardEvent) { - return event; - } - const send = (): void => { event.stopPropagation?.(); event.preventDefault?.(); @@ -61,6 +61,21 @@ export function useInputEventProcessor( onSend(); }; + // this is required to handle edge case image pasting in Safari, see + // https://github.com/vector-im/element-web/issues/25327 and it is caught by the + // `beforeinput` listener attached to the composer + const isInputEventForClipboard = + event instanceof InputEvent && event.inputType === "insertFromPaste" && isNotNull(event.dataTransfer); + const isClipboardEvent = event instanceof ClipboardEvent; + + const shouldHandleAsClipboardEvent = isClipboardEvent || isInputEventForClipboard; + + if (shouldHandleAsClipboardEvent) { + const data = isClipboardEvent ? event.clipboardData : event.dataTransfer; + const handled = handleClipboardEvent(event, data, roomContext, mxClient, eventRelation); + return handled ? null : event; + } + const isKeyboardEvent = event instanceof KeyboardEvent; if (isKeyboardEvent) { return handleKeyboardEvent( @@ -78,7 +93,16 @@ export function useInputEventProcessor( return handleInputEvent(event, send, isCtrlEnterToSend); } }, - [isCtrlEnterToSend, onSend, initialContent, roomContext, composerContext, mxClient, autocompleteRef], + [ + isCtrlEnterToSend, + onSend, + initialContent, + roomContext, + composerContext, + mxClient, + autocompleteRef, + eventRelation, + ], ); } @@ -220,3 +244,88 @@ function handleInputEvent(event: InputEvent, send: Send, isCtrlEnterToSend: bool return event; } + +/** + * Takes an event and handles image pasting. Returns a boolean to indicate if it has handled + * the event or not. Must accept either clipboard or input events in order to prevent issue: + * https://github.com/vector-im/element-web/issues/25327 + * + * @param event - event to process + * @param roomContext - room in which the event occurs + * @param mxClient - current matrix client + * @param eventRelation - used to send the event to the correct place eg timeline vs thread + * @returns - boolean to show if the event was handled or not + */ +export function handleClipboardEvent( + event: ClipboardEvent | InputEvent, + data: DataTransfer | null, + roomContext: IRoomState, + mxClient: MatrixClient, + eventRelation?: IEventRelation, +): boolean { + // Logic in this function follows that of `SendMessageComposer.onPaste` + const { room, timelineRenderingType, replyToEvent } = roomContext; + + function handleError(error: unknown): void { + if (error instanceof Error) { + console.log(error.message); + } else if (typeof error === "string") { + console.log(error); + } + } + + if (event.type !== "paste" || data === null || room === undefined) { + return false; + } + + // Prioritize text on the clipboard over files if RTF is present as Office on macOS puts a bitmap + // in the clipboard as well as the content being copied. Modern versions of Office seem to not do this anymore. + // We check text/rtf instead of text/plain as when copy+pasting a file from Finder or Gnome Image Viewer + // it puts the filename in as text/plain which we want to ignore. + if (data.files.length && !data.types.includes("text/rtf")) { + ContentMessages.sharedInstance() + .sendContentListToRoom(Array.from(data.files), room.roomId, eventRelation, mxClient, timelineRenderingType) + .catch(handleError); + return true; + } + + // Safari `Insert from iPhone or iPad` + // data.getData("text/html") returns a string like: + if (data.types.includes("text/html")) { + const imgElementStr = data.getData("text/html"); + const parser = new DOMParser(); + const imgDoc = parser.parseFromString(imgElementStr, "text/html"); + + if ( + imgDoc.getElementsByTagName("img").length !== 1 || + !imgDoc.querySelector("img")?.src.startsWith("blob:") || + imgDoc.childNodes.length !== 1 + ) { + handleError("Failed to handle pasted content as Safari inserted content"); + return false; + } + const imgSrc = imgDoc.querySelector("img")!.src; + + fetch(imgSrc) + .then((response) => { + response + .blob() + .then((imgBlob) => { + const type = imgBlob.type; + const safetype = getBlobSafeMimeType(type); + const ext = type.split("/")[1]; + const parts = response.url.split("/"); + const filename = parts[parts.length - 1]; + const file = new File([imgBlob], filename + "." + ext, { type: safetype }); + ContentMessages.sharedInstance() + .sendContentToRoom(file, room.roomId, eventRelation, mxClient, replyToEvent) + .catch(handleError); + }) + .catch(handleError); + }) + .catch(handleError); + return true; + } + + return false; +} diff --git a/test/components/views/rooms/wysiwyg_composer/hooks/useInputEventProcessor-test.tsx b/test/components/views/rooms/wysiwyg_composer/hooks/useInputEventProcessor-test.tsx new file mode 100644 index 00000000000..8d6f9d19cc4 --- /dev/null +++ b/test/components/views/rooms/wysiwyg_composer/hooks/useInputEventProcessor-test.tsx @@ -0,0 +1,287 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +import { IEventRelation, MatrixEvent } from "matrix-js-sdk/src/matrix"; +import { waitFor } from "@testing-library/react"; + +import { handleClipboardEvent } from "../../../../../../src/components/views/rooms/wysiwyg_composer/hooks/useInputEventProcessor"; +import { TimelineRenderingType } from "../../../../../../src/contexts/RoomContext"; +import { mkStubRoom, stubClient } from "../../../../../test-utils"; +import ContentMessages from "../../../../../../src/ContentMessages"; +import { IRoomState } from "../../../../../../src/components/structures/RoomView"; + +const mockClient = stubClient(); +const mockRoom = mkStubRoom("mock room", "mock room", mockClient); +const mockRoomState = { + room: mockRoom, + timelineRenderingType: TimelineRenderingType.Room, + replyToEvent: {} as unknown as MatrixEvent, +} as unknown as IRoomState; + +const sendContentListToRoomSpy = jest.spyOn(ContentMessages.sharedInstance(), "sendContentListToRoom"); +const sendContentToRoomSpy = jest.spyOn(ContentMessages.sharedInstance(), "sendContentToRoom"); +const fetchSpy = jest.spyOn(window, "fetch"); +const logSpy = jest.spyOn(console, "log").mockImplementation(() => {}); + +describe("handleClipboardEvent", () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + afterAll(() => { + jest.restoreAllMocks(); + }); + + function createMockClipboardEvent(props: any): ClipboardEvent { + return { clipboardData: { files: [], types: [] }, ...props } as ClipboardEvent; + } + + it("returns false if it is not a paste event", () => { + const originalEvent = createMockClipboardEvent({ type: "copy" }); + const output = handleClipboardEvent(originalEvent, originalEvent.clipboardData, mockRoomState, mockClient); + + expect(output).toBe(false); + }); + + it("returns false if clipboard data is null", () => { + const originalEvent = createMockClipboardEvent({ type: "paste", clipboardData: null }); + const output = handleClipboardEvent(originalEvent, originalEvent.clipboardData, mockRoomState, mockClient); + + expect(output).toBe(false); + }); + + it("returns false if room is undefined", () => { + const originalEvent = createMockClipboardEvent({ type: "paste" }); + const { room, ...roomStateWithoutRoom } = mockRoomState; + const output = handleClipboardEvent( + originalEvent, + originalEvent.clipboardData, + roomStateWithoutRoom, + mockClient, + ); + + expect(output).toBe(false); + }); + + it("returns false if room clipboardData files and types are empty", () => { + const originalEvent = createMockClipboardEvent({ + type: "paste", + clipboardData: { files: [], types: [] }, + }); + const output = handleClipboardEvent(originalEvent, originalEvent.clipboardData, mockRoomState, mockClient); + expect(output).toBe(false); + }); + + it("handles event and calls sendContentListToRoom when data files are present", () => { + const originalEvent = createMockClipboardEvent({ + type: "paste", + clipboardData: { files: ["something here"], types: [] }, + }); + const output = handleClipboardEvent(originalEvent, originalEvent.clipboardData, mockRoomState, mockClient); + + expect(sendContentListToRoomSpy).toHaveBeenCalledTimes(1); + expect(sendContentListToRoomSpy).toHaveBeenCalledWith( + originalEvent.clipboardData?.files, + mockRoom.roomId, + undefined, // this is the event relation, an optional arg + mockClient, + mockRoomState.timelineRenderingType, + ); + expect(output).toBe(true); + }); + + it("calls sendContentListToRoom with eventRelation when present", () => { + const originalEvent = createMockClipboardEvent({ + type: "paste", + clipboardData: { files: ["something here"], types: [] }, + }); + const mockEventRelation = {} as unknown as IEventRelation; + const output = handleClipboardEvent( + originalEvent, + originalEvent.clipboardData, + mockRoomState, + mockClient, + mockEventRelation, + ); + + expect(sendContentListToRoomSpy).toHaveBeenCalledTimes(1); + expect(sendContentListToRoomSpy).toHaveBeenCalledWith( + originalEvent.clipboardData?.files, + mockRoom.roomId, + mockEventRelation, // this is the event relation, an optional arg + mockClient, + mockRoomState.timelineRenderingType, + ); + expect(output).toBe(true); + }); + + it("calls the error handler when sentContentListToRoom errors", async () => { + const mockErrorMessage = "something went wrong"; + sendContentListToRoomSpy.mockRejectedValueOnce(new Error(mockErrorMessage)); + + const originalEvent = createMockClipboardEvent({ + type: "paste", + clipboardData: { files: ["something here"], types: [] }, + }); + const mockEventRelation = {} as unknown as IEventRelation; + const output = handleClipboardEvent( + originalEvent, + originalEvent.clipboardData, + mockRoomState, + mockClient, + mockEventRelation, + ); + + expect(sendContentListToRoomSpy).toHaveBeenCalledTimes(1); + await waitFor(() => { + expect(logSpy).toHaveBeenCalledWith(mockErrorMessage); + }); + expect(output).toBe(true); + }); + + it("calls the error handler when data types has text/html but data can not be parsed", () => { + const originalEvent = createMockClipboardEvent({ + type: "paste", + clipboardData: { + files: [], + types: ["text/html"], + getData: jest.fn().mockReturnValue("
invalid html"), + }, + }); + const mockEventRelation = {} as unknown as IEventRelation; + const output = handleClipboardEvent( + originalEvent, + originalEvent.clipboardData, + mockRoomState, + mockClient, + mockEventRelation, + ); + + expect(logSpy).toHaveBeenCalledWith("Failed to handle pasted content as Safari inserted content"); + expect(output).toBe(false); + }); + + it("calls fetch when data types has text/html and data can parsed", () => { + const originalEvent = createMockClipboardEvent({ + type: "paste", + clipboardData: { + files: [], + types: ["text/html"], + getData: jest.fn().mockReturnValue(``), + }, + }); + const mockEventRelation = {} as unknown as IEventRelation; + handleClipboardEvent(originalEvent, originalEvent.clipboardData, mockRoomState, mockClient, mockEventRelation); + + expect(fetchSpy).toHaveBeenCalledTimes(1); + expect(fetchSpy).toHaveBeenCalledWith("blob:"); + }); + + it("calls error handler when fetch fails", async () => { + const mockErrorMessage = "fetch failed"; + fetchSpy.mockRejectedValueOnce(mockErrorMessage); + const originalEvent = createMockClipboardEvent({ + type: "paste", + clipboardData: { + files: [], + types: ["text/html"], + getData: jest.fn().mockReturnValue(``), + }, + }); + const mockEventRelation = {} as unknown as IEventRelation; + const output = handleClipboardEvent( + originalEvent, + originalEvent.clipboardData, + mockRoomState, + mockClient, + mockEventRelation, + ); + + await waitFor(() => { + expect(logSpy).toHaveBeenCalledWith(mockErrorMessage); + }); + expect(output).toBe(true); + }); + + it("calls sendContentToRoom when parsing is successful", async () => { + fetchSpy.mockResolvedValueOnce({ + url: "test/file", + blob: () => { + return Promise.resolve({ type: "image/jpeg" } as Blob); + }, + } as Response); + + const originalEvent = createMockClipboardEvent({ + type: "paste", + clipboardData: { + files: [], + types: ["text/html"], + getData: jest.fn().mockReturnValue(``), + }, + }); + const mockEventRelation = {} as unknown as IEventRelation; + const output = handleClipboardEvent( + originalEvent, + originalEvent.clipboardData, + mockRoomState, + mockClient, + mockEventRelation, + ); + + await waitFor(() => { + expect(sendContentToRoomSpy).toHaveBeenCalledWith( + expect.any(File), + mockRoom.roomId, + mockEventRelation, + mockClient, + mockRoomState.replyToEvent, + ); + }); + expect(output).toBe(true); + }); + + it("calls error handler when parsing is not successful", async () => { + fetchSpy.mockResolvedValueOnce({ + url: "test/file", + blob: () => { + return Promise.resolve({ type: "image/jpeg" } as Blob); + }, + } as Response); + const mockErrorMessage = "sendContentToRoom failed"; + sendContentToRoomSpy.mockRejectedValueOnce(mockErrorMessage); + + const originalEvent = createMockClipboardEvent({ + type: "paste", + clipboardData: { + files: [], + types: ["text/html"], + getData: jest.fn().mockReturnValue(``), + }, + }); + const mockEventRelation = {} as unknown as IEventRelation; + const output = handleClipboardEvent( + originalEvent, + originalEvent.clipboardData, + mockRoomState, + mockClient, + mockEventRelation, + ); + + await waitFor(() => { + expect(logSpy).toHaveBeenCalledWith(mockErrorMessage); + }); + expect(output).toBe(true); + }); +}); diff --git a/yarn.lock b/yarn.lock index 80345b92458..82baee84e67 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1598,10 +1598,10 @@ resolved "https://registry.yarnpkg.com/@matrix-org/matrix-sdk-crypto-js/-/matrix-sdk-crypto-js-0.1.0-alpha.8.tgz#18dd8e7fb56602d2999d8a502b49e902a2bb3782" integrity sha512-hdmbbGXKrN6JNo3wdBaR5Zs3lXlzllT3U43ViNTlabB3nKkOZQnEAN/Isv+4EQSgz1+8897veI9Q8sqlQX22oA== -"@matrix-org/matrix-wysiwyg@^2.0.0": - version "2.2.1" - resolved "https://registry.yarnpkg.com/@matrix-org/matrix-wysiwyg/-/matrix-wysiwyg-2.2.1.tgz#076b409c0ffe655938d663863b1ee546a7101da6" - integrity sha512-QF4dJsyqBMxZx+GhSdSiRSDIuwE5dxd7vffQ5i6hf67bd0EbVvtf4PzWmNopGHA+ckjMJIc5X1EPT+6DG/wM6Q== +"@matrix-org/matrix-wysiwyg@^2.2.2": + version "2.2.2" + resolved "https://registry.yarnpkg.com/@matrix-org/matrix-wysiwyg/-/matrix-wysiwyg-2.2.2.tgz#911d0a9858a5a4b620f93777085daac8eff6a220" + integrity sha512-FprkgKiqEHoFUfaamKwTGBENqDxbORFgoPjiE1b9yPS3hgRswobVKRl4qrXgVgFj4qQ7gWeTqogiyrHXkm1myw== "@matrix-org/olm@https://gitlab.matrix.org/api/v4/projects/27/packages/npm/@matrix-org/olm/-/@matrix-org/olm-3.2.14.tgz": version "3.2.14"