Support Insert from iPhone or iPad in Safari (#10851)

Co-authored-by: Michael Telatynski <[email protected]>

src/components/views/rooms/BasicMessageComposer.tsx

@@ -15,7 +15,7 @@ limitations under the License.
 */
 
 import classNames from "classnames";
-import React, { createRef, ClipboardEvent } from "react";
+import React, { createRef, ClipboardEvent, SyntheticEvent } from "react";
 import { Room } from "matrix-js-sdk/src/models/room";
 import { MatrixEvent } from "matrix-js-sdk/src/models/event";
 import EMOTICON_REGEX from "emojibase-regex/emoticon";
@@ -108,7 +108,7 @@ interface IProps {
     disabled?: boolean;
 
     onChange?(selection?: Caret, inputType?: string, diff?: IDiff): void;
-    onPaste?(event: ClipboardEvent<HTMLDivElement>, model: EditorModel): boolean;
+    onPaste?(event: Event | SyntheticEvent, data: DataTransfer, model: EditorModel): boolean;
 }
 
 interface IState {
@@ -355,18 +355,18 @@ export default class BasicMessageEditor extends React.Component<IProps, IState>
         this.onCutCopy(event, "cut");
     };
 
-    private onPaste = (event: ClipboardEvent<HTMLDivElement>): boolean | undefined => {
+    private onPasteHandler = (event: Event | SyntheticEvent, data: DataTransfer): boolean | undefined => {
         event.preventDefault(); // we always handle the paste ourselves
         if (!this.editorRef.current) return;
-        if (this.props.onPaste?.(event, this.props.model)) {
+        if (this.props.onPaste?.(event, data, this.props.model)) {
             // to prevent double handling, allow props.onPaste to skip internal onPaste
             return true;
         }
 
         const { model } = this.props;
         const { partCreator } = model;
-        const plainText = event.clipboardData.getData("text/plain");
-        const partsText = event.clipboardData.getData("application/x-element-composer");
+        const plainText = data.getData("text/plain");
+        const partsText = data.getData("application/x-element-composer");
 
         let parts: Part[];
         if (partsText) {
@@ -387,6 +387,21 @@ export default class BasicMessageEditor extends React.Component<IProps, IState>
         }
     };
 
+    private onPaste = (event: ClipboardEvent<HTMLDivElement>): boolean | undefined => {
+        return this.onPasteHandler(event, event.clipboardData);
+    };
+
+    private onBeforeInput = (event: InputEvent): void => {
+        // ignore any input while doing IME compositions
+        if (this.isIMEComposing) {
+            return;
+        }
+
+        if (event.inputType === "insertFromPaste" && event.dataTransfer) {
+            this.onPasteHandler(event, event.dataTransfer);
+        }
+    };
+
     private onInput = (event: Partial<InputEvent>): void => {
         if (!this.editorRef.current) return;
         // ignore any input while doing IME compositions
@@ -703,6 +718,7 @@ export default class BasicMessageEditor extends React.Component<IProps, IState>
 
     public componentWillUnmount(): void {
         document.removeEventListener("selectionchange", this.onSelectionChange);
+        this.editorRef.current?.removeEventListener("beforeinput", this.onBeforeInput, true);
         this.editorRef.current?.removeEventListener("input", this.onInput, true);
         this.editorRef.current?.removeEventListener("compositionstart", this.onCompositionStart, true);
         this.editorRef.current?.removeEventListener("compositionend", this.onCompositionEnd, true);
@@ -728,6 +744,7 @@ export default class BasicMessageEditor extends React.Component<IProps, IState>
         this.updateEditorState(this.getInitialCaretPosition());
         // attach input listener by hand so React doesn't proxy the events,
         // as the proxied event doesn't support inputType, which we need.
+        this.editorRef.current?.addEventListener("beforeinput", this.onBeforeInput, true);
         this.editorRef.current?.addEventListener("input", this.onInput, true);
         this.editorRef.current?.addEventListener("compositionstart", this.onCompositionStart, true);
         this.editorRef.current?.addEventListener("compositionend", this.onCompositionEnd, true);

src/components/views/rooms/SendMessageComposer.tsx

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import React, { ClipboardEvent, createRef, KeyboardEvent } from "react";
+import React, { createRef, KeyboardEvent, SyntheticEvent } from "react";
 import EMOJI_REGEX from "emojibase-regex";
 import { IContent, MatrixEvent, IEventRelation, IMentions } from "matrix-js-sdk/src/models/event";
 import { DebouncedFunc, throttle } from "lodash";
@@ -61,6 +61,7 @@ import { addReplyToMessageContent } from "../../../utils/Reply";
 import { doMaybeLocalRoomAction } from "../../../utils/local-room";
 import { Caret } from "../../../editor/caret";
 import { IDiff } from "../../../editor/diff";
+import { getBlobSafeMimeType } from "../../../utils/blobs";
 
 /**
  * Build the mentions information based on the editor model (and any related events):
@@ -667,15 +668,14 @@ export class SendMessageComposer extends React.Component<ISendMessageComposerPro
         }
     };
 
-    private onPaste = (event: ClipboardEvent<HTMLDivElement>): boolean => {
-        const { clipboardData } = event;
+    private onPaste = (event: Event | SyntheticEvent, data: DataTransfer): boolean => {
         // Prioritize text on the clipboard over files if RTF is present as Office on macOS puts a bitmap
         // in the clipboard as well as the content being copied. Modern versions of Office seem to not do this anymore.
         // We check text/rtf instead of text/plain as when copy+pasting a file from Finder or Gnome Image Viewer
         // it puts the filename in as text/plain which we want to ignore.
-        if (clipboardData.files.length && !clipboardData.types.includes("text/rtf")) {
+        if (data.files.length && !data.types.includes("text/rtf")) {
             ContentMessages.sharedInstance().sendContentListToRoom(
-                Array.from(clipboardData.files),
+                Array.from(data.files),
                 this.props.room.roomId,
                 this.props.relation,
                 this.props.mxClient,
@@ -684,6 +684,57 @@ export class SendMessageComposer extends React.Component<ISendMessageComposerPro
             return true; // to skip internal onPaste handler
         }
 
+        // Safari `Insert from iPhone or iPad`
+        // data.getData("text/html") returns a string like: <img src="blob:https://...">
+        if (data.types.includes("text/html")) {
+            const imgElementStr = data.getData("text/html");
+            const parser = new DOMParser();
+            const imgDoc = parser.parseFromString(imgElementStr, "text/html");
+
+            if (
+                imgDoc.getElementsByTagName("img").length !== 1 ||
+                !imgDoc.querySelector("img")?.src.startsWith("blob:") ||
+                imgDoc.childNodes.length !== 1
+            ) {
+                console.log("Failed to handle pasted content as Safari inserted content");
+
+                // Fallback to internal onPaste handler
+                return false;
+            }
+            const imgSrc = imgDoc!.querySelector("img")!.src;
+
+            fetch(imgSrc).then(
+                (response) => {
+                    response.blob().then(
+                        (imgBlob) => {
+                            const type = imgBlob.type;
+                            const safetype = getBlobSafeMimeType(type);
+                            const ext = type.split("/")[1];
+                            const parts = response.url.split("/");
+                            const filename = parts[parts.length - 1];
+                            const file = new File([imgBlob], filename + "." + ext, { type: safetype });
+                            ContentMessages.sharedInstance().sendContentToRoom(
+                                file,
+                                this.props.room.roomId,
+                                this.props.relation,
+                                this.props.mxClient,
+                                this.context.replyToEvent,
+                            );
+                        },
+                        (error) => {
+                            console.log(error);
+                        },
+                    );
+                },
+                (error) => {
+                    console.log(error);
+                },
+            );
+
+            // Skip internal onPaste handler
+            return true;
+        }
+
         return false;
     };