feat: partially support multimodal_text

pionxzh · Oct 16, 2023 · 7d5c7de · 7d5c7de
1 parent 0aafea4
commit 7d5c7de
Show file tree

Hide file tree

Showing 6 changed files with 41 additions and 4 deletions.
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -1,6 +1,6 @@
 {
   "editor.codeActionsOnSave": {
-    "source.fixAll": false,
+    "source.fixAll": "explicit",
     "source.fixAll.eslint": true, // this allows ESLint to auto fix on save
     "source.organizeImports": false
   },

diff --git a/packages/userscript/src/api.ts b/packages/userscript/src/api.ts
@@ -39,6 +39,14 @@ interface MessageMeta {
 
 export type AuthorRole = 'system' | 'assistant' | 'user' | 'tool'
 
+interface MultiModalInputImage {
+    asset_pointer: string
+    content_type: 'image_asset_pointer' & (string & {})
+    height: number
+    size_bytes: number
+    width: number
+}
+
 export interface ConversationNodeMessage {
     author: {
         role: AuthorRole
@@ -70,6 +78,10 @@ export interface ConversationNodeMessage {
         content_type: 'tether_browsing_display'
         result: string
         summary?: string
+    } | {
+        // multi-modal input
+        content_type: 'multimodal_text'
+        parts: Array<MultiModalInputImage | string>
     }
     create_time: number
     end_turn: boolean

diff --git a/packages/userscript/src/exporter/html.ts b/packages/userscript/src/exporter/html.ts
@@ -109,8 +109,15 @@ const transformContent = (
             }
             return ''
         }
+        case 'multimodal_text': {
+            return content.parts?.map((part) => {
+                if (typeof part === 'string') return part
+                if (part.asset_pointer) return `![image](${part.asset_pointer})`
+                return '[Unsupported multimodal content]'
+            }).join('\n') || ''
+        }
         default:
-            return ''
+            return '[Unsupported Content]'
     }
 }
 

diff --git a/packages/userscript/src/exporter/markdown.ts b/packages/userscript/src/exporter/markdown.ts
@@ -104,8 +104,15 @@ const transformContent = (
             }
             return ''
         }
+        case 'multimodal_text': {
+            return content.parts?.map((part) => {
+                if (typeof part === 'string') return part
+                if (part.asset_pointer) return `![image](${part.asset_pointer})`
+                return '[Unsupported multimodal content]'
+            }).join('\n') || ''
+        }
         default:
-            return ''
+            return '[Unsupported Content]'
     }
 }
 

diff --git a/packages/userscript/src/exporter/text.ts b/packages/userscript/src/exporter/text.ts
@@ -47,8 +47,15 @@ const transformContent = (
             }
             return ''
         }
+        case 'multimodal_text': {
+            return content.parts?.map((part) => {
+                if (typeof part === 'string') return part
+                if (part.asset_pointer) return `![image](${part.asset_pointer})`
+                return '[Unsupported multimodal content]'
+            }).join('\n') || ''
+        }
         default:
-            return ''
+            return '[Unsupported Content]'
     }
 }
 

diff --git a/packages/userscript/src/page.ts b/packages/userscript/src/page.ts
@@ -4,12 +4,15 @@ import { getBase64FromImageUrl, getBase64FromImg } from './utils/dom'
 declare global {
     interface Window {
         __NEXT_DATA__?: {
+            assetPrefix: string
             buildId: string
             gssp: boolean
             isFallback: boolean
             page: string
             props: {
                 pageProps: {
+                    allowBrowserStorage: boolean
+                    canManageBrowserStorage: boolean
                     geoOk: boolean
                     isUserInCanPayGroup: boolean
                     serviceAnnouncement: {
@@ -27,6 +30,7 @@ declare global {
                         name: string
                         picture: string
                     }
+                    userCountry: string
                     serverResponse?: {
                         type: 'data'
                         data: any // Basically ApiConversation