Display other mimes if ipywidget cannot be displayed for exiting nb

package.json

@@ -1974,7 +1974,7 @@
                 "mimeTypes": [
                     "application/vnd.jupyter.widget-view+json"
                 ],
-                "requiresMessaging": "optional"
+                "requiresMessaging": "always"
             },
             {
                 "id": "jupyter-error-renderer",

src/test/datascience/ipywidgets/incompatibleWidgetHandler.unit.test.ts

@@ -2,7 +2,7 @@
 // Licensed under the MIT License.
 
 import { assert } from 'chai';
-import { warnAboutWidgetVersionsThatAreNotSupported } from '../../../webviews/webview-side/ipywidgets/common/incompatibleWidgetHandler';
+import { warnAboutWidgetVersionsThatAreNotSupported } from '../../../webviews/webview-side/ipywidgets/kernel/incompatibleWidgetHandler';
 
 /* eslint-disable , @typescript-eslint/no-explicit-any */
 suite('Incompatible Widgets', () => {

src/test/webviews/webview-side/ipywidgets/common/scriptManager.unit.test.ts

@@ -9,10 +9,10 @@ import { Disposable } from 'vscode';
 import { disposeAllDisposables } from '../../../../../platform/common/helpers';
 import { IDisposable } from '../../../../../platform/common/types';
 import { IInteractiveWindowMapping, IPyWidgetMessages } from '../../../../../messageTypes';
-import { scriptsAlreadyRegisteredInRequireJs } from '../../../../../webviews/webview-side/ipywidgets/common/requirejsRegistry';
-import { ScriptManager } from '../../../../../webviews/webview-side/ipywidgets/common/scriptManager';
 import { IMessageHandler, PostOffice } from '../../../../../webviews/webview-side/react-common/postOffice';
 import { sleep } from '../../../../core';
+import { scriptsAlreadyRegisteredInRequireJs } from '../../../../../webviews/webview-side/ipywidgets/kernel/requirejsRegistry';
+import { ScriptManager } from '../../../../../webviews/webview-side/ipywidgets/kernel/scriptManager';
 
 suite('IPyWidget Script Manager', () => {
     let scriptManager: ScriptManager;

src/webviews/extension-side/ipywidgets/rendererComms.ts

@@ -0,0 +1,105 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+import type * as nbformat from '@jupyterlab/nbformat';
+import type { IKernelConnection } from '@jupyterlab/services/lib/kernel/kernel';
+import { IIOPubMessage, IOPubMessageType } from '@jupyterlab/services/lib/kernel/messages';
+import { injectable, inject } from 'inversify';
+import { Disposable, NotebookDocument, NotebookEditor, NotebookRendererMessaging, notebooks } from 'vscode';
+import { IKernel, IKernelProvider } from '../../../kernels/types';
+import { IExtensionSyncActivationService } from '../../../platform/activation/types';
+import { WIDGET_MIMETYPE } from '../../../platform/common/constants';
+import { disposeAllDisposables } from '../../../platform/common/helpers';
+import { IDisposable } from '../../../platform/common/types';
+import { noop } from '../../../platform/common/utils/misc';
+
+type WidgetData = {
+    model_id: string;
+};
+
+type QueryWidgetStateCommand = { command: 'query-widget-state'; model_id: string };
+
+@injectable()
+export class IPyWidgetRendererComms implements IExtensionSyncActivationService {
+    private readonly disposables: IDisposable[] = [];
+    constructor(@inject(IKernelProvider) private readonly kernelProvider: IKernelProvider) {}
+    private readonly widgetOutputsPerNotebook = new WeakMap<NotebookDocument, Set<string>>();
+    public dispose() {
+        disposeAllDisposables(this.disposables);
+    }
+    activate() {
+        const comms = notebooks.createRendererMessaging('jupyter-ipywidget-renderer');
+        comms.onDidReceiveMessage(this.onDidReceiveMessage.bind(this, comms), this, this.disposables);
+        this.kernelProvider.onDidStartKernel(this.onDidStartKernel, this, this.disposables);
+    }
+    private onDidStartKernel(e: IKernel) {
+        this.hookupKernel(e);
+        e.onStarted(() => this.hookupKernel(e), this, this.disposables);
+        e.onRestarted(() => this.hookupKernel(e), this, this.disposables);
+    }
+    private hookupKernel(kernel: IKernel) {
+        this.widgetOutputsPerNotebook.delete(kernel.notebook);
+        const previousKernelConnection = kernel.session?.kernel;
+        const iopubMessage = kernel.session?.kernel?.iopubMessage;
+        if (!iopubMessage) {
+            return;
+        }
+
+        // eslint-disable-next-line @typescript-eslint/no-require-imports
+        const jupyterLab = require('@jupyterlab/services') as typeof import('@jupyterlab/services');
+        const handler = (kernelConnection: IKernelConnection, msg: IIOPubMessage<IOPubMessageType>) => {
+            if (kernelConnection !== previousKernelConnection) {
+                // Must be some old message from a previous kernel (before a restart or the like.)
+                return;
+            }
+
+            if (
+                jupyterLab.KernelMessage.isDisplayDataMsg(msg) ||
+                jupyterLab.KernelMessage.isUpdateDisplayDataMsg(msg) ||
+                jupyterLab.KernelMessage.isExecuteReplyMsg(msg) ||
+                jupyterLab.KernelMessage.isExecuteResultMsg(msg)
+            ) {
+                this.trackModelId(kernel.notebook, msg);
+            }
+        };
+        iopubMessage.connect(handler);
+        this.disposables.push(new Disposable(() => iopubMessage.disconnect(handler)));
+    }
+    private trackModelId(
+        notebook: NotebookDocument,
+        msg: {
+            content: {
+                data: nbformat.IMimeBundle;
+            };
+        }
+    ) {
+        const output = msg.content;
+        if (output.data && typeof output.data === 'object' && WIDGET_MIMETYPE in output.data) {
+            const widgetData = output.data[WIDGET_MIMETYPE] as WidgetData;
+            if (widgetData && 'model_id' in widgetData) {
+                const set = this.widgetOutputsPerNotebook.get(notebook) || new Set<string>();
+                set.add(widgetData.model_id);
+                this.widgetOutputsPerNotebook.set(notebook, set);
+            }
+        }
+    }
+    private onDidReceiveMessage(
+        comms: NotebookRendererMessaging,
+        { editor, message }: { editor: NotebookEditor; message: QueryWidgetStateCommand }
+    ) {
+        if (message && typeof message === 'object' && message.command === 'query-widget-state') {
+            this.queryWidgetState(comms, editor, message);
+        }
+    }
+    private queryWidgetState(
+        comms: NotebookRendererMessaging,
+        editor: NotebookEditor,
+        message: QueryWidgetStateCommand
+    ) {
+        const availableModels = this.widgetOutputsPerNotebook.get(editor.notebook);
+        const available = !!availableModels?.has(message.model_id);
+        comms
+            .postMessage({ command: 'query-widget-state', model_id: message.model_id, available }, editor)
+            .then(noop, noop);
+    }
+}

src/webviews/extension-side/serviceRegistry.node.ts

@@ -18,6 +18,7 @@ import {
     IJupyterVariableDataProvider,
     IJupyterVariableDataProviderFactory
 } from './dataviewer/types';
+import { IPyWidgetRendererComms } from './ipywidgets/rendererComms';
 import { PlotViewer } from './plotting/plotViewer.node';
 import { PlotViewerProvider } from './plotting/plotViewerProvider';
 import { IPlotViewer, IPlotViewerProvider } from './plotting/types';
@@ -60,6 +61,10 @@ export function registerTypes(serviceManager: IServiceManager) {
         IExtensionSyncActivationService,
         VariableViewActivationService
     );
+    serviceManager.addSingleton<IExtensionSyncActivationService>(
+        IExtensionSyncActivationService,
+        IPyWidgetRendererComms
+    );
     serviceManager.addSingleton<IVariableViewProvider>(IVariableViewProvider, VariableViewProvider);
     serviceManager.add<IJupyterVariableDataProvider>(IJupyterVariableDataProvider, JupyterVariableDataProvider);
     serviceManager.addSingleton<IJupyterVariableDataProviderFactory>(

src/webviews/extension-side/serviceRegistry.web.ts

@@ -29,6 +29,7 @@ import { PlotSaveHandler } from './plotView/plotSaveHandler';
 import { PlotViewHandler } from './plotView/plotViewHandler';
 import { RendererCommunication } from './plotView/rendererCommunication';
 import { IPlotSaveHandler } from './plotView/types';
+import { IPyWidgetRendererComms } from './ipywidgets/rendererComms';
 
 export function registerTypes(serviceManager: IServiceManager) {
     serviceManager.addSingleton<IExtensionSingleActivationService>(
@@ -60,6 +61,10 @@ export function registerTypes(serviceManager: IServiceManager) {
         IExtensionSyncActivationService,
         VariableViewActivationService
     );
+    serviceManager.addSingleton<IExtensionSyncActivationService>(
+        IExtensionSyncActivationService,
+        IPyWidgetRendererComms
+    );
     serviceManager.addSingleton<IVariableViewProvider>(IVariableViewProvider, VariableViewProvider);
     serviceManager.add<IJupyterVariableDataProvider>(IJupyterVariableDataProvider, JupyterVariableDataProvider);
     serviceManager.addSingleton<IJupyterVariableDataProviderFactory>(

src/webviews/webview-side/ipywidgets/kernel/index.ts

@@ -3,23 +3,29 @@
 
 import type * as nbformat from '@jupyterlab/nbformat';
 import { KernelMessagingApi, PostOffice } from '../../react-common/postOffice';
-import { WidgetManager } from '../common/manager';
-import { ScriptManager } from '../common/scriptManager';
 import { OutputItem } from 'vscode-notebook-renderer';
 import { SharedMessages, IInteractiveWindowMapping, InteractiveWindowMessages } from '../../../../messageTypes';
 import { logErrorMessage, logMessage } from '../../react-common/logger';
+import { WidgetManager } from './manager';
+import { ScriptManager } from './scriptManager';
+import { IJupyterLabWidgetManagerCtor } from './types';
 
 class WidgetManagerComponent {
     private readonly widgetManager: WidgetManager;
     private readonly scriptManager: ScriptManager;
     private widgetsCanLoadFromCDN: boolean = false;
-    constructor(private postOffice: PostOffice) {
+    constructor(private postOffice: PostOffice, JupyterLabWidgetManager: IJupyterLabWidgetManagerCtor) {
         this.scriptManager = new ScriptManager(postOffice);
         this.scriptManager.onWidgetLoadError(this.handleLoadError.bind(this));
         this.scriptManager.onWidgetLoadSuccess(this.handleLoadSuccess.bind(this));
         this.scriptManager.onWidgetVersionNotSupported(this.handleUnsupportedWidgetVersion.bind(this));
-        // eslint-disable-next-line @typescript-eslint/no-explicit-any
-        this.widgetManager = new WidgetManager(undefined as any, postOffice, this.scriptManager.getScriptLoader());
+        this.widgetManager = new WidgetManager(
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            undefined as any,
+            postOffice,
+            this.scriptManager.getScriptLoader(),
+            JupyterLabWidgetManager
+        );
 
         postOffice.addHandler({
             // eslint-disable-next-line @typescript-eslint/no-explicit-any
@@ -82,23 +88,21 @@ const renderedWidgets = new Map<string, { container: HTMLElement; widget?: { dis
  * This will be exposed as a public method on window for renderer to render output.
  */
 let stackOfWidgetsRenderStatusByOutputId: { outputId: string; container: HTMLElement; success?: boolean }[] = [];
-export function renderOutput(
+export async function renderOutput(
     outputItem: OutputItem,
+    model: nbformat.IMimeBundle & {
+        model_id: string;
+        version_major: number;
+        /**
+         * This property is only used & added in tests.
+         */
+        _vsc_test_cellIndex?: number;
+    },
     element: HTMLElement,
     logger: (message: string, category?: 'info' | 'error') => void
 ) {
     try {
         stackOfWidgetsRenderStatusByOutputId.push({ outputId: outputItem.id, container: element });
-        const output = convertVSCodeOutputToExecuteResultOrDisplayData(outputItem);
-
-        // eslint-disable-next-line @typescript-eslint/no-explicit-any
-        const model = output.data['application/vnd.jupyter.widget-view+json'] as any;
-        if (!model) {
-            logger(`Error: Model not found to render output ${outputItem.id}`, 'error');
-            // eslint-disable-next-line no-console
-            return console.error('Nothing to render');
-        }
-        /* eslint-disable no-console */
         renderIPyWidget(outputItem.id, model, element, logger);
     } catch (ex) {
         logger(`Error: render output ${outputItem.id} failed ${ex.toString()}`, 'error');
@@ -217,11 +221,12 @@ async function createWidgetView(
     }
 }
 
-function initialize(context?: KernelMessagingApi) {
+let capturedContext: KernelMessagingApi;
+function initialize(JupyterLabWidgetManager: IJupyterLabWidgetManagerCtor) {
     try {
         // Setup the widget manager
-        const postOffice = new PostOffice(context);
-        const mgr = new WidgetManagerComponent(postOffice);
+        const postOffice = new PostOffice(capturedContext);
+        const mgr = new WidgetManagerComponent(postOffice, JupyterLabWidgetManager);
         // eslint-disable-next-line @typescript-eslint/no-explicit-any
         (window as any)._mgr = mgr;
     } catch (ex) {
@@ -231,43 +236,34 @@ function initialize(context?: KernelMessagingApi) {
     }
 }
 
-function convertVSCodeOutputToExecuteResultOrDisplayData(
-    outputItem: OutputItem
-): nbformat.IExecuteResult | nbformat.IDisplayData {
-    return {
-        data: {
-            [outputItem.mime]: outputItem.mime.toLowerCase().includes('json') ? outputItem.json() : outputItem.text()
-        },
-        // eslint-disable-next-line @typescript-eslint/no-explicit-any
-        metadata: (outputItem.metadata as any) || {},
-        execution_count: null,
-        // eslint-disable-next-line @typescript-eslint/no-explicit-any
-        output_type: (outputItem.metadata as any)?.outputType || 'execute_result'
-    };
-}
-
 // Create our window exports
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 (window as any).ipywidgetsKernel = {
     renderOutput,
     disposeOutput
 };
 
-let capturedContext: KernelMessagingApi | undefined;
 // To ensure we initialize after the other scripts, wait for them.
-function attemptInitialize(context?: KernelMessagingApi) {
-    capturedContext = capturedContext || context;
+function attemptInitialize(context: KernelMessagingApi) {
     logMessage(`Attempt Initialize IpyWidgets kernel.js : ${JSON.stringify(context)}`);
+
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
     if ((window as any).vscIPyWidgets) {
         logMessage('IPyWidget kernel initializing...');
-        initialize(capturedContext);
+        // The JupyterLabWidgetManager will be exposed in the global variable `window.ipywidgets.main` (check webpack config - src/ipywidgets/webpack.config.js).
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        const JupyterLabWidgetManager = (window as any).vscIPyWidgets.WidgetManager as IJupyterLabWidgetManagerCtor;
+        if (!JupyterLabWidgetManager) {
+            throw new Error('JupyterLabWidgetManager not defined. Please include/check ipywidgets.js file');
+        }
+        initialize(JupyterLabWidgetManager);
     } else {
         setTimeout(attemptInitialize, 100);
     }
 }
 
 // Has to be this form for VS code to load it correctly
-export function activate(context?: KernelMessagingApi) {
+export function activate(context: KernelMessagingApi) {
+    capturedContext = context;
     return attemptInitialize(context);
 }

src/webviews/webview-side/ipywidgets/common/manager.ts → src/webviews/webview-side/ipywidgets/kernel/manager.ts

@@ -45,7 +45,8 @@ export class WidgetManager implements IIPyWidgetManager, IMessageHandler {
     constructor(
         private readonly widgetContainer: HTMLElement,
         private readonly postOffice: PostOffice,
-        private readonly scriptLoader: ScriptLoader
+        private readonly scriptLoader: ScriptLoader,
+        private readonly JupyterLabWidgetManager: IJupyterLabWidgetManagerCtor
     ) {
         this.postOffice.addHandler(this);
 
@@ -155,14 +156,8 @@ export class WidgetManager implements IIPyWidgetManager, IMessageHandler {
         // Dispose any existing managers.
         this.manager?.dispose(); // NOSONAR
         try {
-            // The JupyterLabWidgetManager will be exposed in the global variable `window.ipywidgets.main` (check webpack config - src/ipywidgets/webpack.config.js).
-            // eslint-disable-next-line @typescript-eslint/no-explicit-any
-            const JupyterLabWidgetManager = (window as any).vscIPyWidgets.WidgetManager as IJupyterLabWidgetManagerCtor;
-            if (!JupyterLabWidgetManager) {
-                throw new Error('JupyterLabWidgetManadger not defined. Please include/check ipywidgets.js file');
-            }
             // Create the real manager and point it at our proxy kernel.
-            this.manager = new JupyterLabWidgetManager(
+            this.manager = new this.JupyterLabWidgetManager(
                 this.proxyKernel,
                 this.widgetContainer,
                 this.scriptLoader,

src/webviews/webview-side/ipywidgets/common/scriptManager.ts → src/webviews/webview-side/ipywidgets/kernel/scriptManager.ts

@@ -6,8 +6,6 @@
 import fastDeepEqual from 'fast-deep-equal';
 import { EventEmitter } from 'events';
 import { PostOffice } from '../../react-common/postOffice';
-import { warnAboutWidgetVersionsThatAreNotSupported } from '../common/incompatibleWidgetHandler';
-import { registerScripts, undefineModule } from '../common/requirejsRegistry';
 import { ScriptLoader } from './types';
 import { logErrorMessage, logMessage } from '../../react-common/logger';
 import { Deferred, createDeferred } from '../../../../platform/common/utils/async';
@@ -18,6 +16,8 @@ import { noop } from '../../../../platform/common/utils/misc';
 import { IDisposable } from '../../../../platform/common/types';
 import { disposeAllDisposables } from '../../../../platform/common/helpers';
 import { WidgetScriptSource } from '../../../../notebooks/controllers/ipywidgets/types';
+import { warnAboutWidgetVersionsThatAreNotSupported } from './incompatibleWidgetHandler';
+import { registerScripts, undefineModule } from './requirejsRegistry';
 
 export class ScriptManager extends EventEmitter {
     public readonly widgetsRegisteredInRequireJs = new Set<string>();