Support server-side execution (jupyterlab#279)

* Support server-side execution

* lint

* again

* Fix jest

* Mention this feature in documentation

* Update UI test snapshots

* Use "dependency_type: minimum" for Minimum Versions check

* Add 'serverSideExecution' to page config

* Replace "fully recovered" with "recovered"

* Add link to jupyter-server issue

Co-authored-by: Michał Krassowski <[email protected]>

---------

Co-authored-by: Michał Krassowski <[email protected]>

Author: 2 people

.github/workflows/test.yml

@@ -152,10 +152,12 @@ jobs:
       - name: Base Setup
         uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1
         with:
-          python_version: "3.11"
+          python_version: "3.8"
+          dependency_type: minimum
 
-      - name: Install minimum versions
-        uses: jupyterlab/maintainer-tools/.github/actions/install-minimums@v1
+      - name: Install the Python dependencies
+        run: |
+          pip install -e ".[test]"
 
       - name: Run the unit tests
         run: |

docs/source/configuration.md

@@ -10,7 +10,7 @@ you happen to delete it, there shouldn't be any serious consequence either.
 There are a number of settings that you can change:
 
 ```bash
-# To enable or disable RTC(Real-Time Collaboration) (default: False).
+# To enable or disable RTC (Real-Time Collaboration) (default: False).
 # If True, RTC will be disabled.
 jupyter lab --YDocExtension.disable_rtc=True
 
@@ -29,3 +29,21 @@ jupyter lab --YDocExtension.document_cleanup_delay=100
 # The YStore class to use for storing Y updates (default: JupyterSQLiteYStore).
 jupyter lab --YDocExtension.ystore_class=pycrdt_websocket.ystore.TempFileYStore
 ```
+
+There is an experimental feature that is currently only supported by the
+[Jupyverse](https://github.com/jupyter-server/jupyverse) server
+(not yet with [jupyter-server](https://github.com/jupyter-server/jupyter_server),
+see the [issue #900](https://github.com/jupyter-server/jupyter_server/issues/900)):
+server-side execution. With this, running notebook code cells is not done in the frontend through
+the low-level kernel protocol over WebSocket API, but through a high-level REST API. Communication
+with the kernel is then delegated to the server, and cell outputs are populated in the notebook
+shared document. The frontend gets these outputs changes and shows them live. What this means is
+that the notebook state can be recovered even if the frontend disconnects, because cell outputs are
+not populated frontend-side but server-side.
+
+This feature is disabled by default, and can be enabled like so:
+```bash
+pip install "jupyterlab>=4.2.0b0"
+pip install "jupyverse[jupyterlab, auth]>=0.4.2"
+jupyverse --set kernels.require_yjs=true --set jupyterlab.server_side_execution=true
+```

jupyter_collaboration/app.py

@@ -57,6 +57,14 @@ class YDocExtension(ExtensionApp):
         directory.""",
     )
 
+    server_side_execution = Bool(
+        False,
+        config=True,
+        help="""Whether to execute notebooks in the server using the REST API, not using the kernel
+        protocol over WebSocket. The frontend only interacts with the notebook through its shared
+        model.""",
+    )
+
     def initialize(self):
         super().initialize()
         self.serverapp.event_logger.register_event_schema(EVENTS_SCHEMA_PATH)
@@ -74,7 +82,11 @@ def initialize_settings(self):
 
     def initialize_handlers(self):
         self.serverapp.web_app.settings.setdefault(
-            "page_config_data", {"disableRTC": self.disable_rtc}
+            "page_config_data",
+            {
+                "disableRTC": self.disable_rtc,
+                "serverSideExecution": self.server_side_execution,
+            },
         )
 
         # Set configurable parameters to YStore class

packages/collaboration-extension/package.json

@@ -56,20 +56,20 @@
     "@jupyter/collaboration": "^2.0.11",
     "@jupyter/docprovider": "^2.0.11",
     "@jupyter/ydoc": "^1.1.0-a0",
-    "@jupyterlab/application": "^4.0.5",
-    "@jupyterlab/apputils": "^4.0.5",
-    "@jupyterlab/codemirror": "^4.0.5",
+    "@jupyterlab/application": "^4.2.0-beta.0",
+    "@jupyterlab/apputils": "^4.2.0-beta.0",
+    "@jupyterlab/codemirror": "^4.2.0-beta.0",
     "@jupyterlab/coreutils": "^6.0.5",
-    "@jupyterlab/docregistry": "^4.0.5",
-    "@jupyterlab/filebrowser": "^4.0.5",
-    "@jupyterlab/fileeditor": "^4.0.5",
-    "@jupyterlab/logconsole": "^4.0.5",
-    "@jupyterlab/notebook": "^4.0.5",
+    "@jupyterlab/docregistry": "^4.2.0-beta.0",
+    "@jupyterlab/filebrowser": "^4.2.0-beta.0",
+    "@jupyterlab/fileeditor": "^4.2.0-beta.0",
+    "@jupyterlab/logconsole": "^4.2.0-beta.0",
+    "@jupyterlab/notebook": "^4.2.0-beta.0",
     "@jupyterlab/services": "^7.0.5",
-    "@jupyterlab/settingregistry": "^4.0.5",
-    "@jupyterlab/statedb": "^4.0.5",
-    "@jupyterlab/translation": "^4.0.5",
-    "@jupyterlab/ui-components": "^4.0.5",
+    "@jupyterlab/settingregistry": "^4.2.0-beta.0",
+    "@jupyterlab/statedb": "^4.2.0-beta.0",
+    "@jupyterlab/translation": "^4.2.0-beta.0",
+    "@jupyterlab/ui-components": "^4.2.0-beta.0",
     "@lumino/commands": "^2.1.0",
     "@lumino/widgets": "^2.1.0",
     "y-protocols": "^1.0.5",
@@ -97,7 +97,8 @@
     "schemaDir": "./schema",
     "outputDir": "../../jupyter_collaboration/labextension",
     "disabledExtensions": [
-      "@jupyterlab/filebrowser-extension:defaultFileBrowser"
+      "@jupyterlab/filebrowser-extension:defaultFileBrowser",
+      "@jupyterlab/notebook-extension:cell-executor"
     ],
     "sharedPackages": {
       "@codemirror/state": {

packages/collaboration-extension/src/collaboration.ts

@@ -14,9 +14,11 @@ import {
   EditorExtensionRegistry,
   IEditorExtensionRegistry
 } from '@jupyterlab/codemirror';
+import { type MarkdownCell } from '@jupyterlab/cells';
+import { INotebookCellExecutor, runCell } from '@jupyterlab/notebook';
 import { WebSocketAwarenessProvider } from '@jupyter/docprovider';
 import { SidePanel, usersIcon } from '@jupyterlab/ui-components';
-import { URLExt } from '@jupyterlab/coreutils';
+import { PageConfig, URLExt } from '@jupyterlab/coreutils';
 import { ServerConnection } from '@jupyterlab/services';
 import { IStateDB, StateDB } from '@jupyterlab/statedb';
 import { ITranslator, nullTranslator } from '@jupyterlab/translation';
@@ -189,3 +191,63 @@ export const userEditorCursors: JupyterFrontEndPlugin<void> = {
     });
   }
 };
+
+export const notebookCellExecutor: JupyterFrontEndPlugin<INotebookCellExecutor> =
+  {
+    id: '@jupyter/collaboration-extension:notebook-cell-executor',
+    description:
+      'Add notebook cell executor that uses REST API instead of kernel protocol over WebSocket.',
+    autoStart: true,
+    provides: INotebookCellExecutor,
+    activate: (app: JupyterFrontEnd): INotebookCellExecutor => {
+      if (PageConfig.getOption('serverSideExecution') === 'true') {
+        return Object.freeze({ runCell: runCellServerSide });
+      }
+      return Object.freeze({ runCell });
+    }
+  };
+
+async function runCellServerSide({
+  cell,
+  notebook,
+  notebookConfig,
+  onCellExecuted,
+  onCellExecutionScheduled,
+  sessionContext,
+  sessionDialogs,
+  translator
+}: INotebookCellExecutor.IRunCellOptions): Promise<boolean> {
+  switch (cell.model.type) {
+    case 'markdown':
+      (cell as MarkdownCell).rendered = true;
+      cell.inputHidden = false;
+      onCellExecuted({ cell, success: true });
+      break;
+    case 'code': {
+      const kernelId = sessionContext?.session?.kernel?.id;
+      const settings = ServerConnection.makeSettings();
+      const apiURL = URLExt.join(
+        settings.baseUrl,
+        `api/kernels/${kernelId}/execute`
+      );
+      const cellId = cell.model.sharedModel.getId();
+      const documentId = `json:notebook:${notebook.sharedModel.getState(
+        'file_id'
+      )}`;
+      const body = `{"cell_id":"${cellId}","document_id":"${documentId}"}`;
+      const init = {
+        method: 'POST',
+        body
+      };
+      try {
+        await ServerConnection.makeRequest(apiURL, init, settings);
+      } catch (error: any) {
+        throw new ServerConnection.NetworkError(error);
+      }
+      break;
+    }
+    default:
+      break;
+  }
+  return Promise.resolve(true);
+}

packages/collaboration-extension/src/index.ts

@@ -19,7 +19,8 @@ import {
   menuBarPlugin,
   rtcGlobalAwarenessPlugin,
   rtcPanelPlugin,
-  userEditorCursors
+  userEditorCursors,
+  notebookCellExecutor
 } from './collaboration';
 import { sharedLink } from './sharedlink';
 
@@ -37,7 +38,8 @@ const plugins: JupyterFrontEndPlugin<any>[] = [
   rtcGlobalAwarenessPlugin,
   rtcPanelPlugin,
   sharedLink,
-  userEditorCursors
+  userEditorCursors,
+  notebookCellExecutor
 ];
 
 export default plugins;

packages/docprovider/jest.config.js

@@ -7,7 +7,11 @@ const jestJupyterLab = require('@jupyterlab/testing/lib/jest-config');
 
 const esModules = [
   '@codemirror',
+  '@microsoft',
+  'exenv-es6',
   '@jupyter/ydoc',
+  '@jupyter/react-components',
+  '@jupyter/web-components',
   '@jupyterlab/',
   'lib0',
   'nanoid',

pyproject.toml

@@ -27,7 +27,7 @@ classifiers = [
     "Framework :: Jupyter :: JupyterLab :: Extensions :: Prebuilt",
 ]
 dependencies = [
-    "jupyter_server>=2.0.0,<3.0.0",
+    "jupyter_server>=2.4.0,<3.0.0",
     "jupyter_ydoc>=2.0.0,<3.0.0",
     "pycrdt-websocket>=0.12.5,<0.13.0",
     "jupyter_events>=0.10.0",
@@ -45,12 +45,12 @@ dev = [
 ]
 test = [
     "coverage",
-    "jupyter_server[test]>=2.0.0",
+    "jupyter_server[test]>=2.4.0",
     "jupyter_server_fileid[test]",
     "pytest>=7.0",
     "pytest-cov",
     "websockets",
-    "importlib_metadata >=3.6; python_version<'3.10'",
+    "importlib_metadata >=4.8.3; python_version<'3.10'",
 ]
 docs = [
     "jupyterlab>=4.0.0",