Add zstd support to View::to_arrow

.github/workflows/build.yaml

@@ -847,7 +847,12 @@ jobs:
                     - ubuntu-22.04
                 python-version:
                     - 3.11
-                node-version: [22.x]
+                node-version: [22.x, 26.x]
+                is-release:
+                    - ${{ startsWith(github.ref, 'refs/tags/v') || github.ref_name == 'master' || github.event.inputs.ci-full }}
+                exclude:
+                    - node-version: 26.x
+                      is-release: false
 
         steps:
             - name: Checkout
@@ -890,7 +895,7 @@ jobs:
               if: ${{ failure() && steps.run_tests.outcome == 'failure' }}
               uses: actions/upload-artifact@v4
               with:
-                  name: perspective-js-test-results
+                  name: perspective-js-test-results-node${{ matrix.node-version }}
                   path: tools/test/dist/results
                   if-no-files-found: ignore
                   overwrite: true

docs/md/how_to/javascript/serializing.md

@@ -19,3 +19,12 @@ console.log(await view.to_ndjson());
 // ArrayBuffer
 console.log(await view.to_arrow());
 ```
+
+`to_arrow()` writes an uncompressed Arrow IPC stream by default; pass
+`compression` to apply LZ4 or ZSTD body compression, which `Client::table` reads
+back transparently:
+
+```javascript
+const compressed = await view.to_arrow({ compression: "zstd" });
+const table2 = await client.table(compressed);
+```

docs/md/how_to/javascript/viewer.md

@@ -153,8 +153,11 @@ never deletes the result.
 | `copy(options?)` | Copy a panel to the clipboard |
 
 `export()`, `download()` and `copy()` all take a `method`, one of `"csv"`,
-`"json"`, `"ndjson"` or `"arrow"` — each with `-all` and `-selected` variants
-(e.g. `"csv-selected"`) — plus `"html"`, `"json-config"`, and `"plugin"`.
+`"json"`, `"ndjson"`, `"arrow"`, `"arrow-lz4"` or `"arrow-zstd"` — each with
+`-all` and `-selected` variants (e.g. `"csv-selected"`, `"arrow-zstd-all"`) —
+plus `"html"`, `"json-config"`, and `"plugin"`. The `"arrow-lz4"` and
+`"arrow-zstd"` methods write the Arrow IPC stream with LZ4 or ZSTD body
+compression.
 The `"plugin"` method asks the plugin to render itself, which produces a PNG
 for charts and text for the datagrid.
 | `getSelection(options?)` / `setSelection(...)` | Get or set the selected region |

package.json

@@ -18,7 +18,7 @@
     "llvm": "17.0.6",
     "pyodide": "0.29.4",
     "engines": {
-        "node": ">=16 <24"
+        "node": ">=16 <27"
     },
     "workspaces": [
         "tools/test",

pnpm-workspace.yaml

@@ -47,8 +47,8 @@ catalog:
     "@fontsource/roboto-mono": "4.5.10"
     "@iarna/toml": "3.0.0"
     "@jupyterlab/builder": "^4"
-    "@playwright/experimental-ct-react": "=1.58.0"
-    "@playwright/test": "=1.58.0"
+    "@playwright/experimental-ct-react": "=1.62.0"
+    "@playwright/test": "=1.62.0"
     "lightningcss": "^1.29.0"
     "@types/lodash": "^4.17.20"
     "@types/node": ">=22"

rust/perspective-client/src/rust/view.rs

@@ -105,8 +105,10 @@ pub struct ViewWindow {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub formatted: Option<bool>,
 
-    /// Only impacts [`View::to_arrow`]
+    /// Arrow IPC body compression for [`View::to_arrow`], `"lz4"` or `"zstd"`
+    /// (uncompressed when omitted).
     #[ts(optional)]
+    #[ts(type = "\"lz4\" | \"zstd\"")]
     #[serde(skip_serializing_if = "Option::is_none")]
     pub compression: Option<String>,
 
@@ -423,7 +425,8 @@ impl View {
         }
     }
 
-    /// Serializes a [`View`] to the Apache Arrow data format.
+    /// Serializes a [`View`] to the Apache Arrow data format, with IPC body
+    /// compression per [`ViewWindow::compression`].
     pub async fn to_arrow(&self, window: ViewWindow) -> ClientResult<Bytes> {
         let msg = self.client_message(ClientReq::ViewToArrowReq(ViewToArrowReq {
             viewport: Some(window.clone().into()),

rust/perspective-js/src/rust/view.rs

@@ -195,6 +195,17 @@ impl View {
     }
 
     /// Serializes a [`View`] to the Apache Arrow data format.
+    ///
+    /// # Arguments
+    ///
+    /// - `window` - a [`ViewWindow`]; its `compression` key selects Arrow IPC
+    ///   body compression, `"lz4"` or `"zstd"` (uncompressed when omitted).
+    ///
+    /// # JavaScript Examples
+    ///
+    /// ```javascript
+    /// const arrow = await view.to_arrow({ compression: "zstd" });
+    /// ```
     #[wasm_bindgen]
     pub async fn to_arrow(&self, window: Option<JsViewWindow>) -> ApiResult<ArrayBuffer> {
         let window = window.into_serde_ext::<Option<ViewWindow>>()?;

rust/perspective-js/src/ts/perspective.node.ts

@@ -181,6 +181,7 @@ async function compile_server_module(wasm_path: string) {
     const bytes = await load_wasm_stage_0(buffer.buffer as ArrayBuffer);
     return await compile_perspective(bytes.buffer as ArrayBuffer, {
         make_disk_bridge: make_node_disk_bridge,
+        env: process.env,
     });
 }
 

rust/perspective-js/src/ts/wasm/emscripten_api.ts

@@ -24,6 +24,8 @@ export interface DiskBridgeHelpers {
 }
 
 export interface CompileOptions {
+    env?: Record<string, string | undefined>;
+
     make_disk_bridge?: (helpers: DiskBridgeHelpers) => {
         store(
             namePtr: number | bigint,
@@ -49,6 +51,7 @@ export async function compile_perspective(
             return x;
         },
         make_disk_bridge: opts?.make_disk_bridge,
+        env: opts?.env,
         instantiateWasm: async (
             imports: any,
             receive: (_: WebAssembly.Instance) => void,

rust/perspective-js/src/ts/wasm/perspective-server.poly.ts

@@ -71,6 +71,10 @@ export default async function (obj: any) {
     // receives a resolved file name string (from `victim_fname`).
     const disk_helpers = { heap, toAddr, readCString };
 
+    const environ_entries: Uint8Array[] = Object.entries(obj.env ?? {})
+        .filter(([, value]) => typeof value === "string")
+        .map(([key, value]) => new TextEncoder().encode(`${key}=${value}\0`));
+
     function makeOpfsBridge() {
         async function opfsOpenFile(name: string, create: boolean) {
             const parts = name.split("/").filter((s) => s.length > 0);
@@ -281,10 +285,51 @@ export default async function (obj: any) {
                 console.error("abort");
             }
         },
-        environ_get(...args: any[]) {
+        environ_get(environ: number | bigint, environ_buf: number | bigint) {
+            const view = new DataView(wasm_memory.buffer);
+            let entry = toAddr(environ);
+            let cursor = toAddr(environ_buf);
+            for (const bytes of environ_entries) {
+                if (is_memory64) {
+                    view.setBigUint64(entry, BigInt(cursor), true);
+                    entry += 8;
+                } else {
+                    view.setUint32(entry, cursor, true);
+                    entry += 4;
+                }
+
+                heap().set(bytes, cursor);
+                cursor += bytes.length;
+            }
+
             return 0;
         },
-        environ_sizes_get(...args: any[]) {
+        environ_sizes_get(
+            environ_count: number | bigint,
+            environ_buf_size: number | bigint,
+        ) {
+            const view = new DataView(wasm_memory.buffer);
+            const total = environ_entries.reduce((n, x) => n + x.length, 0);
+            if (is_memory64) {
+                view.setBigUint64(
+                    toAddr(environ_count),
+                    BigInt(environ_entries.length),
+                    true,
+                );
+                view.setBigUint64(
+                    toAddr(environ_buf_size),
+                    BigInt(total),
+                    true,
+                );
+            } else {
+                view.setUint32(
+                    toAddr(environ_count),
+                    environ_entries.length,
+                    true,
+                );
+                view.setUint32(toAddr(environ_buf_size), total, true);
+            }
+
             return 0;
         },
         fd_close(...args: any[]) {
@@ -392,8 +437,9 @@ export default async function (obj: any) {
         }
         const n = Number(mod.psp_residency_prepare(server));
         for (let i = 0; i < n; i++) {
+            const index = is_memory64 ? BigInt(i) : i;
             const fname = readCString(
-                mod.psp_residency_victim_fname(server, i),
+                mod.psp_residency_victim_fname(server, index),
             );
             if (fname) {
                 await disk.ensureOpen(fname);

rust/perspective-js/test/js/constructors.spec.js

@@ -828,55 +828,47 @@ function validate_typed_array(typed_array, column_data) {
 
     test.describe("Errors", function () {
         test("Table constructor should throw an exception and reject promise", async function () {
-            expect.assertions(1);
-            perspective.table([1, 2, 3]).catch((error) => {
-                expect(error.message).toContain(
-                    "Abort(): Cannot determine data types without column names!\n",
-                );
-            });
+            await expect(perspective.table([1, 2, 3])).rejects.toThrow(
+                "Abort(): Cannot determine data types without column names!\n",
+            );
         });
 
         test("View constructor should throw an exception and reject promise", async function () {
-            expect.assertions(1);
             const table = await perspective.table(int_float_string_data);
-            table
-                .view({
-                    group_by: ["abcd"],
-                })
-                .catch((error) => {
-                    expect(error.message).toContain(
-                        "Abort(): Invalid column 'abcd' found in View group_by.\n",
-                    );
-                    table.delete();
-                });
+            await expect(table.view({ group_by: ["abcd"] })).rejects.toThrow(
+                "Abort(): Invalid column 'abcd' found in View group_by.\n",
+            );
+
+            await table.delete();
         });
 
         test("Table constructor should throw an exception on await", async function () {
-            expect.assertions(1);
-
+            let error;
             try {
                 await perspective.table([1, 2, 3]);
-            } catch (error) {
-                expect(error.message).toContain(
-                    "Abort(): Cannot determine data types without column names!\n",
-                );
+            } catch (e) {
+                error = e;
             }
+
+            expect(error.message).toContain(
+                "Abort(): Cannot determine data types without column names!\n",
+            );
         });
 
         test("View constructor should throw an exception on await", async function () {
-            expect.assertions(1);
             const table = await perspective.table(int_float_string_data);
-
+            let error;
             try {
-                await table.view({
-                    group_by: ["abcd"],
-                });
-            } catch (error) {
-                expect(error.message).toContain(
-                    "Abort(): Invalid column 'abcd' found in View group_by.\n",
-                );
-                table.delete();
+                await table.view({ group_by: ["abcd"] });
+            } catch (e) {
+                error = e;
             }
+
+            expect(error.message).toContain(
+                "Abort(): Invalid column 'abcd' found in View group_by.\n",
+            );
+
+            await table.delete();
         });
 
         test("Table constructor pads short trailing columns with null", async function () {