Add Gc benchmark and Debugging

.vscode/launch.json

@@ -0,0 +1,26 @@
+{
+  // Use IntelliSense to learn about possible attributes.
+  // Hover to view descriptions of existing attributes.
+  // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": "Test",
+      "request": "launch",
+      "runtimeArgs": ["run", "test"],
+      "internalConsoleOptions": "openOnSessionStart",
+      "runtimeExecutable": "pnpm",
+      "skipFiles": ["<node_internals>/**"],
+      "type": "node"
+    },
+    {
+      "name": "Bench",
+      "request": "launch",
+      "runtimeArgs": ["run", "bench:debug"],
+      "internalConsoleOptions": "openOnSessionStart",
+      "runtimeExecutable": "pnpm",
+      "skipFiles": ["<node_internals>/**"],
+      "type": "node"
+    }
+  ]
+}

README.md

@@ -14,7 +14,6 @@ $ pnpm bench
 - Tracks garbage collection overhead per test
 - Outputs a csv file for easy integration with other tools.
 
-
 Current reactivity benchmarks ([S.js](https://github.com/adamhaile/S/blob/master/bench/bench.js), [CellX](https://github.com/Riim/cellx/blob/master/perf/perf.html)) are focused on creation time, and update time for a static graph. Additionally, existing benchmarks aren't very configurable, and don't test for dynamic dependencies. We've created a new benchmark that allows library authors to compare their frameworks against each other, and against the existing benchmarks, as well as against a new configurable benchmark with dynamically changing sources.
 
 We're also working on enabling consistent logging and efficient tracking of GC time across all benchmarks.
@@ -26,3 +25,7 @@ That said, there's learning here to improve performance of all the frameworks.
 ![Performance Results](https://user-images.githubusercontent.com/14153763/221107379-51a93eab-95ac-4c89-9a74-7a1527fc4a03.png)
 
 ![Raw](https://user-images.githubusercontent.com/14153763/222212050-5b651e4d-6e71-4667-94e7-eb94b7030bc1.png)
+
+## Debugging
+
+There are VS Code launch options to run test or bench in debug mode.

package.json

@@ -5,11 +5,13 @@
   "main": "index.js",
   "scripts": {
     "test": "vitest",
+    "bench:debug": "node --allow-natives-syntax --loader @esbuild-kit/esm-loader src/index.ts",
     "bench": "esbuild src/index.ts --external:v8-natives --bundle --format=cjs --platform=node | node --allow-natives-syntax"
   },
   "keywords": [],
   "author": "",
   "license": "ISC",
+  "type": "module",
   "dependencies": {
     "@angular/core": "16.2.1",
     "@preact/signals": "^1.2.1",
@@ -27,6 +29,7 @@
     "usignal": "^0.9.0"
   },
   "devDependencies": {
+    "@esbuild-kit/esm-loader": "^2.6.5",
     "@types/node": "^20.5.1",
     "esbuild": "^0.19.2",
     "rxjs": "^7.8.1",

src/frameworks/solid.ts

@@ -5,7 +5,7 @@ import {
   createMemo,
   createRoot,
   createSignal,
-} from "solid-js/dist/solid.cjs";
+} from "solid-js";
 
 export const solidFramework: ReactiveFramework = {
   name: "SolidJS",

src/gcBench.ts

@@ -0,0 +1,154 @@
+import v8 from "v8-natives";
+import { fastestTest } from "./util/benchRepeat";
+import { logPerfResult } from "./util/perfLogging";
+import { Computed, ReactiveFramework } from "./util/reactiveFramework";
+import { promiseDelay } from "./util/asyncUtil";
+
+const cases = [
+  collectDynamicSources,
+  collectDynamicSourcesAfterUpdate,
+  collectOutOfScopeConsumers,
+];
+
+export async function gcBench(framework: ReactiveFramework) {
+  for (const c of cases) {
+    const iter = framework.withBuild(() => c(framework));
+
+    const { timing } = await fastestTest(1, async () => {
+      for (let i = 0; i < 1; i++) {
+        await iter();
+      }
+    });
+
+    logPerfResult({
+      framework: framework.name,
+      test: `gc-${c.name}`,
+      time: timing.time.toFixed(2),
+      gcTime: timing.gcTime?.toFixed(2),
+    });
+  }
+}
+
+/**
+ * Ensures old inactive dynamic sources get collected
+ */
+function collectDynamicSourcesAfterUpdate(framework: ReactiveFramework) {
+  const head = framework.signal(1);
+  let dynamic = framework.signal(0);
+  const tail = framework.computed(() => {
+    return dynamic.read() + head.read();
+  });
+
+  const weakOldDynamic = new WeakRef(dynamic);
+
+  // Create connection to first dynamic
+  tail.read();
+
+  // Swap dynamic for new
+  dynamic = framework.signal(1);
+
+  // Trigger update to force invalidation of tail
+  head.write(-1);
+
+  return async () => {
+    let didCollect = false;
+
+    // Async GC loop to allow time for frameworks with scheduled cleanups
+    for (let i = 0; i < 200; i++) {
+      await promiseDelay();
+      v8.collectGarbage();
+
+      if (weakOldDynamic.deref() === undefined) {
+        didCollect = true;
+        break;
+      }
+    }
+
+    console.assert(tail.read() === 0, "tail is not 0");
+
+    console.assert(didCollect, "Failed to collect inactive dynamic sources");
+  };
+}
+
+/**
+ * Ensures old inactive dynamic sources get collected
+ */
+function collectDynamicSources(framework: ReactiveFramework) {
+  const head = framework.signal(1);
+  let dynamic = framework.signal(0);
+  const tail = framework.computed(() => {
+    return dynamic.read() + head.read();
+  });
+
+  const weakOldDynamic = new WeakRef(dynamic);
+
+  // Create connection to first dynamic
+  tail.read();
+
+  // Swap dynamic for new
+  dynamic = framework.signal(1);
+
+  return async () => {
+    let didCollect = false;
+
+    // Async GC loop to allow time for frameworks with scheduled cleanups
+    for (let i = 0; i < 200; i++) {
+      await promiseDelay();
+      v8.collectGarbage();
+
+      if (weakOldDynamic.deref() === undefined) {
+        didCollect = true;
+        break;
+      }
+    }
+
+    console.assert(tail.read() === 1, "tail is not 1");
+
+    console.assert(didCollect, "Failed to collect inactive dynamic sources");
+  };
+}
+
+/**
+ * Ensures out of scope (computed) consumers get collected
+ */
+function collectOutOfScopeConsumers(framework: ReactiveFramework) {
+  const head = framework.signal(0);
+  let tail: Computed<number> = head;
+  const weakRefs: WeakRef<Computed<number>>[] = [];
+
+  for (let i = 0; i < 5; i++) {
+    const oldTail = tail;
+    tail = framework.computed(() => {
+      return oldTail.read() + 1;
+    });
+    tail.read();
+    weakRefs.push(new WeakRef(tail));
+  }
+
+  return async () => {
+    let activeRefCount = 0;
+
+    // Async GC loop to allow time for frameworks with scheduled cleanups
+    for (let i = 0; i < 200; i++) {
+      await promiseDelay();
+      v8.collectGarbage();
+
+      activeRefCount = 0;
+      for (const ref of weakRefs) {
+        if (ref.deref()) {
+          activeRefCount++;
+        }
+      }
+      if (activeRefCount === 0) {
+        break;
+      }
+    }
+
+    console.assert(head.read() === 0);
+
+    console.assert(
+      activeRefCount === 0,
+      `Found ${activeRefCount} active references when there should be none`
+    );
+  };
+}

src/index.ts

@@ -5,6 +5,7 @@ import { frameworkInfo } from "./config";
 import { logPerfResult, perfReportHeaders } from "./util/perfLogging";
 import { molBench } from "./molBench";
 import { kairoBench } from "./kairoBench";
+import { gcBench } from "./gcBench";
 
 async function main() {
   logPerfResult(perfReportHeaders());
@@ -28,6 +29,10 @@ async function main() {
   for (const frameworkTest of frameworkInfo) {
     await dynamicBench(frameworkTest);
   }
+
+  for (const { framework } of frameworkInfo) {
+    await gcBench(framework);
+  }
 }
 
 main();

src/util/benchRepeat.ts

@@ -24,7 +24,9 @@ export async function fastestTest<T>(
 async function runTracked<T>(fn: () => T): Promise<TimingResult<T>> {
   v8.collectGarbage();
   const gcTrack = new GarbageTrack();
-  const { result: wrappedResult, trackId } = gcTrack.watch(() => runTimed(fn));
+  const { result: wrappedResult, trackId } = await gcTrack.watch(() =>
+    runTimed(fn)
+  );
   const gcTime = await gcTrack.gcDuration(trackId);
   const { result, time } = wrappedResult;
   gcTrack.destroy();

src/util/garbageTracking.ts

@@ -14,10 +14,13 @@ export class GarbageTrack {
   private periods: WatchPeriod[] = [];
 
   /** look for gc events during the time this function is executing */
-  watch<T>(fn: () => T): { result: T; trackId: number } {
+  async watch<T>(fn: () => T): Promise<{
+    result: Awaited<T>;
+    trackId: number;
+  }> {
     this.trackId++;
     const start = performance.now();
-    const result = fn();
+    const result = await fn();
     const end = performance.now();
     this.periods.push({ trackId: this.trackId, start, end });
 

src/util/perfUtil.ts

@@ -4,9 +4,9 @@ export interface TimedResult<T> {
 }
 
 /** run a function, recording how long it takes */
-export function runTimed<T>(fn: () => T): TimedResult<T> {
+export async function runTimed<T>(fn: () => T): Promise<TimedResult<T>> {
   const start = performance.now();
-  const result = fn();
+  const result = await fn();
   const time = performance.now() - start;
   return { result, time };
 }

tsconfig.json

@@ -2,7 +2,7 @@
   "compilerOptions": {
     "strict": true,
     "target": "ESNext",
-    "moduleResolution": "node",
+    "moduleResolution": "Bundler",
     "lib": ["ESNext", "DOM"],
     "types": ["@types/node"],
     "noEmit": true,