Skip to content

Commit efd4711

Browse files
ampagenttmm
ampagent
and
tmm
committed
feat(swift): benchmarks, next to the JS ones
ZeroBenchmarks ports zero.bench.ts and the shared B-tree and hash benchmarks under their names, and adds the Swift client's own: memory for 100k rows, hydration of up to 100k rows (as structs too), pokes, and mutations including the client's side of a round trip. `swift run -c release ZeroBenchmarks [filter…]`. packages/zero-conformance has the JS side of those that have one (`pnpm --filter zero-conformance bench`). BENCHMARKS.md has both sets of numbers from one machine: Swift is faster on all the client benchmarks and B-tree scans, and slower on B-tree lookups. Amp-Thread-ID: https://ampcode.com/threads/T-c513a037-a257-45e0-b9a9-15c15edfe859 Co-authored-by: tmm <tmm@tmm.dev>
1 parent 08e6d1f commit efd4711

9 files changed

Lines changed: 854 additions & 1 deletion

File tree

‎packages/zero-conformance/package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
"type": "module",
77
"scripts": {
88
"test": "vitest run",
9+
"bench": "vitest run --config vitest.config.bench.ts",
910
"generate": "UPDATE_VECTORS=1 vitest run",
1011
"check-types": "tsc",
1112
"format": "oxfmt .",

‎packages/zero-conformance/src/bench/client.bench.ts‎

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
/**
2+
* The JS side of the Swift client's own benchmarks
3+
* (`swift/Sources/ZeroBenchmarks`), on the same data as
4+
* `zero-client/src/client/zero.bench.ts`: users with ten number columns. The
5+
* Swift numbers are only comparable to these when both run on one machine.
6+
*
7+
* Pokes and the round trip have no JS counterpart: the JS client applies
8+
* pokes on the next frame, so timing them measures the scheduling.
9+
*/
10+
import {expect} from 'vitest';
11+
import {bench, describe} from '../../../shared/src/bench.ts';
12+
import {Zero} from '../../../zero-client/src/client/zero.ts';
13+
import {createBuilder} from '../../../zql/src/query/create-builder.ts';
14+
15+
const user = {
16+
name: 'user',
17+
columns: {
18+
a: {type: 'number'},
19+
b: {type: 'number'},
20+
c: {type: 'number'},
21+
d: {type: 'number'},
22+
e: {type: 'number'},
23+
f: {type: 'number'},
24+
g: {type: 'number'},
25+
h: {type: 'number'},
26+
i: {type: 'number'},
27+
j: {type: 'number'},
28+
},
29+
primaryKey: ['a'],
30+
} as const;
31+
const schema = {
32+
version: 0,
33+
tables: {user},
34+
relationships: {},
35+
enableLegacyMutators: true,
36+
} as const;
37+
const zql = createBuilder(schema);
38+
39+
function deferred<T>() {
40+
let resolve!: (value: T) => void;
41+
const promise = new Promise<T>(r => {
42+
resolve = r;
43+
});
44+
return {promise, resolve};
45+
}
46+
47+
async function makeZero(rows: number) {
48+
const z = new Zero({
49+
schema,
50+
cacheURL: null,
51+
userID: 'bench',
52+
kvStore: 'mem',
53+
storageKey: `bench-${rows}-${Math.random()}`,
54+
});
55+
await z.mutateBatch(async m => {
56+
for (let n = 0; n < rows; n++) {
57+
await m.user.insert({
58+
a: n,
59+
b: n,
60+
c: n,
61+
d: n,
62+
e: n,
63+
f: n,
64+
g: n,
65+
h: n,
66+
i: n,
67+
j: n,
68+
});
69+
}
70+
});
71+
return z;
72+
}
73+
74+
async function materialize(z: Awaited<ReturnType<typeof makeZero>>, n: number) {
75+
const {promise, resolve} = deferred<number>();
76+
const view = z.materialize(zql.user);
77+
view.addListener(data => {
78+
if (data.length === n) {
79+
resolve(data.length);
80+
}
81+
});
82+
expect(await promise).toBe(n);
83+
view.destroy();
84+
}
85+
86+
// What 100k rows cost in memory, as the Swift benchmark measures it: the
87+
// change in resident memory.
88+
{
89+
const n = 100_000;
90+
const before = process.memoryUsage().rss;
91+
const z = await makeZero(n);
92+
const view = z.materialize(zql.user);
93+
const {promise, resolve} = deferred<void>();
94+
view.addListener(data => {
95+
if (data.length === n) {
96+
resolve();
97+
}
98+
});
99+
await promise;
100+
const mb = (process.memoryUsage().rss - before) / 1e6;
101+
// oxlint-disable-next-line no-console
102+
console.log(
103+
`memory > ${n} rows x 10 columns (numbers), with a view: ${mb.toFixed(1)} MB`,
104+
);
105+
view.destroy();
106+
await z.close();
107+
}
108+
109+
for (const n of [1_000, 10_000, 100_000]) {
110+
const z = await makeZero(n);
111+
describe('hydration', () => {
112+
bench(`materialize ${n} rows`, async () => {
113+
await materialize(z, n);
114+
});
115+
});
116+
}
117+
118+
{
119+
const n = 1000;
120+
const z = await makeZero(n);
121+
const view = z.materialize(zql.user);
122+
let value = n;
123+
describe('mutations', () => {
124+
bench(`optimistic update, ${n}-row view`, async () => {
125+
value++;
126+
await z.mutate.user.update({a: value % n, b: value});
127+
});
128+
});
129+
void view;
130+
}

‎packages/zero-conformance/vitest.config.bench.ts‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
import {mergeConfig} from 'vitest/config';
2+
import {benchConfig} from '../shared/src/tool/vitest-config.ts';
3+
4+
export default mergeConfig(benchConfig, {
5+
test: {
6+
name: 'zero-conformance/bench',
7+
browser: {
8+
enabled: false,
9+
},
10+
},
11+
});

‎swift/BENCHMARKS.md‎

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
# Benchmarks
2+
3+
`Sources/ZeroBenchmarks` has ports of the JS client's benchmarks, under the
4+
same names, and benchmarks of the Swift client's own:
5+
6+
```sh
7+
cd swift
8+
swift run -c release ZeroBenchmarks # everything, about a minute
9+
swift run -c release ZeroBenchmarks hydration "pokes > put 100" --time 3 --json out.json
10+
```
11+
12+
A filter is a substring of `group > benchmark`. Each benchmark runs for
13+
`--time` seconds (default 1) after a warm-up and reports the median time per
14+
call with the 75th and 99th percentiles, as `packages/shared/src/bench.ts`
15+
does.
16+
17+
The JS sides:
18+
19+
| Swift group | JS |
20+
| ------------------------------------- | ------------------------------------------------------------------------------------------- |
21+
| `zero.bench.ts` | `pnpm --filter zero-client bench` |
22+
| `btree-set.bench.ts`, `hash.bench.ts` | `pnpm --filter shared bench src/btree-set.bench.ts src/hash.bench.ts` |
23+
| `memory`, `hydration`, `mutations` | `pnpm --filter zero-conformance bench` (`src/bench/client.bench.ts`) |
24+
| `pokes`, the round trip | none: the JS client applies pokes on the next frame, so a JS timing measures the scheduling |
25+
26+
The benchmarks don't run in CI; numbers from shared runners vary too much to
27+
gate on. CI builds them, so they keep compiling.
28+
29+
## Results
30+
31+
Linux, one machine (8-core Intel Xeon at 2.6 GHz), Swift 6.2.4 in release
32+
mode, Node 22.23. Medians; lower is better.
33+
34+
### The client
35+
36+
| Benchmark | JS | Swift |
37+
| ------------------------------------------ | ------- | ------- |
38+
| zero.bench: all 1000 rows x 10 columns | 263 µs | 220 µs |
39+
| zero.bench: pk = N | 7.72 µs | 7.68 µs |
40+
| zero.bench: lower 500 rows | 261 µs | 213 µs |
41+
| materialize 1,000 rows | 282 µs | 135 µs |
42+
| materialize 10,000 rows | 2.70 ms | 1.69 ms |
43+
| materialize 100,000 rows | 43.2 ms | 23.2 ms |
44+
| memory for 100,000 rows and a view of them | 164 MB | 98 MB |
45+
| optimistic update with a 1000-row view | 178 µs | 21 µs |
46+
47+
Swift only:
48+
49+
| Benchmark | Swift |
50+
| --------------------------------------------------------- | ------- |
51+
| materialize 1,000 rows as structs | 552 µs |
52+
| materialize 10,000 rows as structs | 6.19 ms |
53+
| materialize 100,000 rows as structs | 85.5 ms |
54+
| optimistic update with a 1000-row live query of structs | 92 µs |
55+
| poke putting 1 row, 1000-row view | 28 µs |
56+
| poke putting 100 rows, 1000-row view | 1.08 ms |
57+
| poke putting 1000 rows, 1000-row view | 10.7 ms |
58+
| round trip (update, push, confirming poke), 1000-row view | 79 µs |
59+
60+
JS views hand out their rows; Swift apps read them as structs, which costs
61+
about four times the materialization. A live query reads only the rows
62+
that changed after the first read.
63+
64+
### Data structures
65+
66+
| Benchmark | JS | Swift |
67+
| -------------------------------------- | --------------------------- | ------------------------ |
68+
| BTreeSet values() full scan (1000) | 3.60 µs | 2.67 µs |
69+
| BTreeSet valuesFrom() from mid | 2.35 µs | 1.40 µs |
70+
| BTreeSet valuesReversed() full scan | 4.29 µs | 0.92 µs |
71+
| BTreeSet valuesFromReversed() from mid | 2.55 µs | 0.59 µs |
72+
| BTreeSet has() hit (x3) | 67 ns | 199 ns |
73+
| BTreeSet has() miss (x3) | 63 ns | 91 ns |
74+
| BTreeSet get() hit (x3) | 70 ns | 193 ns |
75+
| BTreeSet add() then delete() | 109 ns | 160 ns |
76+
| BTreeSet add() 100 sequential keys | 6.29 µs | 4.62 µs |
77+
| BTreeSet add() 1000 sequential keys | 63.6 µs | 71.5 µs |
78+
| BTreeSet sort, then add() 1000 keys | 104 µs | 42 µs |
79+
| h32, 64 / 1024 / 8192 chars | 290 ns / 2.78 µs / 23.7 µs | 23 ns / 168 ns / 1.26 µs |
80+
| h64, 64 / 1024 / 8192 chars | 552 ns / 5.46 µs / 41.3 µs | 35 ns / 318 ns / 2.45 µs |
81+
| h128, 64 / 1024 / 8192 chars | 1.02 µs / 10.2 µs / 82.0 µs | 54 ns / 628 ns / 5.0 µs |
82+
83+
As `hash.bench.ts` notes, vitest keeps imported functions from being
84+
inlined, so its JS hash numbers are pessimistic; bundled, they are a few
85+
times faster.
86+
87+
## What they found
88+
89+
- The mutation tracker looked mutations up by scanning a list, so each
90+
mutation cost more than the last while mutations were pending (229 µs on
91+
average over a one-second run, now 21 µs).
92+
- A live query read every row again on every change (now only the changed
93+
ones: 92 µs instead of about 640 µs for 1000 rows).
94+
- B-tree nodes paid for dynamic exclusivity checks on every read, and the
95+
forward iterator read its leaf's keys through the node for every element
96+
(a full scan of 1000 took 23 µs, now 2.7 µs). Views and sources iterate
97+
B-trees, so materializing got faster too.
98+
- Hydrating a view took its row list out of the view and put it back for
99+
every row.
100+
- Comparing strings in UTF-8 order walked them with iterators rather than
101+
their bytes.
102+
103+
Where Swift is still slower: B-tree lookups call their comparator through
104+
a closure, which V8 inlines. And a row is a `[String: JSONValue]`, so
105+
reading a column hashes its name: comparing rows by key and reading rows as
106+
structs mostly hash. Rows keyed by column index would remove most of that.

‎swift/DESIGN.md‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ swift/ Swift package (source of truth)
5757
Sources/ZeroUI/ SwiftUI, UIKit, AppKit helpers; URLSession transport
5858
Sources/ZeroTesting/ drift-harness replay API and test helpers
5959
Sources/ZeroNIOTesting/ SwiftNIO WebSocket transport for Linux end-to-end tests
60+
Sources/ZeroBenchmarks/ benchmarks (BENCHMARKS.md)
6061
Tests/...
6162
VERSION
6263
packages/zero-conformance/ private JS package: fixtures, vector generators, JS runners
@@ -485,7 +486,9 @@ the Zero convention of passing IDs and timestamps as arguments.
485486
(`zero-client/src/client/zero.bench.ts`, the shared B-tree and hash
486487
benchmarks) plus Swift benchmarks for query hydration over N rows, poke
487488
apply, mutation round trip, and memory per 100k rows. Tracked next to the JS
488-
numbers, non-blocking in v1, with a goal of no slower than JS.
489+
numbers, non-blocking in v1, with a goal of no slower than JS. CI builds
490+
them; [BENCHMARKS.md](BENCHMARKS.md) has how to run them and the latest
491+
numbers.
489492

490493
## Dogfood app: `apps/zbugs-swift`
491494

‎swift/Package.swift‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,8 @@ let package = Package(
3131
.product(name: "NIOHTTP1", package: "swift-nio"),
3232
.product(name: "NIOWebSocket", package: "swift-nio"),
3333
]),
34+
// `swift run -c release ZeroBenchmarks`.
35+
.executableTarget(name: "ZeroBenchmarks", dependencies: ["Zero", "ZeroTesting"]),
3436
.testTarget(name: "ZeroTests", dependencies: ["Zero", "ZeroTesting"]),
3537
.testTarget(name: "ConformanceTests", dependencies: ["Zero", "ZeroTesting"]),
3638
// The Apple layer: the URLSession transport, SwiftUI, UIKit, and AppKit.

0 commit comments

Comments
 (0)