Skip to content

Commit 99168aa

Browse files
committed
feat(zero-swift): the inspector, query metrics, and slowMaterializeThreshold
zero.inspector ports @rocicorp/zero's Inspector over the sync connection's inspect messages. Where JS prompts for the admin password, a request throws InspectorError.authenticationRequired until authenticate(password:) succeeds; the accepted password is presented again on an unauthenticated later connection. The client records JS's query metrics in a port of shared/src/tdigest.ts, with fdlibm's asin and sin so its quantiles match Node's bit for bit, and ZeroOptions.slowMaterializeThreshold logs slow end-to-end materializations.
1 parent 8f4916a commit 99168aa

15 files changed

Lines changed: 2624 additions & 7 deletions

‎.changeset/zero-swift-inspector.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
'zero-swift': patch
3+
---
4+
5+
Add JS's inspector and query metrics. `zero.inspector` lists the client's and its group's queries on the server, with their client and server timings and ZQL, reads the clients the local store knows and their local data, reports the client's query metrics merged with the server's, shows zero-cache's version, and analyzes a query, a server AST, or a named query on the server. Where JS asks for zero-cache's admin password in a dialog, a request throws `InspectorError.authenticationRequired`: call `zero.inspector.authenticate(password:)` first, and the password it accepts is presented again after a reconnect. The client records JS's query metrics (`query-materialization-client`, `query-materialization-end-to-end`, `query-update-client`) in a `TDigest` that gives the same quantiles as JS's, and `ZeroOptions.slowMaterializeThreshold` (5 seconds by default) logs a warning for a query that takes longer to materialize, server and network included.
Lines changed: 219 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,219 @@
1+
/// The query builder code of an AST after its table, such as
2+
/// `.where('id', '=', 123)` (`astToZQL` in `packages/ast-to-zql/src/ast-to-zql.ts`,
3+
/// without `redactLiterals`). The code is JS's, as the JS inspector shows it.
4+
func astToZQL(_ ast: AST) -> String {
5+
var code = ""
6+
if let condition = ast.where {
7+
var args = Set<String>()
8+
code += zqlCondition(condition, where: true, &args)
9+
}
10+
for related in ast.related ?? [] {
11+
if related.hidden == true {
12+
if let nested = related.subquery.related?.first { code += zqlRelated(nested) }
13+
} else {
14+
code += zqlRelated(related)
15+
}
16+
}
17+
// distinct and groupBy go before orderBy, which then orders their rows.
18+
if let groupBy = ast.groupBy {
19+
let columns = groupBy.map(jsStringLiteral).joined(separator: ", ")
20+
if let aggregate = ast.aggregate {
21+
code += ".groupBy(\(columns)).\(aggregate.fn.rawValue)(\(zqlAggregateArg(aggregate)))"
22+
} else {
23+
code += ".distinct(\(columns))"
24+
}
25+
} else if let distinct = ast.aggregate?.distinct {
26+
code += ".distinct(\(distinct.map(jsStringLiteral).joined(separator: ", ")))"
27+
}
28+
if let orderBy = ast.orderBy, !orderBy.isEmpty {
29+
for part in orderBy { code += ".orderBy('\(part.field)', '\(part.direction.rawValue)')" }
30+
}
31+
if let limit = ast.limit { code += ".limit(\(JSNumber.format(limit)))" }
32+
if let start = ast.start {
33+
code +=
34+
".start(\(JSONSerializer.stringify(.object(start.row)))"
35+
+ (start.exclusive ? "" : ", { inclusive: true }") + ")"
36+
}
37+
// A count of the distinct values ends the chain.
38+
if ast.aggregate?.distinct != nil { code += ".count()" }
39+
return code
40+
}
41+
42+
/// Whether a subquery has more than a table: JS tests these properties'
43+
/// truthiness, so an empty ordering counts and a limit of 0 doesn't.
44+
private func hasSubqueryProps(_ ast: AST) -> Bool {
45+
ast.where != nil || !(ast.related ?? []).isEmpty || ast.orderBy != nil
46+
|| (ast.limit.map { $0 != 0 && !$0.isNaN } ?? false)
47+
}
48+
49+
/// `wrap`: `code`, a builder-callback expression, at the top level of a
50+
/// `.where` (a callback of its own) or inside a callback (its names join the
51+
/// enclosing callback's).
52+
private func zqlWrap(
53+
_ isWhere: Bool, _ args: inout Set<String>, _ needed: [String], _ code: String
54+
) -> String {
55+
if isWhere {
56+
let names = Set(needed).sorted(by: StringCompare.utf16Less).joined(separator: ", ")
57+
return ".where(({\(names)}) => \(code))"
58+
}
59+
args.formUnion(needed)
60+
return code
61+
}
62+
63+
private func zqlCondition(
64+
_ condition: AST.Condition, where isWhere: Bool, _ args: inout Set<String>
65+
) -> String {
66+
let prefix = isWhere ? ".where" : "cmp"
67+
switch condition {
68+
case .simple(let op, let left, let right):
69+
let leftCode = zqlValue(left)
70+
let rightCode = zqlValue(right)
71+
let argsCode =
72+
op == .eq ? "\(leftCode), \(rightCode)" : "\(leftCode), '\(op.rawValue)', \(rightCode)"
73+
if case .json = left {
74+
// A JSON path operand only exists through the expression builder's `json()`.
75+
return zqlWrap(isWhere, &args, ["cmp", "json"], "cmp(\(argsCode))")
76+
}
77+
return "\(prefix)(\(argsCode))"
78+
case .and(let conditions), .or(let conditions):
79+
let type = condition.isAnd ? "and" : "or"
80+
if conditions.count == 1 { return zqlCondition(conditions[0], where: isWhere, &args) }
81+
if condition.isAnd {
82+
let parts = conditions.map { zqlCondition($0, where: isWhere, &args) }
83+
if isWhere { return parts.joined() }
84+
args.insert("and")
85+
return "and(" + parts.joined(separator: ", ") + ")"
86+
}
87+
// An OR gets a callback: of its own at the top level, the enclosing one
88+
// inside a callback.
89+
var own = Set<String>()
90+
let code: String
91+
if isWhere {
92+
code = conditions.map { zqlCondition($0, where: false, &own) }.joined(separator: ", ")
93+
} else {
94+
code = conditions.map { zqlCondition($0, where: false, &args) }.joined(separator: ", ")
95+
own = args
96+
}
97+
return zqlWrap(isWhere, &args, Array(own) + ["cmp", type], "\(type)(\(code))")
98+
case .correlatedSubquery(let related, let op, let flip, let scalar):
99+
let relationship = zqlRelationshipName(related)
100+
let next = nextExistsSubquery(related)
101+
let props = hasSubqueryProps(next)
102+
var options: [String] = []
103+
if let flip { options.append("flip: \(flip)") }
104+
if let scalar { options.append("scalar: \(scalar)") }
105+
let optionsCode = options.isEmpty ? "" : ", {\(options.joined(separator: ", "))}"
106+
let subquery = props ? ", q => q\(astToZQL(next))" : ""
107+
switch op {
108+
case .exists:
109+
if isWhere { return ".whereExists('\(relationship)'\(subquery)\(optionsCode))" }
110+
args.insert("exists")
111+
return "exists('\(relationship)'\(subquery)\(optionsCode))"
112+
case .notExists:
113+
return zqlWrap(
114+
isWhere, &args, ["exists", "not"],
115+
"not(exists('\(relationship)'\(subquery)\(optionsCode)))")
116+
}
117+
}
118+
}
119+
120+
extension AST.Condition {
121+
fileprivate var isAnd: Bool {
122+
if case .and = self { return true }
123+
return false
124+
}
125+
}
126+
127+
/// `getNextExistsSubquery`: an `exists` over a junction edge has both hops
128+
/// as exists conditions with the same alias; the code names the far one.
129+
private func nextExistsSubquery(_ related: AST.CorrelatedSubquery) -> AST {
130+
if case .correlatedSubquery(let inner, _, _, _)? = related.subquery.where,
131+
inner.subquery.alias?.contains(subqueryAliasPrefix + "zhidden_") == true
132+
{
133+
return nextExistsSubquery(inner)
134+
}
135+
return related.subquery
136+
}
137+
138+
private func zqlRelationshipName(_ related: AST.CorrelatedSubquery) -> String {
139+
let alias = related.subquery.alias ?? ""
140+
return alias.hasPrefix(subqueryAliasPrefix)
141+
? String(alias.dropFirst(subqueryAliasPrefix.count)) : alias
142+
}
143+
144+
private func zqlRelated(_ related: AST.CorrelatedSubquery) -> String {
145+
guard let alias = related.subquery.alias else { return "" }
146+
var code = ".related('\(alias)'"
147+
if hasSubqueryProps(related.subquery) { code += ", q => q" + astToZQL(related.subquery) }
148+
return code + ")"
149+
}
150+
151+
private func zqlValue(_ value: AST.ValuePosition) -> String {
152+
switch value {
153+
case .literal(let v): return zqlLiteral(v)
154+
case .column(let name): return "'\(name)'"
155+
case .json(let column, let path):
156+
return "json('\(column)', \(zqlPath(path)))"
157+
case .static(_, let field):
158+
switch field {
159+
case .single(let f): return "authParam('\(f)')"
160+
case .path(let fields):
161+
return "authParam([\(fields.map { "'\($0)'" }.joined(separator: ", "))])"
162+
}
163+
}
164+
}
165+
166+
private func zqlPath(_ path: [AST.JSONPathSegment]) -> String {
167+
path.map { segment in
168+
switch segment {
169+
case .key(let k): jsStringLiteral(k)
170+
case .index(let i): String(i)
171+
}
172+
}.joined(separator: ", ")
173+
}
174+
175+
/// `transformLiteral`: what `String()` prints, with strings quoted.
176+
private func zqlLiteral(_ value: JSONValue) -> String {
177+
switch value {
178+
case .null: "null"
179+
case .array: JSONSerializer.stringify(value)
180+
case .string(let s): jsStringLiteral(s)
181+
case .number(let n): JSNumber.format(n)
182+
case .bool(let b): b ? "true" : "false"
183+
case .object: "[object Object]"
184+
}
185+
}
186+
187+
/// `transformAggregateArg`: nothing for a count, a column, or a callback
188+
/// that picks a leaf of a `json()` column.
189+
private func zqlAggregateArg(_ aggregate: AST.Aggregate) -> String {
190+
guard let field = aggregate.field else { return "" }
191+
let name = jsStringLiteral(field)
192+
guard let path = aggregate.path else { return name }
193+
return "({json}) => json(\(name), \(zqlPath(path)))"
194+
}
195+
196+
/// A single-quoted JS string literal of `s`, escaped as `JSON.stringify`
197+
/// escapes it (`jsStringLiteral`).
198+
private func jsStringLiteral(_ s: String) -> String {
199+
let quoted = Array(JSONSerializer.stringify(.string(s)).unicodeScalars)
200+
var out = String.UnicodeScalarView()
201+
var i = 1
202+
let end = quoted.count - 1
203+
while i < end {
204+
let c = quoted[i]
205+
if c == "\\", i + 1 < end, quoted[i + 1] == "\"" {
206+
out.append("\"")
207+
i += 2
208+
continue
209+
}
210+
if c == "'" {
211+
out.append("\\")
212+
out.append("'")
213+
} else {
214+
out.append(c)
215+
}
216+
i += 1
217+
}
218+
return "'" + String(out) + "'"
219+
}

‎swift/Sources/Zero/Client/Engine.swift‎

Lines changed: 53 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,10 @@ public struct ZeroOptions: Sendable {
7575
/// hydration), with its listener calls. It must call the function it is
7676
/// given, once, before it returns.
7777
public var batchViewUpdates: (@MainActor @Sendable (() -> Void) -> Void)?
78+
/// How long, in milliseconds, a query may take to materialize, until the
79+
/// server has sent its rows, before a warning is logged
80+
/// (`slowMaterializeThreshold`). Faster ones are logged at debug level.
81+
public var slowMaterializeThreshold: Double = 5_000
7882

7983
public init(
8084
cacheURL: String? = nil, userID: String? = nil, auth: String? = nil,
@@ -145,6 +149,8 @@ final class Engine {
145149
private(set) var mutationTracker: MutationTracker!
146150
private(set) var queryManager: QueryManager!
147151
private var pokeHandler: PokeHandler!
152+
/// The inspector's requests (`zero.inspector`).
153+
let inspectRPCs = InspectRPCs()
148154

149155
/// The current token: `options.auth`, then the last one `connect(auth:)` set.
150156
private(set) var auth: String?
@@ -302,12 +308,17 @@ final class Engine {
302308
closedReason: { [weak self] in
303309
guard case .closed(let reason) = self?.connectionManager.state else { return nil }
304310
return reason
305-
})
311+
},
312+
addMetric: { [weak self] metric, value, queryID, ast in
313+
self?.queryManager.addMetric(metric, value, queryID: queryID, ast: ast)
314+
},
315+
monotonicNow: { [scheduler] in scheduler.monotonicNow })
306316
_ = connectionManager.subscribe { [weak self] state in self?.onConnectionStateChange(state) }
307317
onConnectionStateChange(connectionManager.state)
308318
queryManager = QueryManager(
309319
clientID: clientID, schema: schema, mutationTracker: mutationTracker, scheduler: scheduler,
310320
queryChangeThrottleMs: options.queryChangeThrottleMs, recentQueriesMaxSize: 0,
321+
slowMaterializeThreshold: options.slowMaterializeThreshold, log: log,
311322
send: { [weak self] body in
312323
guard let self else { return }
313324
var body = body
@@ -915,6 +926,7 @@ final class Engine {
915926
connectedAt = nil
916927
socket?.close(code: closeCode)
917928
socket = nil
929+
inspectRPCs.connectionLost(reason)
918930
lastMutationIDsSent.removeAll()
919931
pokeHandler.handleDisconnect()
920932
queryManager.clearGotQueriesAuthoritative()
@@ -1047,8 +1059,8 @@ final class Engine {
10471059
try mutationTracker.processPushResponse(body)
10481060
case .transformError(let errors):
10491061
queryManager.handleTransformErrors(errors)
1050-
case .inspect:
1051-
break
1062+
case .inspect(let body):
1063+
inspectRPCs.receive(body)
10521064
}
10531065
} catch {
10541066
disconnect(.client(.internal, message: errorMessage(error)))
@@ -1265,6 +1277,44 @@ final class Engine {
12651277
database.release()
12661278
database.removeWriteFailureListener(clientID)
12671279
connectionManager.closed()
1280+
inspectRPCs.clientClosed()
1281+
}
1282+
1283+
/// The socket of the connection, once the current attempt has made it
1284+
/// (the inspector's `getSocket`).
1285+
func inspectorSocket(_ done: @escaping (Result<SocketHandle, InspectorError>) -> Void) {
1286+
if repClosed { return done(.failure(.clientClosed)) }
1287+
// Swift only: JS waits for a connection that never comes.
1288+
if socketOrigin == nil {
1289+
return done(.failure(.connectionLost("No server socket origin provided")))
1290+
}
1291+
let waiter = inspectRPCs.addSocketWaiter(done)
1292+
connectionReady.onSettle(loop) { [weak self] connected in
1293+
guard let self, let done = self.inspectRPCs.takeSocketWaiter(waiter) else { return }
1294+
if connected, let socket = self.socket {
1295+
done(.success(socket))
1296+
} else {
1297+
let reason =
1298+
Self.connectionError(self.connectionManager.state)
1299+
?? self.connectionManager.state.reason
1300+
done(.failure(.connectionLost(reason?.message ?? "Not connected")))
1301+
}
1302+
}
1303+
}
1304+
1305+
/// Sends an inspector request on `socket`, which must still be the
1306+
/// connection's, and calls `done` with its response.
1307+
func sendInspect(
1308+
_ socket: SocketHandle, _ body: InspectUpBody,
1309+
_ done: @escaping (Result<InspectDownBody, InspectorError>) -> Void
1310+
) {
1311+
if repClosed { return done(.failure(.clientClosed)) }
1312+
guard socket === self.socket else {
1313+
return done(
1314+
.failure(.connectionLost(connectionManager.state.reason?.message ?? "Disconnected")))
1315+
}
1316+
inspectRPCs.expect(body.id, done)
1317+
socket.send(Upstream.inspect(body).text)
12681318
}
12691319

12701320
// MARK: - Storage and the client lifecycle

0 commit comments

Comments
 (0)