|
| 1 | +import Foundation |
| 2 | +import Testing |
| 3 | +import Zero |
| 4 | + |
| 5 | +/// Random chaos against the real zero-cache. Each seed runs two clients, on |
| 6 | +/// stores of their own, through a `ChaosProxy`, and takes 40 random steps: |
| 7 | +/// mutations, connections cut, refused, stalled, or cut partway through |
| 8 | +/// what the server sends, writes to Postgres behind the clients' back, and |
| 9 | +/// relaunches on the same store. Then the network heals, and the oracle is |
| 10 | +/// convergence: Postgres holds every mutation once, each client's last |
| 11 | +/// mutation ID counts all of its mutations, and both clients see what |
| 12 | +/// Postgres holds. |
| 13 | +/// |
| 14 | +/// PRs run `ZERO_CHAOS_SEEDS` seeds (2 by default) from |
| 15 | +/// `ZERO_CHAOS_FIRST_SEED` (0); the nightly workflow runs more. To replay |
| 16 | +/// one: `ZERO_CHAOS_SEED=<seed> ZERO_E2E_URL=... swift test --filter ChaosFuzzTests`. |
| 17 | +@Suite(.enabled(if: E2E.url != nil), .serialized, .timeLimit(.minutes(60))) |
| 18 | +@MainActor |
| 19 | +struct ChaosFuzzTests { |
| 20 | + @Test func randomSeeds() async throws { |
| 21 | + let env = ProcessInfo.processInfo.environment |
| 22 | + let seeds: Range<UInt64> |
| 23 | + if let only = env["ZERO_CHAOS_SEED"].flatMap(UInt64.init) { |
| 24 | + seeds = only..<only + 1 |
| 25 | + } else { |
| 26 | + let first = env["ZERO_CHAOS_FIRST_SEED"].flatMap(UInt64.init) ?? 0 |
| 27 | + seeds = first..<first + (env["ZERO_CHAOS_SEEDS"].flatMap(UInt64.init) ?? 2) |
| 28 | + } |
| 29 | + var failed = 0 |
| 30 | + for seed in seeds { |
| 31 | + let start = ContinuousClock.now |
| 32 | + let failure = try await ChaosRun(seed: seed).run() |
| 33 | + print( |
| 34 | + "[chaos fuzz] seed \(seed): \(failure ?? "converged") in \(ContinuousClock.now - start)") |
| 35 | + guard let failure else { continue } |
| 36 | + Issue.record("seed \(seed): \(failure)") |
| 37 | + failed += 1 |
| 38 | + if failed == 3 { return } |
| 39 | + } |
| 40 | + } |
| 41 | +} |
| 42 | + |
| 43 | +/// SplitMix64: the same steps for a seed on every platform. |
| 44 | +struct ChaosRandom: RandomNumberGenerator { |
| 45 | + var state: UInt64 |
| 46 | + mutating func next() -> UInt64 { |
| 47 | + state &+= 0x9E37_79B9_7F4A_7C15 |
| 48 | + var z = state |
| 49 | + z = (z ^ (z >> 30)) &* 0xBF58_476D_1CE4_E5B9 |
| 50 | + z = (z ^ (z >> 27)) &* 0x94D0_49BB_1331_11EB |
| 51 | + return z ^ (z >> 31) |
| 52 | + } |
| 53 | +} |
| 54 | + |
| 55 | +/// One seed's clients, proxy, and model of what Postgres should end up with. |
| 56 | +@MainActor |
| 57 | +final class ChaosRun { |
| 58 | + /// A client and the relaunches on its store. |
| 59 | + @MainActor final class Lineage { |
| 60 | + let storageKey: String |
| 61 | + var zero: Zero<Void> |
| 62 | + var open: LiveQuery<[TestIssue]> |
| 63 | + /// What the client now running created and hasn't closed: only it may |
| 64 | + /// close them, since its mutations apply in order. |
| 65 | + var closable: [String] = [] |
| 66 | + |
| 67 | + init(zero: Zero<Void>, storageKey: String) throws { |
| 68 | + self.zero = zero |
| 69 | + self.storageKey = storageKey |
| 70 | + open = try zero.liveQuery(E2EApp.open()) |
| 71 | + } |
| 72 | + } |
| 73 | + |
| 74 | + let seed: UInt64 |
| 75 | + var rng: ChaosRandom |
| 76 | + let chaos = Chaos() |
| 77 | + var proxy: ChaosProxy! |
| 78 | + var lineages: [Lineage] = [] |
| 79 | + /// The open issues Postgres should end with. |
| 80 | + var expectedOpen: Set<String> = [] |
| 81 | + var created = 0 |
| 82 | + /// Mutations made by each client ID. |
| 83 | + var mutations: [String: Int] = [:] |
| 84 | + /// Issues written straight to Postgres and still open. |
| 85 | + var written: [String] = [] |
| 86 | + var steps: [String] = [] |
| 87 | + |
| 88 | + init(seed: UInt64) { |
| 89 | + self.seed = seed |
| 90 | + rng = ChaosRandom(state: seed) |
| 91 | + } |
| 92 | + |
| 93 | + func makeZero(_ storageKey: String) throws -> Zero<Void> { |
| 94 | + try chaos.makeZero(cacheURL: proxy.cacheURL, storageKey: storageKey) { |
| 95 | + $0.pingTimeoutMs = 2000 |
| 96 | + } |
| 97 | + } |
| 98 | + |
| 99 | + /// The problems at the end; nil when everything converged. |
| 100 | + func run() async throws -> String? { |
| 101 | + defer { chaos.cleanUp() } |
| 102 | + proxy = try await ChaosProxy(upstream: E2E.url!) |
| 103 | + for _ in 0..<2 { |
| 104 | + let key = UUID().uuidString |
| 105 | + lineages.append(try Lineage(zero: try makeZero(key), storageKey: key)) |
| 106 | + } |
| 107 | + defer { |
| 108 | + for l in lineages { l.zero.close() } |
| 109 | + } |
| 110 | + for l in lineages { try await chaos.connected(l.zero) } |
| 111 | + for n in 0..<40 { |
| 112 | + steps.append("\(n): " + (try await step(n))) |
| 113 | + try await Task.sleep(for: .milliseconds(Int.random(in: 0...100, using: &rng))) |
| 114 | + } |
| 115 | + await proxy.set { |
| 116 | + $0 = ChaosProxy.Knobs() |
| 117 | + } |
| 118 | + let problems = try await converge() |
| 119 | + await proxy.shutdown() |
| 120 | + guard !problems.isEmpty else { return nil } |
| 121 | + return problems.joined(separator: "; ") |
| 122 | + + " (last steps: \(steps.suffix(8).joined(separator: " | ")))" |
| 123 | + } |
| 124 | + |
| 125 | + func mutate(_ l: Lineage, _ request: MutateRequest<Void>) { |
| 126 | + _ = l.zero.mutate(request) |
| 127 | + mutations[l.zero.clientID, default: 0] += 1 |
| 128 | + } |
| 129 | + |
| 130 | + func step(_ n: Int) async throws -> String { |
| 131 | + let i = Int.random(in: 0..<lineages.count, using: &rng) |
| 132 | + let l = lineages[i] |
| 133 | + switch Int.random(in: 0..<100, using: &rng) { |
| 134 | + case 0..<35: |
| 135 | + let id = "\(chaos.prefix)c\(n)" |
| 136 | + mutate(l, E2EApp.create(.init(id: id, title: "Chaos \(n)", ownerId: nil))) |
| 137 | + l.closable.append(id) |
| 138 | + expectedOpen.insert(id) |
| 139 | + created += 1 |
| 140 | + return "client \(i) creates \(id)" |
| 141 | + case 35..<45: |
| 142 | + guard !l.closable.isEmpty else { return "client \(i) has nothing to close" } |
| 143 | + let id = l.closable.remove(at: Int.random(in: 0..<l.closable.count, using: &rng)) |
| 144 | + mutate(l, E2EApp.close(id)) |
| 145 | + expectedOpen.remove(id) |
| 146 | + return "client \(i) closes \(id)" |
| 147 | + case 45..<53: |
| 148 | + await proxy.cutAll() |
| 149 | + return "cut every connection" |
| 150 | + case 53..<58: |
| 151 | + let ms = Int.random(in: 100...1500, using: &rng) |
| 152 | + await proxy.set { $0.accept = .refuse } |
| 153 | + await proxy.cutAll() |
| 154 | + try await Task.sleep(for: .milliseconds(ms)) |
| 155 | + await proxy.set { $0.accept = .pass } |
| 156 | + return "down for \(ms) ms" |
| 157 | + case 58..<63: |
| 158 | + let ms = Int.random(in: 200...3000, using: &rng) |
| 159 | + await proxy.set { $0.downstreamRate = 0 } |
| 160 | + try await Task.sleep(for: .milliseconds(ms)) |
| 161 | + await proxy.set { $0.downstreamRate = nil } |
| 162 | + return "server stalled for \(ms) ms" |
| 163 | + case 63..<68: |
| 164 | + let bytes = Int.random(in: 500...50_000, using: &rng) |
| 165 | + let ms = Int.random(in: 200...1500, using: &rng) |
| 166 | + await proxy.set { $0.cutAfterDownstreamBytes = bytes } |
| 167 | + try await Task.sleep(for: .milliseconds(ms)) |
| 168 | + await proxy.set { $0.cutAfterDownstreamBytes = nil } |
| 169 | + return "cuts after \(bytes) bytes for \(ms) ms" |
| 170 | + case 68..<76: |
| 171 | + let id = "\(chaos.prefix)pg\(n)" |
| 172 | + try Postgres.query( |
| 173 | + """ |
| 174 | + INSERT INTO issues (id, title, description, closed, owner_id, "createdAt") |
| 175 | + VALUES ('\(id)', 'Written \(n)', '', false, NULL, 2000) |
| 176 | + """) |
| 177 | + written.append(id) |
| 178 | + expectedOpen.insert(id) |
| 179 | + created += 1 |
| 180 | + return "Postgres inserts \(id)" |
| 181 | + case 76..<82: |
| 182 | + guard !written.isEmpty else { return "Postgres has nothing to close" } |
| 183 | + let id = written.remove(at: Int.random(in: 0..<written.count, using: &rng)) |
| 184 | + try Postgres.query("UPDATE issues SET closed = true WHERE id = '\(id)'") |
| 185 | + expectedOpen.remove(id) |
| 186 | + return "Postgres closes \(id)" |
| 187 | + case 82..<90: |
| 188 | + l.zero.close() |
| 189 | + l.zero = try makeZero(l.storageKey) |
| 190 | + l.open = try l.zero.liveQuery(E2EApp.open()) |
| 191 | + l.closable = [] |
| 192 | + return "client \(i) relaunches" |
| 193 | + default: |
| 194 | + let ms = Int.random(in: 0...500, using: &rng) |
| 195 | + try await Task.sleep(for: .milliseconds(ms)) |
| 196 | + return "wait \(ms) ms" |
| 197 | + } |
| 198 | + } |
| 199 | + |
| 200 | + func ids(_ l: Lineage) -> Set<String> { |
| 201 | + Set(l.open.data.map(\.id).filter { $0.hasPrefix(chaos.prefix) }) |
| 202 | + } |
| 203 | + |
| 204 | + func short(_ got: Set<String>, _ want: Set<String>) -> String { |
| 205 | + let missing = want.subtracting(got).sorted() |
| 206 | + let extra = got.subtracting(want).sorted() |
| 207 | + return "missing \(missing.prefix(5)), extra \(extra.prefix(5))" |
| 208 | + } |
| 209 | + |
| 210 | + /// Waits for the clients and Postgres to settle, then compares them with |
| 211 | + /// the model. |
| 212 | + func converge() async throws -> [String] { |
| 213 | + var problems: [String] = [] |
| 214 | + // Every mutation pushed first: until then, a create closed in the same |
| 215 | + // run leaves the open issues right but is missing from Postgres. |
| 216 | + for (clientID, count) in mutations.sorted(by: { $0.key < $1.key }) { |
| 217 | + let ok = try await within(.seconds(30)) { try Postgres.lastMutationID(clientID) == count } |
| 218 | + if !ok { |
| 219 | + let last = try Postgres.lastMutationID(clientID).map(String.init) ?? "none" |
| 220 | + problems.append( |
| 221 | + "client \(clientID) made \(count) mutations; its last mutation ID is \(last)") |
| 222 | + } |
| 223 | + } |
| 224 | + let inPostgres = try await within(.seconds(30)) { |
| 225 | + try Postgres.openIssueIDs(self.chaos.prefix) == self.expectedOpen |
| 226 | + && Postgres.issues(self.chaos.prefix).count == self.created |
| 227 | + } |
| 228 | + if !inPostgres { |
| 229 | + let open = try Postgres.openIssueIDs(chaos.prefix) |
| 230 | + if open != expectedOpen { |
| 231 | + problems.append("Postgres's open issues: \(short(open, expectedOpen))") |
| 232 | + } |
| 233 | + let all = try Postgres.issues(chaos.prefix).count |
| 234 | + if all != created { problems.append("\(all) issues in Postgres, \(created) created") } |
| 235 | + } |
| 236 | + for (i, l) in lineages.enumerated() { |
| 237 | + let ok = try await within(.seconds(60)) { |
| 238 | + l.open.resultType == .complete && self.ids(l) == self.expectedOpen |
| 239 | + } |
| 240 | + if !ok { |
| 241 | + problems.append( |
| 242 | + "client \(i) (\(l.zero.connection.state.status)) sees \(short(ids(l), expectedOpen))") |
| 243 | + } |
| 244 | + } |
| 245 | + return problems |
| 246 | + } |
| 247 | +} |
0 commit comments