Skip to content

Commit c3da8c9

Browse files
ampagenttmm
andcommitted
ci(zbugs-swift): warm the app up before the screenshot tests
Every failure of testListPagesPastTheFirst200 since Nov 3 was the first UI test after the simulator booted: a back tap lost, a launch query or a screenshot request timed out. Its launch took 9-79 s on CI against 3-12 s for testScreens on the warm simulator, which hasn't failed, and XCTest answers queries on the app's main thread, which the first sync keeps busy. The job now launches the app once after the boot and waits until it has taken in its first sync and gone quiet, then the tests run on the same build. On iPhone the test also taps back again if the filters haven't appeared and the back button is still there. Amp-Thread-ID: https://ampcode.com/threads/T-fbae32eb-2a12-4ebe-a716-b1b0bf7c246f Co-authored-by: tmm <tmm@tmm.dev>
1 parent e4f199a commit c3da8c9

2 files changed

Lines changed: 57 additions & 4 deletions

File tree

‎.github/workflows/zbugs-screenshots.yml‎

Lines changed: 49 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -139,16 +139,64 @@ jobs:
139139
xcrun simctl shutdown all
140140
xcrun simctl bootstatus "$1" -b
141141
}
142+
# The app launched once on the simulator, outside XCTest, until it
143+
# has taken in its first sync and gone quiet. On this runner the
144+
# first launch after a boot and the first sync take up to a minute
145+
# and a half, and XCTest answers its queries on the app's main
146+
# thread, which they keep busy. A test that started on the cold app
147+
# had a query time out, a back tap lost, or a screenshot request
148+
# time out; the ones that started on a warm app haven't failed.
149+
# Best effort, for up to 4 minutes: the tests check the rest.
150+
warm() {
151+
xcodebuild build-for-testing -project ZBugs.xcodeproj -scheme ZBugsApp \
152+
-derivedDataPath "$RUNNER_TEMP/dd" -destination "id=$1" \
153+
-skipPackagePluginValidation CODE_SIGNING_ALLOWED=NO -quiet || return 0
154+
xcrun simctl install "$1" "$RUNNER_TEMP/dd/Build/Products/Debug-iphonesimulator/ZBugs.app" ||
155+
return 0
156+
preloads() { grep -c 'queryName=issuePreloadV2' "$RUNNER_TEMP/zbugs.log" || true; }
157+
# Seconds of CPU the app has used.
158+
cpu() {
159+
{ ps -o time= -p "$pid" || true; } |
160+
awk -F: '{ s = 0; for (i = 1; i <= NF; i++) s = s * 60 + $i; print s }'
161+
}
162+
before="$(preloads)"
163+
start=$SECONDS
164+
pid="$(SIMCTL_CHILD_ZBUGS_TOKEN="$TEST_RUNNER_ZBUGS_TOKEN" \
165+
xcrun simctl launch "$1" dev.rocicorp.zbugs | awk '{ print $NF }')" || return 0
166+
# zero-cache has sent the app its preload...
167+
while [ "$(preloads)" -le "$before" ] && [ $((SECONDS - start)) -lt 240 ]; do
168+
sleep 2
169+
done
170+
# ...and the app has taken it in: under 0.2 s of CPU in 3 s, three
171+
# times running.
172+
quiet=0
173+
while [ "$quiet" -lt 3 ] && [ $((SECONDS - start)) -lt 240 ]; do
174+
was="$(cpu)"
175+
sleep 3
176+
if awk -v a="$(cpu)" -v b="$was" 'BEGIN { exit !(a - b < 0.2) }'; then
177+
quiet=$((quiet + 1))
178+
else
179+
quiet=0
180+
fi
181+
done
182+
echo "Warmed up the app in $((SECONDS - start)) s"
183+
xcrun simctl terminate "$1" dev.rocicorp.zbugs || true
184+
}
142185
# Every run, even after one fails, so each keeps its screenshots.
143186
status=0
144187
screenshots() {
145188
name="$1"
146189
shift
147190
case "$1 $2" in
148-
"-destination id="*) boot "${2#id=}" ;;
191+
"-destination id="*)
192+
boot "${2#id=}"
193+
# Offline, the app has no sync to take in.
194+
case "$name" in *-offline) ;; *) warm "${2#id=}" ;; esac
195+
;;
149196
*) xcrun simctl shutdown all ;;
150197
esac
151198
xcodebuild test -project ZBugs.xcodeproj -scheme ZBugsApp \
199+
-derivedDataPath "$RUNNER_TEMP/dd" \
152200
-resultBundlePath "$RUNNER_TEMP/results/$name.xcresult" \
153201
-skipPackagePluginValidation "$@" || status=1
154202
}

‎apps/zbugs-swift/App/UITests/ScreenshotTests.swift‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -120,9 +120,6 @@ final class ScreenshotTests: XCTestCase {
120120
guard expect(app, rows.firstMatch, "an issue in the list", timeout: 120) else { return }
121121

122122
// Status: All.
123-
if isPhone {
124-
back(app)
125-
}
126123
#if os(macOS)
127124
let status = app.popUpButtons.matching(NSPredicate(format: "value == 'Open'")).firstMatch
128125
let all = app.menuItems["All"]
@@ -131,6 +128,14 @@ final class ScreenshotTests: XCTestCase {
131128
.firstMatch
132129
let all = app.buttons["All"]
133130
#endif
131+
// A tap while the app is still taking in its first sync can highlight
132+
// the back button and not go back (CI's iPhone did twice), so tap it
133+
// again if the filters haven't appeared and the list is still there.
134+
if isPhone {
135+
back(app)
136+
let backButton = app.navigationBars.buttons["BackButton"]
137+
if !status.waitForExistence(timeout: 10), backButton.exists { press(backButton) }
138+
}
134139
guard expect(app, status, "the Status picker") else { return }
135140
press(status)
136141
guard expect(app, all, "the All status") else { return }

0 commit comments

Comments
 (0)