diff --git a/packages/playwright/src/common/ipc.ts b/packages/playwright/src/common/ipc.ts index 4d9ca7f846280..89f17bad4edb1 100644 --- a/packages/playwright/src/common/ipc.ts +++ b/packages/playwright/src/common/ipc.ts @@ -84,11 +84,15 @@ export type AttachmentPayload = { stepId?: string; }; +export type TestErrorsPayload = { + testId: string; + errors: TestInfoErrorImpl[]; +}; + export type TestInfoErrorImpl = TestInfoError; export type TestPausedPayload = { testId: string; - errors: TestInfoErrorImpl[]; }; export type CustomMessageRequestPayload = { @@ -105,7 +109,6 @@ export type TestEndPayload = { testId: string; duration: number; status: TestStatus; - errors: TestInfoErrorImpl[]; hasNonRetriableError: boolean; expectedStatus: TestStatus; annotations: { type: string, description?: string }[]; diff --git a/packages/playwright/src/isomorphic/teleReceiver.ts b/packages/playwright/src/isomorphic/teleReceiver.ts index cef009c730941..d0fced2ece384 100644 --- a/packages/playwright/src/isomorphic/teleReceiver.ts +++ b/packages/playwright/src/isomorphic/teleReceiver.ts @@ -96,7 +96,8 @@ export type JsonTestResultEnd = { id: string; duration: number; status: reporterTypes.TestStatus; - errors: reporterTypes.TestError[]; + /** No longer emitted, but kept for backwards compatibility */ + errors?: reporterTypes.TestError[]; /** No longer emitted, but kept for backwards compatibility */ attachments?: JsonAttachment[]; annotations?: TestAnnotation[]; @@ -132,7 +133,7 @@ export type JsonFullResult = { }; export type JsonEvent = JsonOnConfigureEvent | JsonOnBlobReportMetadataEvent | JsonOnEndEvent | JsonOnExitEvent | JsonOnProjectEvent | JsonOnBeginEvent | JsonOnTestBeginEvent - | JsonOnTestEndEvent | JsonOnStepBeginEvent | JsonOnStepEndEvent | JsonOnAttachEvent | JsonOnErrorEvent | JsonOnStdIOEvent; + | JsonOnTestEndEvent | JsonOnStepBeginEvent | JsonOnStepEndEvent | JsonOnAttachEvent | JsonOnTestErrorEvent | JsonOnErrorEvent | JsonOnStdIOEvent; export type JsonOnConfigureEvent = { method: 'onConfigure'; @@ -198,6 +199,15 @@ export type JsonOnAttachEvent = { params: JsonTestResultOnAttach; }; +export type JsonOnTestErrorEvent = { + method: 'onTestError'; + params: { + testId: string; + resultId: string; + error: reporterTypes.TestError; + } +}; + export type JsonOnErrorEvent = { method: 'onError'; params: { @@ -294,6 +304,10 @@ export class TeleReporterReceiver { this._onAttach(params.testId, params.resultId, params.attachments); return; } + if (method === 'onTestError') { + this._onTestError(params.testId, params.resultId, params.error); + return; + } if (method === 'onStepEnd') { this._onStepEnd(params.testId, params.resultId, params.step); return; @@ -353,8 +367,11 @@ export class TeleReporterReceiver { const result = test.results.find(r => r._id === payload.id)!; result.duration = payload.duration; result.status = payload.status; - result.errors = payload.errors; - result.error = result.errors?.[0]; + // Errors are only present here from legacy blobs. These override all _onTestError events + if (!!payload.errors) { + result.errors = payload.errors; + result.error = result.errors[0]; + } // Attachments are only present here from legacy blobs. These override all _onAttach events if (!!payload.attachments) result.attachments = this._parseAttachments(payload.attachments); @@ -404,6 +421,13 @@ export class TeleReporterReceiver { }))); } + private _onTestError(testId: string, resultId: string, error: reporterTypes.TestError) { + const test = this._tests.get(testId)!; + const result = test.results.find(r => r._id === resultId)!; + result.errors.push(error); + result.error = result.errors[0]; + } + private _onError(error: reporterTypes.TestError) { this._reporter.onError?.(error); } diff --git a/packages/playwright/src/reporters/internalReporter.ts b/packages/playwright/src/reporters/internalReporter.ts index 946c16932ba2f..c48bfdf70b609 100644 --- a/packages/playwright/src/reporters/internalReporter.ts +++ b/packages/playwright/src/reporters/internalReporter.ts @@ -67,8 +67,12 @@ export class InternalReporter implements ReporterV2 { this._reporter.onStdErr?.(chunk, test, result); } + onTestError(test: TestCase, result: TestResult, error: TestError): void { + addLocationAndSnippetToError(this._config, error, test.location.file); + this._reporter.onTestError?.(test, result, error); + } + onTestEnd(test: TestCase, result: TestResult) { - this._addSnippetToTestErrors(test, result); this._reporter.onTestEnd?.(test, result); } @@ -89,7 +93,7 @@ export class InternalReporter implements ReporterV2 { } onError(error: TestError) { - addLocationAndSnippetToError(this._config, error); + addLocationAndSnippetToError(this._config, error, undefined); this._reporter.onError?.(error); } @@ -98,26 +102,17 @@ export class InternalReporter implements ReporterV2 { } onStepEnd(test: TestCase, result: TestResult, step: TestStep) { - this._addSnippetToStepError(test, step); + if (step.error) + addLocationAndSnippetToError(this._config, step.error, test.location.file); this._reporter.onStepEnd?.(test, result, step); } printsToStdio() { return this._reporter.printsToStdio ? this._reporter.printsToStdio() : true; } - - private _addSnippetToTestErrors(test: TestCase, result: TestResult) { - for (const error of result.errors) - addLocationAndSnippetToError(this._config, error, test.location.file); - } - - private _addSnippetToStepError(test: TestCase, step: TestStep) { - if (step.error) - addLocationAndSnippetToError(this._config, step.error, test.location.file); - } } -export function addLocationAndSnippetToError(config: FullConfig, error: TestError, file?: string) { +export function addLocationAndSnippetToError(config: FullConfig, error: TestError, file: string | undefined) { if (error.stack && !error.location) error.location = prepareErrorStack(error.stack).location; const location = error.location; diff --git a/packages/playwright/src/reporters/merge.ts b/packages/playwright/src/reporters/merge.ts index c0ed7fd6e90da..3571f6297290c 100644 --- a/packages/playwright/src/reporters/merge.ts +++ b/packages/playwright/src/reporters/merge.ts @@ -404,6 +404,7 @@ class IdsPatcher { case 'onProject': this._onProject(params.project); return; + case 'onTestError': case 'onAttach': case 'onTestBegin': case 'onStepBegin': @@ -498,7 +499,7 @@ class PathSeparatorPatcher { test.annotations?.forEach(annotation => this._updateAnnotationLocation(annotation)); const testResult = jsonEvent.params.result; testResult.annotations?.forEach(annotation => this._updateAnnotationLocation(annotation)); - testResult.errors.forEach(error => this._updateErrorLocations(error)); + testResult.errors?.forEach(error => this._updateErrorLocations(error)); (testResult.attachments ?? []).forEach(attachment => { if (attachment.path) attachment.path = this._updatePath(attachment.path); @@ -516,6 +517,10 @@ class PathSeparatorPatcher { step.annotations?.forEach(annotation => this._updateAnnotationLocation(annotation)); return; } + if (jsonEvent.method === 'onTestError') { + this._updateErrorLocations(jsonEvent.params.error); + return; + } if (jsonEvent.method === 'onAttach') { const attach = jsonEvent.params; attach.attachments.forEach(attachment => { diff --git a/packages/playwright/src/reporters/multiplexer.ts b/packages/playwright/src/reporters/multiplexer.ts index 2a11b9358319a..0c8bb5ff14f74 100644 --- a/packages/playwright/src/reporters/multiplexer.ts +++ b/packages/playwright/src/reporters/multiplexer.ts @@ -54,6 +54,11 @@ export class Multiplexer implements ReporterV2 { wrap(() => reporter.onStdErr?.(chunk, test, result)); } + onTestError(test: TestCase, result: TestResult, error: TestError) { + for (const reporter of this._reporters) + wrap(() => reporter.onTestError?.(test, result, error)); + } + onTestEnd(test: TestCase, result: TestResult) { for (const reporter of this._reporters) wrap(() => reporter.onTestEnd?.(test, result)); diff --git a/packages/playwright/src/reporters/reporterV2.ts b/packages/playwright/src/reporters/reporterV2.ts index 280ef2bfd2f19..5085a63a85bb9 100644 --- a/packages/playwright/src/reporters/reporterV2.ts +++ b/packages/playwright/src/reporters/reporterV2.ts @@ -22,6 +22,7 @@ export interface ReporterV2 { onTestBegin?(test: TestCase, result: TestResult): void; onStdOut?(chunk: string | Buffer, test?: TestCase, result?: TestResult): void; onStdErr?(chunk: string | Buffer, test?: TestCase, result?: TestResult): void; + onTestError?(test: TestCase, result: TestResult, error: TestError): void; onTestEnd?(test: TestCase, result: TestResult): void; onEnd?(result: FullResult): Promise<{ status?: FullResult['status'] } | undefined | void> | void; onExit?(): void | Promise; diff --git a/packages/playwright/src/reporters/teleEmitter.ts b/packages/playwright/src/reporters/teleEmitter.ts index d191104b7099b..8a792d1849251 100644 --- a/packages/playwright/src/reporters/teleEmitter.ts +++ b/packages/playwright/src/reporters/teleEmitter.ts @@ -71,6 +71,17 @@ export class TeleReporterEmitter implements ReporterV2 { }); } + onTestError(test: reporterTypes.TestCase, result: reporterTypes.TestResult, error: reporterTypes.TestError): void { + this._messageSink({ + method: 'onTestError', + params: { + testId: test.id, + resultId: (result as any)[this._idSymbol], + error, + } + }); + } + onTestEnd(test: reporterTypes.TestCase, result: reporterTypes.TestResult): void { const testEnd: teleReceiver.JsonTestEnd = { testId: test.id, @@ -248,7 +259,6 @@ export class TeleReporterEmitter implements ReporterV2 { id: (result as any)[this._idSymbol], duration: result.duration, status: result.status, - errors: result.errors, annotations: result.annotations?.length ? this._relativeAnnotationLocations(result.annotations) : undefined, }; } diff --git a/packages/playwright/src/runner/dispatcher.ts b/packages/playwright/src/runner/dispatcher.ts index 358f639ac01aa..1e8958569f96c 100644 --- a/packages/playwright/src/runner/dispatcher.ts +++ b/packages/playwright/src/runner/dispatcher.ts @@ -28,7 +28,7 @@ import type { ProcessExitData } from './processHost'; import type { TestGroup } from './testGroups'; import type { TestError, TestResult, TestStep } from '../../types/testReporter'; import type { FullConfigInternal } from '../common/config'; -import type { AttachmentPayload, DonePayload, RunPayload, SerializedConfig, StepBeginPayload, StepEndPayload, TeardownErrorsPayload, TestBeginPayload, TestEndPayload, TestOutputPayload, TestPausedPayload } from '../common/ipc'; +import type { AttachmentPayload, DonePayload, RunPayload, SerializedConfig, StepBeginPayload, StepEndPayload, TeardownErrorsPayload, TestBeginPayload, TestEndPayload, TestErrorsPayload, TestOutputPayload, TestPausedPayload } from '../common/ipc'; import type { Suite } from '../common/test'; import type { TestCase } from '../common/test'; import type { ReporterV2 } from '../reporters/reporterV2'; @@ -322,7 +322,6 @@ class JobDispatcher { // Do not show more than one error to avoid confusion, but report // as interrupted to indicate that we did actually start the test. params.status = 'interrupted'; - params.errors = []; } const data = this._dataByTestId.get(params.testId); if (!data) { @@ -333,8 +332,6 @@ class JobDispatcher { this._remainingByTestId.delete(params.testId); const { result, test } = data; result.duration = params.duration; - result.errors = params.errors; - result.error = result.errors[0]; result.status = params.status; result.annotations = params.annotations; test.annotations = [...params.annotations]; // last test result wins @@ -429,6 +426,23 @@ class JobDispatcher { } } + private _onTestErrors(params: TestErrorsPayload) { + if (this._failureTracker.hasReachedMaxFailures()) { + // Do not show more than one error to avoid confusion. + return; + } + + const data = this._dataByTestId.get(params.testId)!; + if (!data) + return; + const { test, result } = data; + for (const error of params.errors) { + result.errors.push(error); + result.error = result.errors[0]; + this._reporter.onTestError?.(test, result, error); + } + } + private _failTestWithErrors(test: TestCase, errors: TestError[]) { const runData = this._dataByTestId.get(test.id); // There might be a single test that has started but has not finished yet. @@ -439,8 +453,11 @@ class JobDispatcher { result = test._appendTestResult(); this._reporter.onTestBegin?.(test, result); } - result.errors = [...errors]; - result.error = result.errors[0]; + for (const error of errors) { + result.errors.push(error); + result.error = result.errors[0]; + this._reporter.onTestError?.(test, result, error); + } result.status = errors.length ? 'failed' : 'skipped'; this._reportTestEnd(test, result); this._failedTests.add(test); @@ -578,6 +595,7 @@ class JobDispatcher { eventsHelper.addEventListener(worker, 'stepBegin', this._onStepBegin.bind(this)), eventsHelper.addEventListener(worker, 'stepEnd', this._onStepEnd.bind(this)), eventsHelper.addEventListener(worker, 'attach', this._onAttach.bind(this)), + eventsHelper.addEventListener(worker, 'testErrors', this._onTestErrors.bind(this)), eventsHelper.addEventListener(worker, 'testPaused', this._onTestPaused.bind(this, worker)), eventsHelper.addEventListener(worker, 'done', this._onDone.bind(this)), eventsHelper.addEventListener(worker, 'exit', this.onExit.bind(this)), @@ -585,24 +603,28 @@ class JobDispatcher { } private _onTestPaused(worker: WorkerHost, params: TestPausedPayload) { + const data = this._dataByTestId.get(params.testId); + if (!data) + return; + + const { test, result } = data; + const sendMessage = async (message: { request: any }) => { try { if (this.jobResult.isDone()) throw new Error('Test has already stopped'); const response = await worker.sendCustomMessage({ testId: params.testId, request: message.request }); if (response.error) - addLocationAndSnippetToError(this._config.config, response.error); + addLocationAndSnippetToError(this._config.config, response.error, test.location.file); return response; } catch (e) { const error = serializeError(e); - addLocationAndSnippetToError(this._config.config, error); + addLocationAndSnippetToError(this._config.config, error, test.location.file); return { response: undefined, error }; } }; - for (const error of params.errors) - addLocationAndSnippetToError(this._config.config, error); - this._failureTracker.onTestPaused?.({ ...params, sendMessage }); + this._failureTracker.onTestPaused?.({ errors: result.errors, sendMessage }); } skipWholeJob(): boolean { diff --git a/packages/playwright/src/runner/testRunner.ts b/packages/playwright/src/runner/testRunner.ts index ec2397e31fee8..b7b80c6bca14b 100644 --- a/packages/playwright/src/runner/testRunner.ts +++ b/packages/playwright/src/runner/testRunner.ts @@ -470,7 +470,9 @@ export async function runAllTestsWithConfig(config: FullConfigInternal): Promise createLoadTask('in-process', { filterOnly: true, failOnLoadErrors: true }), ...createRunTestsTasks(config), ]; - const status = await runTasks(new TestRun(config, reporter), tasks, config.config.globalTimeout); + + const testRun = new TestRun(config, reporter, { pauseAtEnd: config.configCLIOverrides.debug, pauseOnError: config.configCLIOverrides.debug }); + const status = await runTasks(testRun, tasks, config.config.globalTimeout); // Calling process.exit() might truncate large stdout/stderr output. // See https://github.com/nodejs/node/issues/6456. diff --git a/packages/playwright/src/transform/babelHighlightUtils.ts b/packages/playwright/src/transform/babelHighlightUtils.ts new file mode 100644 index 0000000000000..4611233686377 --- /dev/null +++ b/packages/playwright/src/transform/babelHighlightUtils.ts @@ -0,0 +1,45 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import path from 'path'; +import { traverse, babelParse, T, types as t } from './babelBundle'; +import type { Location } from '../../types/testReporter'; + +function containsLocation(range: T.SourceLocation, location: Location): boolean { + if (location.line < range.start.line || location.line > range.end.line) + return false; + if (location.line === range.start.line && location.column < range.start.column) + return false; + if (location.line === range.end.line && location.column > range.end.column) + return false; + return true; +} + +export function findTestEndLocation(text: string, testStartLocation: Location): Location | undefined { + const ast = babelParse(text, path.basename(testStartLocation.file), false); + let result: Location | undefined; + traverse(ast, { + enter(path) { + if (t.isCallExpression(path.node) && path.node.loc && containsLocation(path.node.loc, testStartLocation)) { + const callNode = path.node; + const funcNode = callNode.arguments[callNode.arguments.length - 1]; + if (callNode.arguments.length >= 2 && t.isFunction(funcNode) && funcNode.body.loc) + result = { file: testStartLocation.file, ...funcNode.body.loc.end }; + } + } + }); + return result; +} diff --git a/packages/playwright/src/worker/testInfo.ts b/packages/playwright/src/worker/testInfo.ts index c9fd959dd9fab..5838b28a57a29 100644 --- a/packages/playwright/src/worker/testInfo.ts +++ b/packages/playwright/src/worker/testInfo.ts @@ -24,12 +24,13 @@ import { addSuffixToFilePath, filteredStackTrace, getContainedPath, normalizeAnd import { TestTracing } from './testTracing'; import { testInfoError } from './util'; import { wrapFunctionWithLocation } from '../transform/transform'; +import { findTestEndLocation } from '../transform/babelHighlightUtils'; import type { RunnableDescription } from './timeoutManager'; import type { FullProject, TestInfo, TestStatus, TestStepInfo, TestAnnotation } from '../../types/test'; import type { FullConfig, Location } from '../../types/testReporter'; import type { FullConfigInternal, FullProjectInternal } from '../common/config'; -import type { AttachmentPayload, StepBeginPayload, StepEndPayload, TestInfoErrorImpl, TestPausedPayload, WorkerInitParams } from '../common/ipc'; +import type { AttachmentPayload, StepBeginPayload, StepEndPayload, TestErrorsPayload, TestInfoErrorImpl, TestPausedPayload, WorkerInitParams } from '../common/ipc'; import type { TestCase } from '../common/test'; import type { StackFrame } from '@protocol/channels'; @@ -69,6 +70,7 @@ export class TestInfoImpl implements TestInfo { private _onStepBegin: (payload: StepBeginPayload) => void; private _onStepEnd: (payload: StepEndPayload) => void; private _onAttach: (payload: AttachmentPayload) => void; + private _onErrors: (errors: TestErrorsPayload) => void; private _onTestPaused: (payload: TestPausedPayload) => void; private _snapshotNames: SnapshotNames = { lastAnonymousSnapshotIndex: 0, lastNamedSnapshotIndex: {} }; private _ariaSnapshotNames: SnapshotNames = { lastAnonymousSnapshotIndex: 0, lastNamedSnapshotIndex: {} }; @@ -121,6 +123,7 @@ export class TestInfoImpl implements TestInfo { readonly outputDir: string; readonly snapshotDir: string; errors: TestInfoErrorImpl[] = []; + private _reportedErrorCount = 0; readonly _attachmentsPush: (...items: TestInfo['attachments']) => number; private _workerParams: WorkerInitParams; @@ -164,12 +167,14 @@ export class TestInfoImpl implements TestInfo { onStepBegin: (payload: StepBeginPayload) => void, onStepEnd: (payload: StepEndPayload) => void, onAttach: (payload: AttachmentPayload) => void, + onErrors: (payload: TestErrorsPayload) => void, onTestPaused: (payload: TestPausedPayload) => void, ) { this.testId = test?.id ?? ''; this._onStepBegin = onStepBegin; this._onStepEnd = onStepEnd; this._onAttach = onAttach; + this._onErrors = onErrors; this._onTestPaused = onTestPaused; this._startTime = monotonicTime(); this._startWallTime = Date.now(); @@ -406,7 +411,7 @@ export class TestInfoImpl implements TestInfo { this._tracing.appendForError(serialized); } - async _runAsStep(stepInfo: { title: string, category: 'hook' | 'fixture', location?: Location, group?: string }, cb: () => Promise) { + async _runAsStep(stepInfo: { title: string, category: 'hook' | 'fixture' | 'test.step', location?: Location, group?: string }, cb: () => Promise) { const step = this._addStep(stepInfo); try { await cb(); @@ -464,12 +469,35 @@ export class TestInfoImpl implements TestInfo { async _didFinishTestFunction() { const shouldPause = (this._workerParams.pauseAtEnd && !this._isFailure()) || (this._workerParams.pauseOnError && this._isFailure()); if (shouldPause) { - this._onTestPaused({ testId: this.testId, errors: this._isFailure() ? this.errors : [] }); - await this._interruptedPromise; + const location = (this._isFailure() ? this._errorLocation() : await this._testEndLocation()) ?? { file: this.file, line: this.line, column: this.column }; + this._emitErrors(); + this._onTestPaused({ testId: this.testId }); + await this._runAsStep({ title: this._isFailure() ? 'Paused on Error' : 'Paused at End', category: 'test.step', location }, async () => { + await this._interruptedPromise; + }); } await this._onDidFinishTestFunctionCallback?.(); } + _emitErrors() { + const errors = this.errors.slice(this._reportedErrorCount); + this._reportedErrorCount = Math.max(this._reportedErrorCount, this.errors.length); + if (errors.length) + this._onErrors({ testId: this.testId, errors }); + } + + private _errorLocation(): Location | undefined { + if (this.error?.stack) + return filteredStackTrace(this.error.stack.split('\n'))[0]; + } + + private async _testEndLocation(): Promise { + try { + const source = await fs.promises.readFile(this.file, 'utf-8'); + return findTestEndLocation(source, { file: this.file, line: this.line, column: this.column }); + } catch {} + } + // ------------ TestInfo methods ------------ async attach(name: string, options: { path?: string, body?: string | Buffer, contentType?: string } = {}) { diff --git a/packages/playwright/src/worker/workerMain.ts b/packages/playwright/src/worker/workerMain.ts index f1a96cbdd848d..b49d6482daf30 100644 --- a/packages/playwright/src/worker/workerMain.ts +++ b/packages/playwright/src/worker/workerMain.ts @@ -118,7 +118,7 @@ export class WorkerMain extends ProcessRunner { return; } // Ignore top-level errors, they are already inside TestInfo.errors. - const fakeTestInfo = new TestInfoImpl(this._config, this._project, this._params, undefined, 0, () => {}, () => {}, () => {}, () => {}); + const fakeTestInfo = new TestInfoImpl(this._config, this._project, this._params, undefined, 0, () => {}, () => {}, () => {}, () => {}, () => {}); const runnable = { type: 'teardown' } as const; // We have to load the project to get the right deadline below. await fakeTestInfo._runWithTimeout(runnable, () => this._loadIfNeeded()).catch(() => {}); @@ -283,6 +283,7 @@ export class WorkerMain extends ProcessRunner { stepBeginPayload => this.dispatchEvent('stepBegin', stepBeginPayload), stepEndPayload => this.dispatchEvent('stepEnd', stepEndPayload), attachment => this.dispatchEvent('attach', attachment), + errors => this.dispatchEvent('testErrors', errors), testPausedPayload => this.dispatchEvent('testPaused', testPausedPayload)); const processAnnotation = (annotation: TestAnnotation) => { @@ -331,6 +332,7 @@ export class WorkerMain extends ProcessRunner { if (isSkipped && nextTest && !hasAfterAllToRunBeforeNextTest) { // Fast path - this test is skipped, and there are more tests that will handle cleanup. testInfo.status = 'skipped'; + testInfo._emitErrors(); this.dispatchEvent('testEnd', buildTestEndPayload(testInfo)); return; } @@ -497,6 +499,7 @@ export class WorkerMain extends ProcessRunner { this._currentTest = null; setCurrentTestInfo(null); + testInfo._emitErrors(); this.dispatchEvent('testEnd', buildTestEndPayload(testInfo)); const preserveOutput = this._config.config.preserveOutput === 'always' || @@ -622,7 +625,6 @@ function buildTestEndPayload(testInfo: TestInfoImpl): TestEndPayload { testId: testInfo.testId, duration: testInfo.duration, status: testInfo.status!, - errors: testInfo.errors, hasNonRetriableError: testInfo._hasNonRetriableError, expectedStatus: testInfo.expectedStatus, annotations: testInfo.annotations, diff --git a/tests/playwright-test/pause-at-end.spec.ts b/tests/playwright-test/pause-at-end.spec.ts new file mode 100644 index 0000000000000..6021f70225f8c --- /dev/null +++ b/tests/playwright-test/pause-at-end.spec.ts @@ -0,0 +1,125 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Reporter, TestCase, TestResult, TestStep } from 'packages/playwright-test/reporter'; +import { test, expect, parseTestRunnerOutput } from './playwright-test-fixtures'; + +class LocationReporter implements Reporter { + onStepBegin(test: TestCase, result: TestResult, step: TestStep): void { + if (step.title.startsWith('Paused')) { + console.log('\n'); + console.log(`%%${step.title} at :${step.location?.line}:${step.location?.column}`); + if (result.error) + console.log(`%%result.error at :${result.error.location?.line}:${result.error.location?.column}`); + for (const [index, error] of result.errors.entries()) + console.log(`%%result.errors[${index}] at :${error.location?.line}:${error.location?.column}`); + console.log('\n'); + } + } +} + +test('--debug should pause at end', async ({ interactWithTestRunner, }) => { + test.skip(process.platform === 'win32', 'No sending SIGINT on Windows'); + const testProcess = await interactWithTestRunner({ + 'location-reporter.js': `export default ${LocationReporter}`, + 'playwright.config.js': ` + module.exports = { reporter: [['list'], ['./location-reporter.js']] }; + `, + 'a.test.js': ` + import { test, expect } from '@playwright/test'; + test('pass', () => { + }); + test.afterEach(() => { + console.log('teardown'.toUpperCase()); // uppercase so we dont confuse it with source snippets + }); + ` + }, { debug: true }, { PLAYWRIGHT_FORCE_TTY: 'true' }); + await testProcess.waitForOutput('Paused at End'); + await testProcess.kill('SIGINT'); + expect(testProcess.output).toContain('TEARDOWN'); + expect(testProcess.outputLines()).toEqual(['Paused at End at :4:7']); + + const result = parseTestRunnerOutput(testProcess.output); + expect(result.interrupted).toBe(1); +}); + +test('--debug should pause at end with setup project', async ({ interactWithTestRunner, }) => { + test.skip(process.platform === 'win32', 'No sending SIGINT on Windows'); + const testProcess = await interactWithTestRunner({ + 'location-reporter.js': `export default ${LocationReporter}`, + 'playwright.config.js': ` + module.exports = { + reporter: [['list'], ['./location-reporter.js']], + projects: [ + { name: 'setup', testMatch: /setup\\.test\\.js/ }, + { name: 'main', dependencies: ['setup'] } + ] + }; + `, + 'setup.test.js': ` + import { test } from '@playwright/test'; + test('setup', () => { + }); + `, + 'a.test.js': ` + import { test, expect } from '@playwright/test'; + test('pass', () => { + console.log('main test started'); + }); + ` + }, { debug: true }, { PLAYWRIGHT_FORCE_TTY: 'true' }); + await testProcess.waitForOutput('main test started'); + await testProcess.waitForOutput('Paused at End'); + await testProcess.kill('SIGINT'); + expect(testProcess.outputLines()).toEqual(['Paused at End at :5:7']); + + const result = parseTestRunnerOutput(testProcess.output); + expect(result.interrupted).toBe(1); +}); + +test('--debug should pause on error', async ({ interactWithTestRunner, mergeReports }) => { + test.skip(process.platform === 'win32', 'No sending SIGINT on Windows'); + const testProcess = await interactWithTestRunner({ + 'location-reporter.js': `export default ${LocationReporter}`, + 'playwright.config.js': ` + module.exports = { reporter: [['list'], ['blob'], ['./location-reporter.js']] }; + `, + 'a.test.js': ` + import { test, expect } from '@playwright/test'; + test('pass', () => { + expect.soft(1).toBe(2); + expect(2).toBe(3); + console.log('after error'.toUpperCase()); + }); + ` + }, { debug: true }, { PLAYWRIGHT_FORCE_TTY: 'true' }); + await testProcess.waitForOutput('Paused on Error'); + expect(testProcess.output).not.toContain('AFTER ERROR'); + await testProcess.kill('SIGINT'); + expect(testProcess.output).not.toContain('AFTER ERROR'); + expect(testProcess.outputLines()).toEqual([ + 'Paused on Error at :4:24', + 'result.error at :4:24', + 'result.errors[0] at :4:24', + 'result.errors[1] at :5:19', + ]); + + const result = parseTestRunnerOutput(testProcess.output); + expect(result.failed).toBe(1); + + const merged = await mergeReports('blob-report', undefined, { additionalArgs: ['--reporter', 'location-reporter.js'] }); + expect(merged.outputLines).toEqual(testProcess.outputLines()); +}); diff --git a/tests/playwright-test/timeout.spec.ts b/tests/playwright-test/timeout.spec.ts index 9d6001078d0c2..03b626d9cef79 100644 --- a/tests/playwright-test/timeout.spec.ts +++ b/tests/playwright-test/timeout.spec.ts @@ -143,8 +143,8 @@ test('should respect test.slow', async ({ runInlineTest }) => { expect(result.output).toContain('Test timeout of 1000ms exceeded.'); }); -test('should ignore test.setTimeout when debugging', async ({ runInlineTest }) => { - const result = await runInlineTest({ +test('should ignore test.setTimeout when debugging', async ({ interactWithTestRunner }) => { + const testProcess = await interactWithTestRunner({ 'a.spec.ts': ` import { test as base, expect } from '@playwright/test'; const test = base.extend({ @@ -159,15 +159,15 @@ test('should ignore test.setTimeout when debugging', async ({ runInlineTest }) = await new Promise(f => setTimeout(f, 2000)); }); ` - }, { debug: true }); - expect(result.exitCode).toBe(0); - expect(result.passed).toBe(1); + }, { debug: true }, { PLAYWRIGHT_FORCE_TTY: 'true' }); + await testProcess.waitForOutput('Paused at End'); + await testProcess.kill('SIGINT'); }); test('should ignore globalTimeout when debugging', { annotation: { type: 'issue', description: 'https://github.com/microsoft/playwright/issues/34911' }, -}, async ({ runInlineTest }) => { - const result = await runInlineTest({ +}, async ({ interactWithTestRunner }) => { + const testProcess = await interactWithTestRunner({ 'playwright.config.ts': ` export default { globalTimeout: 100, @@ -179,9 +179,9 @@ test('should ignore globalTimeout when debugging', { await new Promise(f => setTimeout(f, 2000)); }); ` - }, { debug: true }); - expect(result.exitCode).toBe(0); - expect(result.passed).toBe(1); + }, { debug: true }, { PLAYWRIGHT_FORCE_TTY: 'true' }); + await testProcess.waitForOutput('Paused at End'); + await testProcess.kill('SIGINT'); }); test('should respect fixture timeout', async ({ runInlineTest }) => {